2023-09-23T14:31:19.819 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T14:31:19.819 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --owner scheduled_teuthology@teuthology --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691 --verbose --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/orig.config.yaml 2023-09-23T14:31:19.872 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T14:31:19.951 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401691' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi017.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi027.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:3.0 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T14:31:19.952 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T14:31:19.952 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T14:31:19.952 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T14:31:19.954 INFO:teuthology.task.internal:Checking packages... 2023-09-23T14:31:19.973 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T14:31:19.973 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:31:19.973 INFO:teuthology.packaging:ref: None 2023-09-23T14:31:19.973 INFO:teuthology.packaging:tag: None 2023-09-23T14:31:19.973 INFO:teuthology.packaging:branch: reef 2023-09-23T14:31:19.973 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:31:19.974 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T14:31:20.275 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T14:31:20.276 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T14:31:20.285 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T14:31:20.285 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T14:31:20.290 INFO:teuthology.task.internal:Saving configuration 2023-09-23T14:31:20.310 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T14:31:20.316 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T14:31:20.341 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi017.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 14:24:40.326010', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T14:31:20.366 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 14:24:40.324539', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T14:31:20.366 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T14:31:20.372 INFO:teuthology.task.internal:roles: ubuntu@smithi017.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T14:31:20.373 INFO:teuthology.task.internal:roles: ubuntu@smithi027.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T14:31:20.373 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T14:31:20.438 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0335cfbaf0>, signals=[15]) 2023-09-23T14:31:20.439 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T14:31:20.445 INFO:teuthology.task.internal:Opening connections... 2023-09-23T14:31:20.445 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi017.front.sepia.ceph.com 2023-09-23T14:31:20.446 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T14:31:20.514 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi027.front.sepia.ceph.com 2023-09-23T14:31:20.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T14:31:20.581 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T14:31:20.588 DEBUG:teuthology.orchestra.run.smithi017:> uname -m 2023-09-23T14:31:20.609 INFO:teuthology.orchestra.run.smithi017.stdout:x86_64 2023-09-23T14:31:20.609 DEBUG:teuthology.orchestra.run.smithi017:> cat /etc/os-release 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:ID="rhel" 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:ID_LIKE="fedora" 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:VERSION_ID="8.6" 2023-09-23T14:31:20.668 INFO:teuthology.orchestra.run.smithi017.stdout:PLATFORM_ID="platform:el8" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:ANSI_COLOR="0;31" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T14:31:20.669 INFO:teuthology.orchestra.run.smithi017.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T14:31:20.670 INFO:teuthology.lock.ops:Updating smithi017.front.sepia.ceph.com on lock server 2023-09-23T14:31:20.698 DEBUG:teuthology.orchestra.run.smithi027:> uname -m 2023-09-23T14:31:20.718 INFO:teuthology.orchestra.run.smithi027.stdout:x86_64 2023-09-23T14:31:20.719 DEBUG:teuthology.orchestra.run.smithi027:> cat /etc/os-release 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:ID="rhel" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:ID_LIKE="fedora" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:VERSION_ID="8.6" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:PLATFORM_ID="platform:el8" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:ANSI_COLOR="0;31" 2023-09-23T14:31:20.778 INFO:teuthology.orchestra.run.smithi027.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T14:31:20.779 INFO:teuthology.orchestra.run.smithi027.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T14:31:20.779 INFO:teuthology.lock.ops:Updating smithi027.front.sepia.ceph.com on lock server 2023-09-23T14:31:20.798 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T14:31:20.807 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T14:31:20.812 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T14:31:20.812 DEBUG:teuthology.orchestra.run.smithi017:> test '!' -e /home/ubuntu/cephtest 2023-09-23T14:31:20.814 DEBUG:teuthology.orchestra.run.smithi027:> test '!' -e /home/ubuntu/cephtest 2023-09-23T14:31:20.836 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T14:31:20.842 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T14:31:20.842 DEBUG:teuthology.orchestra.run.smithi017:> test -z $(ls -A /var/lib/ceph) 2023-09-23T14:31:20.874 DEBUG:teuthology.orchestra.run.smithi027:> test -z $(ls -A /var/lib/ceph) 2023-09-23T14:31:20.901 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T14:31:20.957 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T14:31:20.972 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T14:31:20.972 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T14:31:20.972 DEBUG:teuthology.orchestra.run.smithi017:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T14:31:20.973 DEBUG:teuthology.orchestra.run.smithi027:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T14:31:20.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T14:31:20.991 DEBUG:teuthology.orchestra.run.smithi027:> uname -r 2023-09-23T14:31:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T14:31:20.992 DEBUG:teuthology.orchestra.run.smithi017:> uname -r 2023-09-23T14:31:21.050 INFO:teuthology.orchestra.run.smithi027.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:21.050 INFO:teuthology.task.kernel:Running kernel on smithi027: 4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:21.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum install -y kernel 2023-09-23T14:31:21.053 INFO:teuthology.orchestra.run.smithi017.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:21.053 INFO:teuthology.task.kernel:Running kernel on smithi017: 4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:21.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum install -y kernel 2023-09-23T14:31:22.175 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:22.175 INFO:teuthology.orchestra.run.smithi027.stdout:Unable to read consumer identity 2023-09-23T14:31:22.288 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:22.288 INFO:teuthology.orchestra.run.smithi017.stdout:Unable to read consumer identity 2023-09-23T14:31:23.660 INFO:teuthology.orchestra.run.smithi027.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2023-09-23T14:31:23.904 INFO:teuthology.orchestra.run.smithi017.stdout:Copr repo for python3-asyncssh owned by ceph 12 kB/s | 3.5 kB 00:00 2023-09-23T14:31:24.335 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 24 MB/s | 16 MB 00:00 2023-09-23T14:31:24.604 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 23 MB/s | 16 MB 00:00 2023-09-23T14:31:28.094 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 429 kB/s | 24 kB 00:00 2023-09-23T14:31:28.271 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 437 kB/s | 24 kB 00:00 2023-09-23T14:31:28.830 INFO:teuthology.orchestra.run.smithi027.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T14:31:28.883 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:31:28.884 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:31:28.885 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:31:28.991 INFO:teuthology.orchestra.run.smithi017.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T14:31:29.012 DEBUG:teuthology.orchestra.run.smithi027:> echo no | sudo yum reinstall kernel || true 2023-09-23T14:31:29.029 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:31:29.031 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:31:29.031 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:31:29.152 DEBUG:teuthology.orchestra.run.smithi017:> echo no | sudo yum reinstall kernel || true 2023-09-23T14:31:29.365 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:29.365 INFO:teuthology.orchestra.run.smithi027.stdout:Unable to read consumer identity 2023-09-23T14:31:29.504 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:29.504 INFO:teuthology.orchestra.run.smithi017.stdout:Unable to read consumer identity 2023-09-23T14:31:29.639 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 02:31:28 PM UTC. 2023-09-23T14:31:29.682 INFO:teuthology.orchestra.run.smithi027.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T14:31:29.716 INFO:teuthology.orchestra.run.smithi027.stderr:Error: No packages marked for reinstall. 2023-09-23T14:31:29.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum reinstall -y kernel || true 2023-09-23T14:31:29.778 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:01 ago on Sat 23 Sep 2023 02:31:28 PM UTC. 2023-09-23T14:31:29.823 INFO:teuthology.orchestra.run.smithi017.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T14:31:29.855 INFO:teuthology.orchestra.run.smithi017.stderr:Error: No packages marked for reinstall. 2023-09-23T14:31:29.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum reinstall -y kernel || true 2023-09-23T14:31:30.112 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:30.112 INFO:teuthology.orchestra.run.smithi027.stdout:Unable to read consumer identity 2023-09-23T14:31:30.251 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:31:30.251 INFO:teuthology.orchestra.run.smithi017.stdout:Unable to read consumer identity 2023-09-23T14:31:30.383 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 02:31:28 PM UTC. 2023-09-23T14:31:30.428 INFO:teuthology.orchestra.run.smithi027.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T14:31:30.449 INFO:teuthology.orchestra.run.smithi027.stderr:Error: No packages marked for reinstall. 2023-09-23T14:31:30.500 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T14:31:30.520 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 02:31:28 PM UTC. 2023-09-23T14:31:30.563 INFO:teuthology.orchestra.run.smithi017.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T14:31:30.582 INFO:teuthology.orchestra.run.smithi027.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:30.583 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:30.583 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T14:31:30.583 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T14:31:30.583 DEBUG:teuthology.orchestra.run.smithi027:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T14:31:30.586 INFO:teuthology.orchestra.run.smithi017.stderr:Error: No packages marked for reinstall. 2023-09-23T14:31:30.614 INFO:teuthology.orchestra.run.smithi027.stdout:ttyS1 2023-09-23T14:31:30.626 DEBUG:teuthology.parallel:result is None 2023-09-23T14:31:30.636 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T14:31:30.718 INFO:teuthology.orchestra.run.smithi017.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:30.718 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T14:31:30.718 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T14:31:30.718 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T14:31:30.719 DEBUG:teuthology.orchestra.run.smithi017:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T14:31:30.749 INFO:teuthology.orchestra.run.smithi017.stdout:ttyS1 2023-09-23T14:31:30.767 DEBUG:teuthology.parallel:result is None 2023-09-23T14:31:30.767 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T14:31:30.774 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T14:31:30.775 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T14:31:30.809 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T14:31:30.827 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T14:31:30.833 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T14:31:30.841 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T14:31:30.842 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T14:31:30.866 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T14:31:30.888 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T14:31:30.895 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T14:31:30.895 DEBUG:teuthology.orchestra.run.smithi017:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T14:31:30.924 DEBUG:teuthology.orchestra.run.smithi027:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T14:31:30.952 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T14:31:30.958 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T14:31:30.963 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T14:31:30.968 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T14:31:30.970 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T14:31:30.977 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T14:31:30.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T14:31:31.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T14:31:31.040 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T14:31:31.048 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T14:31:31.049 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T14:31:31.081 DEBUG:teuthology.orchestra.run.smithi027:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T14:31:31.097 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T14:31:31.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T14:31:31.242 DEBUG:teuthology.orchestra.run.smithi017:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T14:31:31.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T14:31:31.327 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:31:31.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T14:31:31.402 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T14:31:31.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T14:31:31.499 DEBUG:teuthology.orchestra.run.smithi027:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T14:31:31.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T14:31:31.624 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:31:31.625 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T14:31:31.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo service rsyslog restart 2023-09-23T14:31:31.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo service rsyslog restart 2023-09-23T14:31:31.744 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T14:31:31.788 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T14:31:32.208 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T14:31:32.216 INFO:teuthology.task.internal:Starting timer... 2023-09-23T14:31:32.216 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T14:31:32.225 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T14:31:32.234 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T14:31:32.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo service auditd rotate 2023-09-23T14:31:32.310 INFO:teuthology.orchestra.run.smithi017.stdout:Rotating logs: 2023-09-23T14:31:32.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo service auditd rotate 2023-09-23T14:31:32.372 INFO:teuthology.orchestra.run.smithi027.stdout:Rotating logs: 2023-09-23T14:31:32.374 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T14:31:32.374 DEBUG:teuthology.orchestra.run.smithi017:> /usr/sbin/getenforce 2023-09-23T14:31:32.392 INFO:teuthology.orchestra.run.smithi017.stdout:Permissive 2023-09-23T14:31:32.392 DEBUG:teuthology.orchestra.run.smithi027:> /usr/sbin/getenforce 2023-09-23T14:31:32.418 INFO:teuthology.orchestra.run.smithi027.stdout:Permissive 2023-09-23T14:31:32.418 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi017.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi027.front.sepia.ceph.com': 'permissive'} 2023-09-23T14:31:32.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T14:31:32.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T14:31:32.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T14:31:32.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T14:31:32.487 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T14:31:32.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo /usr/sbin/setenforce permissive 2023-09-23T14:31:32.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo /usr/sbin/setenforce permissive 2023-09-23T14:31:32.556 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T14:31:32.565 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2023-09-23T14:31:32.570 INFO:teuthology.repo_utils:Fetching main from origin 2023-09-23T14:31:32.706 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T14:31:32.720 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T14:31:32.721 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi017.front.sepia.ceph.com,smithi027.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T14:41:26.618 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi017.front.sepia.ceph.com'), Remote(name='ubuntu@smithi027.front.sepia.ceph.com')] 2023-09-23T14:41:26.619 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi017.front.sepia.ceph.com' 2023-09-23T14:41:26.619 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi017.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T14:41:26.689 DEBUG:teuthology.orchestra.run.smithi017:> true 2023-09-23T14:41:26.760 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi017.front.sepia.ceph.com' 2023-09-23T14:41:26.761 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi027.front.sepia.ceph.com' 2023-09-23T14:41:26.761 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi027.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T14:41:26.832 DEBUG:teuthology.orchestra.run.smithi027:> true 2023-09-23T14:41:26.903 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi027.front.sepia.ceph.com' 2023-09-23T14:41:26.903 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T14:41:26.914 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T14:41:26.914 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T14:41:26.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T14:41:26.917 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T14:41:26.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T14:41:26.950 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T14:41:26.966 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T14:41:26.978 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T14:41:26.994 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T14:41:27.040 INFO:teuthology.orchestra.run.smithi017.stderr:sudo: ntpd: command not found 2023-09-23T14:41:27.052 INFO:teuthology.orchestra.run.smithi017.stdout:506 Cannot talk to daemon 2023-09-23T14:41:27.067 INFO:teuthology.orchestra.run.smithi027.stderr:sudo: ntpd: command not found 2023-09-23T14:41:27.067 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T14:41:27.079 INFO:teuthology.orchestra.run.smithi027.stdout:506 Cannot talk to daemon 2023-09-23T14:41:27.082 INFO:teuthology.orchestra.run.smithi017.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T14:41:27.095 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T14:41:27.109 INFO:teuthology.orchestra.run.smithi027.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T14:41:27.151 INFO:teuthology.orchestra.run.smithi017.stderr:bash: ntpq: command not found 2023-09-23T14:41:27.154 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T14:41:27.155 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2023-09-23T14:41:27.155 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.155 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.155 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.155 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.195 INFO:teuthology.orchestra.run.smithi027.stderr:bash: ntpq: command not found 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.199 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T14:41:27.199 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T14:41:27.211 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T14:41:27.211 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi017.front.sepia.ceph.com 2023-09-23T14:41:27.211 DEBUG:teuthology.orchestra.run.smithi017:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T14:41:27.212 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi027.front.sepia.ceph.com 2023-09-23T14:41:27.212 DEBUG:teuthology.orchestra.run.smithi027:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T14:41:27.604 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:41:27.629 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:41:29.176 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:29.295 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:31.210 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout:Resetting modules: 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout: container-tools 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:41:31.211 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:41:31.212 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:41:31.308 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout:Resetting modules: 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout: container-tools 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:41:31.309 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:41:31.310 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:41:32.210 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:32.287 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:41.473 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:41:41.952 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:41:42.965 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:41:43.434 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:41:43.597 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:06 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:44.860 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:41:45.557 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:41:45.659 INFO:teuthology.orchestra.run.smithi017.stdout:========================================================================================================================== 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout:========================================================================================================================== 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout:Installing group/module packages: 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:41:45.660 INFO:teuthology.orchestra.run.smithi017.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout:Downgrading: 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T14:41:45.661 INFO:teuthology.orchestra.run.smithi017.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:Installing module profiles: 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: container-tools/common 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling module streams: 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: container-tools 3.0 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:========================================================================================================================== 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:Install 7 Packages 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout:Downgrade 12 Packages 2023-09-23T14:41:45.662 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:41:45.663 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 51 M 2023-09-23T14:41:45.663 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:41:46.000 INFO:teuthology.orchestra.run.smithi017.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 152 kB/s | 51 kB 00:00 2023-09-23T14:41:46.192 INFO:teuthology.orchestra.run.smithi017.stdout:(2/19): containers-common-1.2.4-1.module+el8.6. 172 kB/s | 91 kB 00:00 2023-09-23T14:41:46.259 INFO:teuthology.orchestra.run.smithi017.stdout:(3/19): criu-3.15-1.module+el8.6.0+14874+644362 1.9 MB/s | 511 kB 00:00 2023-09-23T14:41:46.276 INFO:teuthology.orchestra.run.smithi017.stdout:(4/19): container-selinux-2.178.0-2.module+el8. 86 kB/s | 53 kB 00:00 2023-09-23T14:41:46.410 INFO:teuthology.orchestra.run.smithi017.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 318 kB/s | 69 kB 00:00 2023-09-23T14:41:46.435 INFO:teuthology.orchestra.run.smithi017.stdout:(6/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 454 kB/s | 72 kB 00:00 2023-09-23T14:41:46.569 INFO:teuthology.orchestra.run.smithi017.stdout:(7/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 9.3 MB/s | 2.9 MB 00:00 2023-09-23T14:41:46.669 INFO:teuthology.orchestra.run.smithi017.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 239 kB/s | 56 kB 00:00 2023-09-23T14:41:46.829 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:41:46.852 INFO:teuthology.orchestra.run.smithi017.stdout:(9/19): podman-3.0.1-9.module+el8.6.0+14874+644 25 MB/s | 11 MB 00:00 2023-09-23T14:41:46.929 INFO:teuthology.orchestra.run.smithi027.stdout:========================================================================================================================== 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout:========================================================================================================================== 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout:Installing group/module packages: 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: buildah x86_64 1.19.9-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.8 M 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: cockpit-podman noarch 29-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 1.1 M 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: crun x86_64 0.18-3.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 185 k 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: skopeo x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 6.4 M 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: toolbox x86_64 0.0.99.3-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T14:41:46.930 INFO:teuthology.orchestra.run.smithi027.stdout: udica noarch 0.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout:Downgrading: 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: conmon x86_64 2:2.0.26-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: container-selinux noarch 2:2.178.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 53 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: containernetworking-plugins x86_64 0.9.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 19 M 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: containers-common x86_64 1:1.2.4-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 91 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: criu x86_64 3.15-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 511 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: fuse-overlayfs x86_64 1.4.0-2.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 72 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: libslirp x86_64 4.3.1-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 69 k 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: podman x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 11 M 2023-09-23T14:41:46.931 INFO:teuthology.orchestra.run.smithi027.stdout: podman-catatonit x86_64 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 322 k 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: podman-docker noarch 3.0.1-9.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 56 k 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: runc x86_64 1.0.0-73.rc95.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 2.9 M 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: slirp4netns x86_64 1.1.8-1.module+el8.6.0+14874+64436299 rhel-8-for-x86_64-appstream-rpms 51 k 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout:Installing module profiles: 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: container-tools/common 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout:Enabling module streams: 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: container-tools 3.0 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout:========================================================================================================================== 2023-09-23T14:41:46.932 INFO:teuthology.orchestra.run.smithi027.stdout:Install 7 Packages 2023-09-23T14:41:46.933 INFO:teuthology.orchestra.run.smithi027.stdout:Downgrade 12 Packages 2023-09-23T14:41:46.933 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:41:46.933 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 51 M 2023-09-23T14:41:46.933 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:41:46.964 INFO:teuthology.orchestra.run.smithi017.stdout:(10/19): conmon-2.0.26-1.module+el8.6.0+14874+6 129 kB/s | 51 kB 00:00 2023-09-23T14:41:47.070 INFO:teuthology.orchestra.run.smithi017.stdout:(11/19): yajl-2.1.0-10.el8.x86_64.rpm 386 kB/s | 41 kB 00:00 2023-09-23T14:41:47.090 INFO:teuthology.orchestra.run.smithi027.stdout:(1/19): slirp4netns-1.1.8-1.module+el8.6.0+1487 328 kB/s | 51 kB 00:00 2023-09-23T14:41:47.107 INFO:teuthology.orchestra.run.smithi027.stdout:(2/19): container-selinux-2.178.0-2.module+el8. 305 kB/s | 53 kB 00:00 2023-09-23T14:41:47.112 INFO:teuthology.orchestra.run.smithi017.stdout:(12/19): podman-catatonit-3.0.1-9.module+el8.6. 1.2 MB/s | 322 kB 00:00 2023-09-23T14:41:47.133 INFO:teuthology.orchestra.run.smithi027.stdout:(3/19): containers-common-1.2.4-1.module+el8.6. 458 kB/s | 91 kB 00:00 2023-09-23T14:41:47.183 INFO:teuthology.orchestra.run.smithi027.stdout:(4/19): criu-3.15-1.module+el8.6.0+14874+644362 5.4 MB/s | 511 kB 00:00 2023-09-23T14:41:47.217 INFO:teuthology.orchestra.run.smithi027.stdout:(5/19): libslirp-4.3.1-1.module+el8.6.0+14874+6 634 kB/s | 69 kB 00:00 2023-09-23T14:41:47.337 INFO:teuthology.orchestra.run.smithi017.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 8.2 MB/s | 2.2 MB 00:00 2023-09-23T14:41:47.342 INFO:teuthology.orchestra.run.smithi027.stdout:(6/19): runc-1.0.0-73.rc95.module+el8.6.0+14874 14 MB/s | 2.9 MB 00:00 2023-09-23T14:41:47.367 INFO:teuthology.orchestra.run.smithi027.stdout:(7/19): fuse-overlayfs-1.4.0-2.module+el8.6.0+1 391 kB/s | 72 kB 00:00 2023-09-23T14:41:47.467 INFO:teuthology.orchestra.run.smithi027.stdout:(8/19): podman-docker-3.0.1-9.module+el8.6.0+14 447 kB/s | 56 kB 00:00 2023-09-23T14:41:47.529 INFO:teuthology.orchestra.run.smithi017.stdout:(14/19): containernetworking-plugins-0.9.1-1.mo 22 MB/s | 19 MB 00:00 2023-09-23T14:41:47.609 INFO:teuthology.orchestra.run.smithi027.stdout:(9/19): podman-3.0.1-9.module+el8.6.0+14874+644 28 MB/s | 11 MB 00:00 2023-09-23T14:41:47.634 INFO:teuthology.orchestra.run.smithi027.stdout:(10/19): conmon-2.0.26-1.module+el8.6.0+14874+6 191 kB/s | 51 kB 00:00 2023-09-23T14:41:47.677 INFO:teuthology.orchestra.run.smithi027.stdout:(11/19): podman-catatonit-3.0.1-9.module+el8.6. 4.7 MB/s | 322 kB 00:00 2023-09-23T14:41:47.687 INFO:teuthology.orchestra.run.smithi017.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 11 MB/s | 6.4 MB 00:00 2023-09-23T14:41:47.760 INFO:teuthology.orchestra.run.smithi027.stdout:(12/19): yajl-2.1.0-10.el8.x86_64.rpm 325 kB/s | 41 kB 00:00 2023-09-23T14:41:47.788 INFO:teuthology.orchestra.run.smithi017.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 15 MB/s | 6.8 MB 00:00 2023-09-23T14:41:47.821 INFO:teuthology.orchestra.run.smithi017.stdout:(17/19): cockpit-podman-29-2.module+el8.6.0+148 3.7 MB/s | 1.1 MB 00:00 2023-09-23T14:41:47.902 INFO:teuthology.orchestra.run.smithi027.stdout:(13/19): toolbox-0.0.99.3-1.module+el8.6.0+1487 9.7 MB/s | 2.2 MB 00:00 2023-09-23T14:41:47.988 INFO:teuthology.orchestra.run.smithi017.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 616 kB/s | 185 kB 00:00 2023-09-23T14:41:48.038 INFO:teuthology.orchestra.run.smithi017.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 202 kB/s | 51 kB 00:00 2023-09-23T14:41:48.039 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:41:48.039 INFO:teuthology.orchestra.run.smithi017.stdout:Total 21 MB/s | 51 MB 00:02 2023-09-23T14:41:48.111 INFO:teuthology.orchestra.run.smithi027.stdout:(14/19): containernetworking-plugins-0.9.1-1.mo 29 MB/s | 19 MB 00:00 2023-09-23T14:41:48.261 INFO:teuthology.orchestra.run.smithi027.stdout:(15/19): skopeo-1.2.4-1.module+el8.6.0+14874+64 13 MB/s | 6.4 MB 00:00 2023-09-23T14:41:48.353 INFO:teuthology.orchestra.run.smithi027.stdout:(16/19): buildah-1.19.9-3.module+el8.6.0+14874+ 15 MB/s | 6.8 MB 00:00 2023-09-23T14:41:48.387 INFO:teuthology.orchestra.run.smithi027.stdout:(17/19): cockpit-podman-29-2.module+el8.6.0+148 3.9 MB/s | 1.1 MB 00:00 2023-09-23T14:41:48.437 INFO:teuthology.orchestra.run.smithi027.stdout:(18/19): crun-0.18-3.module+el8.6.0+14874+64436 1.0 MB/s | 185 kB 00:00 2023-09-23T14:41:48.462 INFO:teuthology.orchestra.run.smithi027.stdout:(19/19): udica-0.2.4-1.module+el8.6.0+14874+644 466 kB/s | 51 kB 00:00 2023-09-23T14:41:48.463 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:41:48.463 INFO:teuthology.orchestra.run.smithi027.stdout:Total 33 MB/s | 51 MB 00:01 2023-09-23T14:41:48.530 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:41:48.575 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:41:48.576 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:41:48.953 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:41:49.000 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:41:49.000 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:41:49.037 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:41:49.041 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:41:49.457 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:41:49.461 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:41:50.258 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:41:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T14:41:50.496 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T14:41:50.506 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T14:41:50.547 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:41:50.602 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:41:50.639 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299 1/1 2023-09-23T14:41:50.737 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:41:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T14:41:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 1/31 2023-09-23T14:41:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:41:50.990 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:42:08.640 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:42:08.987 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 2/31 2023-09-23T14:42:10.517 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T14:42:10.721 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T14:42:10.840 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : yajl-2.1.0-10.el8.x86_64 3/31 2023-09-23T14:42:10.925 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T14:42:11.022 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : containernetworking-plugins-0.9.1-1.module+el8.6.0 4/31 2023-09-23T14:42:11.116 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T14:42:11.222 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 5/31 2023-09-23T14:42:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T14:42:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T14:42:11.313 INFO:teuthology.orchestra.run.smithi017.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T14:42:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:42:11.378 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 6/31 2023-09-23T14:42:11.573 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 7/31 2023-09-23T14:42:11.573 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 8/31 2023-09-23T14:42:11.574 INFO:teuthology.orchestra.run.smithi027.stdout:warning: /etc/containers/registries.conf created as /etc/containers/registries.conf.rpmnew 2023-09-23T14:42:11.574 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:42:12.137 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T14:42:12.344 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T14:42:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 9/31 2023-09-23T14:42:12.564 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 10/31 2023-09-23T14:42:13.896 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T14:42:14.068 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 11/31 2023-09-23T14:42:14.201 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T14:42:14.363 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 12/31 2023-09-23T14:42:14.572 INFO:teuthology.orchestra.run.smithi017.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T14:42:14.759 INFO:teuthology.orchestra.run.smithi027.stdout: Downgrading : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 13/31 2023-09-23T14:42:14.847 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T14:42:15.053 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 14/31 2023-09-23T14:42:15.599 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T14:42:15.820 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 15/31 2023-09-23T14:42:16.494 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T14:42:16.554 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 16/31 2023-09-23T14:42:16.742 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T14:42:16.754 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 17/31 2023-09-23T14:42:16.930 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T14:42:16.946 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 18/31 2023-09-23T14:42:17.104 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T14:42:17.134 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 19/31 2023-09-23T14:42:17.228 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T14:42:17.229 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T14:42:17.267 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 20/31 2023-09-23T14:42:17.268 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T14:42:17.513 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T14:42:17.601 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 21/31 2023-09-23T14:42:17.704 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T14:42:17.704 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T14:42:17.758 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : containers-common-2:1-27.module+el8.6.0+14877+f643 22/31 2023-09-23T14:42:17.759 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T14:42:17.888 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T14:42:17.927 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: container-selinux-2:2.179.1-1.module+el8.6.0+14877 23/31 2023-09-23T14:42:18.046 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T14:42:18.076 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T14:42:18.172 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T14:42:18.227 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 25/31 2023-09-23T14:42:18.305 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T14:42:18.377 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 26/31 2023-09-23T14:42:18.445 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T14:42:18.546 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T14:42:18.558 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 27/31 2023-09-23T14:42:18.646 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T14:42:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 28/31 2023-09-23T14:42:18.801 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T14:42:18.801 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:18.851 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 29/31 2023-09-23T14:42:18.937 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:19.071 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 30/31 2023-09-23T14:42:19.071 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:19.487 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T14:42:19.773 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: container-selinux-2:2.178.0-2.module+el8.6.0+14874 31/31 2023-09-23T14:42:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T14:42:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T14:42:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T14:42:19.920 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T14:42:19.920 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T14:42:19.920 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T14:42:19.920 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T14:42:19.920 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T14:42:20.170 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: containernetworking-plugins-1:1.0.1-2.module+el8.6 31/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : containers-common-1:1.2.4-1.module+el8.6.0+14874+6 1/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : containers-common-2:1-27.module+el8.6.0+14877+f643 2/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : container-selinux-2:2.178.0-2.module+el8.6.0+14874 3/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : container-selinux-2:2.179.1-1.module+el8.6.0+14877 4/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299. 5/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : slirp4netns-1.1.8-2.module+el8.6.0+14877+f643d2d6. 6/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 7/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : criu-3.15-3.module+el8.6.0+14877+f643d2d6.x86_64 8/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86 9/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libslirp-4.4.0-1.module+el8.6.0+14877+f643d2d6.x86 10/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x 11/31 2023-09-23T14:42:20.171 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : runc-1:1.0.3-2.module+el8.6.0+14877+f643d2d6.x86_6 12/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+644362 13/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : fuse-overlayfs-1.8.2-1.module+el8.6.0+14877+f643d2 14/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_6 15/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-2:4.1.1-7.module+el8.6.0+16889+9f154fa9.x86 16/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-docker-3.0.1-9.module+el8.6.0+14874+6443629 17/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-docker-2:4.1.1-7.module+el8.6.0+16889+9f154 18/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x8 19/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : conmon-2:2.1.0-1.module+el8.6.0+14877+f643d2d6.x86 20/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : containernetworking-plugins-0.9.1-1.module+el8.6.0 21/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : containernetworking-plugins-1:1.0.1-2.module+el8.6 22/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-catatonit-3.0.1-9.module+el8.6.0+14874+6443 23/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : podman-catatonit-2:4.1.1-7.module+el8.6.0+16889+9f 24/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 25/31 2023-09-23T14:42:20.172 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x 26/31 2023-09-23T14:42:20.173 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86 27/31 2023-09-23T14:42:20.173 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86 28/31 2023-09-23T14:42:20.173 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : cockpit-podman-29-2.module+el8.6.0+14874+64436299. 29/31 2023-09-23T14:42:20.173 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 30/31 2023-09-23T14:42:21.355 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T14:42:21.355 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:43 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:42:21.652 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 31/31 2023-09-23T14:42:21.652 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:44 ago on Sat 23 Sep 2023 02:39:37 PM UTC. 2023-09-23T14:42:27.765 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:42:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:42:27.971 INFO:teuthology.orchestra.run.smithi027.stdout:Downgraded: 2023-09-23T14:42:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.972 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:42:27.973 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:42:28.152 DEBUG:teuthology.parallel:result is None 2023-09-23T14:42:31.743 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout:Downgraded: 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: conmon-2:2.0.26-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: container-selinux-2:2.178.0-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: containernetworking-plugins-0.9.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: containers-common-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: criu-3.15-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.957 INFO:teuthology.orchestra.run.smithi017.stdout: fuse-overlayfs-1.4.0-2.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: libslirp-4.3.1-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: podman-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: podman-catatonit-3.0.1-9.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: podman-docker-3.0.1-9.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: runc-1.0.0-73.rc95.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: slirp4netns-1.1.8-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: buildah-1.19.9-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: cockpit-podman-29-2.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: crun-0.18-3.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: skopeo-1:1.2.4-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: toolbox-0.0.99.3-1.module+el8.6.0+14874+64436299.x86_64 2023-09-23T14:42:31.958 INFO:teuthology.orchestra.run.smithi017.stdout: udica-0.2.4-1.module+el8.6.0+14874+64436299.noarch 2023-09-23T14:42:31.959 INFO:teuthology.orchestra.run.smithi017.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T14:42:31.959 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:42:31.959 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:42:32.139 DEBUG:teuthology.parallel:result is None 2023-09-23T14:42:32.139 INFO:teuthology.run_tasks:Running task install... 2023-09-23T14:42:32.147 DEBUG:teuthology.task.install:project ceph 2023-09-23T14:42:32.147 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T14:42:32.147 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T14:42:32.147 INFO:teuthology.task.install:Using flavor: default 2023-09-23T14:42:32.153 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T14:42:32.153 INFO:teuthology.task.install:extra packages: [] 2023-09-23T14:42:32.154 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T14:42:32.154 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:42:32.154 INFO:teuthology.packaging:ref: None 2023-09-23T14:42:32.154 INFO:teuthology.packaging:tag: None 2023-09-23T14:42:32.154 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:42:32.154 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:42:32.154 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T14:42:32.156 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T14:42:32.156 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:42:32.156 INFO:teuthology.packaging:ref: None 2023-09-23T14:42:32.156 INFO:teuthology.packaging:tag: None 2023-09-23T14:42:32.156 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:42:32.156 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:42:32.157 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T14:42:32.412 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T14:42:32.412 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T14:42:32.520 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T14:42:32.520 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T14:42:32.532 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T14:42:32.532 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:42:32.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T14:42:32.566 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T14:42:32.566 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:42:32.567 INFO:teuthology.packaging:ref: None 2023-09-23T14:42:32.567 INFO:teuthology.packaging:tag: None 2023-09-23T14:42:32.567 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:42:32.567 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:42:32.567 DEBUG:teuthology.orchestra.run.smithi027:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T14:42:32.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T14:42:32.722 DEBUG:teuthology.orchestra.run.smithi027:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T14:42:32.729 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T14:42:32.730 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:42:32.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T14:42:32.761 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T14:42:32.761 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:42:32.761 INFO:teuthology.packaging:ref: None 2023-09-23T14:42:32.761 INFO:teuthology.packaging:tag: None 2023-09-23T14:42:32.761 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:42:32.762 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:42:32.762 DEBUG:teuthology.orchestra.run.smithi017:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T14:42:32.791 INFO:teuthology.orchestra.run.smithi027.stdout:check_obsoletes = 1 2023-09-23T14:42:32.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum clean all 2023-09-23T14:42:32.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T14:42:32.913 DEBUG:teuthology.orchestra.run.smithi017:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T14:42:32.982 INFO:teuthology.orchestra.run.smithi017.stdout:check_obsoletes = 1 2023-09-23T14:42:32.983 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum clean all 2023-09-23T14:42:33.185 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:42:33.377 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:42:34.291 INFO:teuthology.orchestra.run.smithi017.stdout:44 files removed 2023-09-23T14:42:34.338 INFO:teuthology.orchestra.run.smithi027.stdout:44 files removed 2023-09-23T14:42:34.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-radosgw 2023-09-23T14:42:34.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-radosgw 2023-09-23T14:42:34.698 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:42:34.744 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:42:36.228 INFO:teuthology.orchestra.run.smithi017.stdout:ceph packages for x86_64 125 kB/s | 78 kB 00:00 2023-09-23T14:42:36.355 INFO:teuthology.orchestra.run.smithi027.stdout:ceph packages for x86_64 150 kB/s | 78 kB 00:00 2023-09-23T14:42:37.004 INFO:teuthology.orchestra.run.smithi017.stdout:ceph noarch packages 12 kB/s | 9.1 kB 00:00 2023-09-23T14:42:37.089 INFO:teuthology.orchestra.run.smithi027.stdout:ceph noarch packages 13 kB/s | 9.1 kB 00:00 2023-09-23T14:42:37.371 INFO:teuthology.orchestra.run.smithi017.stdout:ceph source packages 4.2 kB/s | 1.5 kB 00:00 2023-09-23T14:42:37.523 INFO:teuthology.orchestra.run.smithi027.stdout:ceph source packages 3.5 kB/s | 1.5 kB 00:00 2023-09-23T14:42:37.571 INFO:teuthology.orchestra.run.smithi017.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-09-23T14:42:37.691 INFO:teuthology.orchestra.run.smithi027.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-09-23T14:42:38.247 INFO:teuthology.orchestra.run.smithi017.stdout:Extra Packages for Enterprise Linux 24 MB/s | 16 MB 00:00 2023-09-23T14:42:41.477 INFO:teuthology.orchestra.run.smithi027.stdout:Extra Packages for Enterprise Linux 4.2 MB/s | 16 MB 00:03 2023-09-23T14:42:41.931 INFO:teuthology.orchestra.run.smithi017.stdout:lab-extras 416 kB/s | 24 kB 00:00 2023-09-23T14:42:43.249 INFO:teuthology.orchestra.run.smithi017.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 43 MB/s | 53 MB 00:01 2023-09-23T14:42:45.259 INFO:teuthology.orchestra.run.smithi027.stdout:lab-extras 398 kB/s | 24 kB 00:00 2023-09-23T14:42:46.435 INFO:teuthology.orchestra.run.smithi027.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 48 MB/s | 53 MB 00:01 2023-09-23T14:42:52.067 INFO:teuthology.orchestra.run.smithi017.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 45 MB/s | 47 MB 00:01 2023-09-23T14:42:55.270 INFO:teuthology.orchestra.run.smithi027.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 45 MB/s | 47 MB 00:01 2023-09-23T14:43:03.819 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:43:03.821 INFO:teuthology.orchestra.run.smithi017.stdout:==================================================================================================== 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout:==================================================================================================== 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrading: 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T14:43:03.822 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T14:43:03.823 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout:==================================================================================================== 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout:Install 21 Packages 2023-09-23T14:43:03.824 INFO:teuthology.orchestra.run.smithi017.stdout:Upgrade 2 Packages 2023-09-23T14:43:03.825 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:03.825 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 63 M 2023-09-23T14:43:03.825 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:43:04.709 INFO:teuthology.orchestra.run.smithi017.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 7.9 MB/s | 5.2 MB 00:00 2023-09-23T14:43:04.793 INFO:teuthology.orchestra.run.smithi017.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 292 kB/s | 24 kB 00:00 2023-09-23T14:43:04.910 INFO:teuthology.orchestra.run.smithi017.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 6.7 MB/s | 796 kB 00:00 2023-09-23T14:43:05.001 INFO:teuthology.orchestra.run.smithi017.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 5.4 MB/s | 506 kB 00:00 2023-09-23T14:43:05.194 INFO:teuthology.orchestra.run.smithi017.stdout:(5/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 15 MB/s | 17 MB 00:01 2023-09-23T14:43:05.235 INFO:teuthology.orchestra.run.smithi017.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 1.1 MB/s | 45 kB 00:00 2023-09-23T14:43:05.352 INFO:teuthology.orchestra.run.smithi017.stdout:(7/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 16 MB/s | 5.5 MB 00:00 2023-09-23T14:43:05.378 INFO:teuthology.orchestra.run.smithi017.stdout:(8/23): python3-ceph-common-17.2.6-1123.g61431f 810 kB/s | 115 kB 00:00 2023-09-23T14:43:05.403 INFO:teuthology.orchestra.run.smithi017.stdout:(9/23): python3-cephfs-17.2.6-1123.g61431f6b.el 3.7 MB/s | 190 kB 00:00 2023-09-23T14:43:05.428 INFO:teuthology.orchestra.run.smithi017.stdout:(10/23): python3-rados-17.2.6-1123.g61431f6b.el 7.2 MB/s | 371 kB 00:00 2023-09-23T14:43:05.487 INFO:teuthology.orchestra.run.smithi017.stdout:(11/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.2 MB/s | 358 kB 00:00 2023-09-23T14:43:05.504 INFO:teuthology.orchestra.run.smithi017.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.4 MB/s | 106 kB 00:00 2023-09-23T14:43:05.671 INFO:teuthology.orchestra.run.smithi017.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 353 kB/s | 59 kB 00:00 2023-09-23T14:43:05.771 INFO:teuthology.orchestra.run.smithi017.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 751 kB/s | 75 kB 00:00 2023-09-23T14:43:06.097 INFO:teuthology.orchestra.run.smithi017.stdout:(15/23): thrift-0.13.0-2.el8.x86_64.rpm 5.3 MB/s | 1.7 MB 00:00 2023-09-23T14:43:06.122 INFO:teuthology.orchestra.run.smithi017.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 483 kB/s | 306 kB 00:00 2023-09-23T14:43:06.289 INFO:teuthology.orchestra.run.smithi017.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 244 kB/s | 47 kB 00:00 2023-09-23T14:43:06.314 INFO:teuthology.orchestra.run.smithi017.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 1.3 MB/s | 259 kB 00:00 2023-09-23T14:43:06.398 INFO:teuthology.orchestra.run.smithi017.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 405 kB/s | 44 kB 00:00 2023-09-23T14:43:06.448 INFO:teuthology.orchestra.run.smithi017.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 2.6 MB/s | 354 kB 00:00 2023-09-23T14:43:06.748 INFO:teuthology.orchestra.run.smithi017.stdout:(21/23): librados2-17.2.6-1123.g61431f6b.el8.x8 9.9 MB/s | 3.5 MB 00:00 2023-09-23T14:43:06.915 INFO:teuthology.orchestra.run.smithi017.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 7.7 MB/s | 3.6 MB 00:00 2023-09-23T14:43:07.071 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout:==================================================================================================== 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout:==================================================================================================== 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout:Upgrading: 2023-09-23T14:43:07.074 INFO:teuthology.orchestra.run.smithi027.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T14:43:07.075 INFO:teuthology.orchestra.run.smithi027.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:43:07.076 INFO:teuthology.orchestra.run.smithi027.stdout:==================================================================================================== 2023-09-23T14:43:07.077 INFO:teuthology.orchestra.run.smithi027.stdout:Install 21 Packages 2023-09-23T14:43:07.077 INFO:teuthology.orchestra.run.smithi027.stdout:Upgrade 2 Packages 2023-09-23T14:43:07.077 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:43:07.077 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 63 M 2023-09-23T14:43:07.077 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:43:08.495 INFO:teuthology.orchestra.run.smithi027.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 3.7 MB/s | 5.2 MB 00:01 2023-09-23T14:43:08.581 INFO:teuthology.orchestra.run.smithi027.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 292 kB/s | 24 kB 00:00 2023-09-23T14:43:08.796 INFO:teuthology.orchestra.run.smithi027.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 3.6 MB/s | 796 kB 00:00 2023-09-23T14:43:08.989 INFO:teuthology.orchestra.run.smithi027.stdout:(4/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8. 8.9 MB/s | 17 MB 00:01 2023-09-23T14:43:09.015 INFO:teuthology.orchestra.run.smithi017.stdout:(23/23): ceph-common-17.2.6-1123.g61431f6b.el8. 4.7 MB/s | 23 MB 00:04 2023-09-23T14:43:09.017 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:43:09.018 INFO:teuthology.orchestra.run.smithi017.stdout:Total 12 MB/s | 63 MB 00:05 2023-09-23T14:43:09.029 INFO:teuthology.orchestra.run.smithi027.stdout:(5/23): libradosstriper1-17.2.6-1123.g61431f6b. 2.1 MB/s | 506 kB 00:00 2023-09-23T14:43:09.064 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:43:09.088 INFO:teuthology.orchestra.run.smithi027.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 764 kB/s | 45 kB 00:00 2023-09-23T14:43:09.114 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:43:09.114 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:43:09.155 INFO:teuthology.orchestra.run.smithi027.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 1.7 MB/s | 115 kB 00:00 2023-09-23T14:43:09.230 INFO:teuthology.orchestra.run.smithi027.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.5 MB/s | 190 kB 00:00 2023-09-23T14:43:09.314 INFO:teuthology.orchestra.run.smithi027.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 4.3 MB/s | 371 kB 00:00 2023-09-23T14:43:09.406 INFO:teuthology.orchestra.run.smithi027.stdout:(10/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_ 13 MB/s | 5.5 MB 00:00 2023-09-23T14:43:09.431 INFO:teuthology.orchestra.run.smithi027.stdout:(11/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 3.0 MB/s | 358 kB 00:00 2023-09-23T14:43:09.457 INFO:teuthology.orchestra.run.smithi027.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 2.1 MB/s | 106 kB 00:00 2023-09-23T14:43:09.472 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:43:09.476 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:43:09.907 INFO:teuthology.orchestra.run.smithi027.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 131 kB/s | 59 kB 00:00 2023-09-23T14:43:10.066 INFO:teuthology.orchestra.run.smithi027.stdout:(14/23): gperftools-libs-2.7-9.el8.x86_64.rpm 483 kB/s | 306 kB 00:00 2023-09-23T14:43:10.116 INFO:teuthology.orchestra.run.smithi027.stdout:(15/23): libunwind-1.3.1-3.el8.x86_64.rpm 360 kB/s | 75 kB 00:00 2023-09-23T14:43:10.300 INFO:teuthology.orchestra.run.smithi027.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 256 kB/s | 47 kB 00:00 2023-09-23T14:43:10.425 INFO:teuthology.orchestra.run.smithi027.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.0 MB/s | 259 kB 00:00 2023-09-23T14:43:10.467 INFO:teuthology.orchestra.run.smithi027.stdout:(18/23): thrift-0.13.0-2.el8.x86_64.rpm 4.3 MB/s | 1.7 MB 00:00 2023-09-23T14:43:10.559 INFO:teuthology.orchestra.run.smithi027.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 329 kB/s | 44 kB 00:00 2023-09-23T14:43:10.609 INFO:teuthology.orchestra.run.smithi027.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 2.4 MB/s | 354 kB 00:00 2023-09-23T14:43:10.633 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:43:10.969 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T14:43:11.218 INFO:teuthology.orchestra.run.smithi027.stdout:(21/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 5.9 MB/s | 3.6 MB 00:00 2023-09-23T14:43:11.305 INFO:teuthology.orchestra.run.smithi027.stdout:(22/23): librados2-17.2.6-1123.g61431f6b.el8.x8 4.6 MB/s | 3.5 MB 00:00 2023-09-23T14:43:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T14:43:11.527 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:11.625 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:11.799 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T14:43:11.852 INFO:teuthology.orchestra.run.smithi027.stdout:(23/23): ceph-common-17.2.6-1123.g61431f6b.el8. 4.9 MB/s | 23 MB 00:04 2023-09-23T14:43:11.854 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:43:11.855 INFO:teuthology.orchestra.run.smithi027.stdout:Total 13 MB/s | 63 MB 00:04 2023-09-23T14:43:11.902 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:43:11.951 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:43:11.951 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:43:12.072 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T14:43:12.235 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:12.319 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:43:12.323 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:43:12.653 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:12.902 INFO:teuthology.orchestra.run.smithi017.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T14:43:12.992 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T14:43:13.152 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T14:43:13.222 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T14:43:13.378 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:43:13.563 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T14:43:13.565 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T14:43:13.961 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T14:43:14.122 INFO:teuthology.orchestra.run.smithi027.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:14.290 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T14:43:14.403 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T14:43:14.476 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:14.542 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:14.625 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T14:43:14.802 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T14:43:14.930 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:14.997 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T14:43:15.214 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T14:43:15.349 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:15.401 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T14:43:15.538 INFO:teuthology.orchestra.run.smithi027.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T14:43:15.560 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T14:43:15.615 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T14:43:15.642 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T14:43:15.763 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T14:43:15.780 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T14:43:15.842 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T14:43:15.938 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T14:43:16.153 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T14:43:16.201 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T14:43:16.469 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T14:43:16.861 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T14:43:17.054 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:17.122 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:17.241 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T14:43:17.391 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T14:43:17.600 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T14:43:17.786 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T14:43:17.981 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T14:43:18.051 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T14:43:18.164 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T14:43:18.343 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T14:43:18.565 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T14:43:18.730 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T14:43:20.497 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:20.718 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:21.313 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:22.083 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T14:43:22.157 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T14:43:22.317 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:22.912 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:23.506 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T14:43:23.762 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T14:43:23.833 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T14:43:23.962 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:41.723 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /sys 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /proc 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /dev 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /run 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /mnt 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /var/tmp 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /home 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /tmp 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout:skipping the directory /dev 2023-09-23T14:43:41.724 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:43.355 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /sys 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /proc 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /dev 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /run 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /mnt 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /var/tmp 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /home 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /tmp 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout:skipping the directory /dev 2023-09-23T14:43:43.356 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:43:43.614 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T14:43:43.720 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T14:43:43.720 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T14:43:43.925 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T14:43:43.925 INFO:teuthology.orchestra.run.smithi017.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:44.530 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:44.531 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T14:43:44.532 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T14:43:45.142 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T14:43:45.218 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T14:43:45.218 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T14:43:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T14:43:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:45.937 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:45.937 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:00:54 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T14:43:46.031 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T14:43:46.032 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T14:43:46.033 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T14:43:46.033 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T14:43:46.033 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T14:43:47.393 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T14:43:47.393 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:00:53 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:43:53.727 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout:Upgraded: 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.929 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T14:43:53.930 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:53.931 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:43:54.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-test 2023-09-23T14:43:54.468 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:43:55.983 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:04 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:43:57.521 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:43:57.522 INFO:teuthology.orchestra.run.smithi017.stdout:============================================================================================ 2023-09-23T14:43:57.522 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:43:57.522 INFO:teuthology.orchestra.run.smithi017.stdout:============================================================================================ 2023-09-23T14:43:57.522 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T14:43:57.523 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:57.524 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:43:57.524 INFO:teuthology.orchestra.run.smithi017.stdout:============================================================================================ 2023-09-23T14:43:57.524 INFO:teuthology.orchestra.run.smithi017.stdout:Install 6 Packages 2023-09-23T14:43:57.524 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:43:57.525 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 53 M 2023-09-23T14:43:57.525 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 233 M 2023-09-23T14:43:57.525 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:43:57.555 INFO:teuthology.orchestra.run.smithi017.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.2 MB/s | 70 kB 00:00 2023-09-23T14:43:57.747 INFO:teuthology.orchestra.run.smithi017.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.6 MB/s | 323 kB 00:00 2023-09-23T14:43:57.922 INFO:teuthology.orchestra.run.smithi017.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.0 MB/s | 187 kB 00:00 2023-09-23T14:43:58.048 INFO:teuthology.orchestra.run.smithi017.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 1.6 MB/s | 202 kB 00:00 2023-09-23T14:43:58.098 INFO:teuthology.orchestra.run.smithi017.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 304 kB/s | 174 kB 00:00 2023-09-23T14:43:58.609 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout:Upgraded: 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.814 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T14:43:58.815 INFO:teuthology.orchestra.run.smithi027.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:43:58.816 INFO:teuthology.orchestra.run.smithi027.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T14:43:58.817 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:43:58.817 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:43:58.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-test 2023-09-23T14:43:59.346 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:44:01.069 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:07 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:44:02.581 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:44:02.582 INFO:teuthology.orchestra.run.smithi027.stdout:============================================================================================ 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout:============================================================================================ 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T14:44:02.583 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout:============================================================================================ 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout:Install 6 Packages 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 53 M 2023-09-23T14:44:02.584 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 233 M 2023-09-23T14:44:02.585 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:44:02.616 INFO:teuthology.orchestra.run.smithi027.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.2 MB/s | 70 kB 00:00 2023-09-23T14:44:02.800 INFO:teuthology.orchestra.run.smithi027.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.7 MB/s | 323 kB 00:00 2023-09-23T14:44:02.933 INFO:teuthology.orchestra.run.smithi027.stdout:(3/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 500 kB/s | 174 kB 00:00 2023-09-23T14:44:02.959 INFO:teuthology.orchestra.run.smithi027.stdout:(4/6): oniguruma-6.8.2-2.el8.x86_64.rpm 1.2 MB/s | 187 kB 00:00 2023-09-23T14:44:03.101 INFO:teuthology.orchestra.run.smithi027.stdout:(5/6): jq-1.6-3.el8.x86_64.rpm 1.2 MB/s | 202 kB 00:00 2023-09-23T14:44:10.792 INFO:teuthology.orchestra.run.smithi027.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 6.3 MB/s | 52 MB 00:08 2023-09-23T14:44:10.794 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:44:10.794 INFO:teuthology.orchestra.run.smithi027.stdout:Total 6.4 MB/s | 53 MB 00:08 2023-09-23T14:44:10.832 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:44:10.855 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:44:10.855 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:44:11.146 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:44:11.149 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:44:11.622 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:44:11.789 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T14:44:11.854 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T14:44:12.045 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T14:44:12.190 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T14:44:12.314 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T14:44:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T14:44:17.646 INFO:teuthology.orchestra.run.smithi017.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 2.6 MB/s | 52 MB 00:20 2023-09-23T14:44:17.649 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:44:17.649 INFO:teuthology.orchestra.run.smithi017.stdout:Total 2.6 MB/s | 53 MB 00:20 2023-09-23T14:44:17.686 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:44:17.688 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T14:44:17.708 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:44:17.708 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:44:17.937 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T14:44:17.996 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:44:18.000 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:44:18.167 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T14:44:18.167 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T14:44:18.168 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T14:44:18.168 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T14:44:18.168 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T14:44:18.168 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T14:44:18.471 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:44:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T14:44:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T14:44:18.949 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T14:44:19.095 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T14:44:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T14:44:19.452 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T14:44:19.635 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T14:44:19.635 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:25 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:44:24.710 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T14:44:24.974 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T14:44:25.212 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T14:44:26.317 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T14:44:26.317 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:35 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:44:26.724 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: jq-1.6-3.el8.x86_64 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T14:44:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T14:44:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T14:44:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:26.924 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:44:27.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph 2023-09-23T14:44:27.417 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:44:29.156 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:35 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:44:30.628 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:44:30.631 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================= 2023-09-23T14:44:30.631 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:44:30.631 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================= 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T14:44:30.632 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T14:44:30.633 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T14:44:30.634 INFO:teuthology.orchestra.run.smithi027.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T14:44:30.635 INFO:teuthology.orchestra.run.smithi027.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:30.636 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:44:30.637 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================= 2023-09-23T14:44:30.637 INFO:teuthology.orchestra.run.smithi027.stdout:Install 32 Packages 2023-09-23T14:44:30.637 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:30.638 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 28 M 2023-09-23T14:44:30.638 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 110 M 2023-09-23T14:44:30.638 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:44:30.920 INFO:teuthology.orchestra.run.smithi027.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 50 kB/s | 6.4 kB 00:00 2023-09-23T14:44:31.320 INFO:teuthology.orchestra.run.smithi027.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.4 MB/s | 2.3 MB 00:00 2023-09-23T14:44:31.604 INFO:teuthology.orchestra.run.smithi027.stdout:(3/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 5.8 MB/s | 3.9 MB 00:00 2023-09-23T14:44:31.666 INFO:teuthology.orchestra.run.smithi027.stdout:(4/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 1.8 MB/s | 1.6 MB 00:00 2023-09-23T14:44:31.696 INFO:teuthology.orchestra.run.smithi027.stdout:(5/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.5 MB/s | 238 kB 00:00 2023-09-23T14:44:32.030 INFO:teuthology.orchestra.run.smithi027.stdout:(6/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 120 kB/s | 44 kB 00:00 2023-09-23T14:44:32.172 INFO:teuthology.orchestra.run.smithi027.stdout:(7/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 389 kB/s | 185 kB 00:00 2023-09-23T14:44:32.331 INFO:teuthology.orchestra.run.smithi027.stdout:(8/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 17 MB/s | 17 MB 00:01 2023-09-23T14:44:32.356 INFO:teuthology.orchestra.run.smithi027.stdout:(9/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 530 kB/s | 173 kB 00:00 2023-09-23T14:44:32.415 INFO:teuthology.orchestra.run.smithi027.stdout:(10/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.6 MB/s | 384 kB 00:00 2023-09-23T14:44:32.457 INFO:teuthology.orchestra.run.smithi027.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 175 kB/s | 18 kB 00:00 2023-09-23T14:44:32.515 INFO:teuthology.orchestra.run.smithi027.stdout:(12/32): python3-jaraco-6.2-6.el8.noarch.rpm 60 kB/s | 11 kB 00:00 2023-09-23T14:44:32.532 INFO:teuthology.orchestra.run.smithi027.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 415 kB/s | 49 kB 00:00 2023-09-23T14:44:32.574 INFO:teuthology.orchestra.run.smithi027.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 500 kB/s | 59 kB 00:00 2023-09-23T14:44:32.625 INFO:teuthology.orchestra.run.smithi027.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 170 kB/s | 16 kB 00:00 2023-09-23T14:44:32.675 INFO:teuthology.orchestra.run.smithi027.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 186 kB/s | 19 kB 00:00 2023-09-23T14:44:32.717 INFO:teuthology.orchestra.run.smithi027.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 260 kB/s | 24 kB 00:00 2023-09-23T14:44:32.775 INFO:teuthology.orchestra.run.smithi027.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 292 kB/s | 29 kB 00:00 2023-09-23T14:44:32.809 INFO:teuthology.orchestra.run.smithi027.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 300 kB/s | 27 kB 00:00 2023-09-23T14:44:32.901 INFO:teuthology.orchestra.run.smithi027.stdout:(20/32): python3-pecan-1.3.2-9.el8.noarch.rpm 734 kB/s | 283 kB 00:00 2023-09-23T14:44:32.934 INFO:teuthology.orchestra.run.smithi027.stdout:(21/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-09-23T14:44:32.968 INFO:teuthology.orchestra.run.smithi027.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.3 MB/s | 255 kB 00:00 2023-09-23T14:44:33.018 INFO:teuthology.orchestra.run.smithi027.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 728 kB/s | 85 kB 00:00 2023-09-23T14:44:33.035 INFO:teuthology.orchestra.run.smithi027.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 228 kB/s | 23 kB 00:00 2023-09-23T14:44:33.127 INFO:teuthology.orchestra.run.smithi027.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 686 kB/s | 109 kB 00:00 2023-09-23T14:44:33.161 INFO:teuthology.orchestra.run.smithi027.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 3.8 MB/s | 559 kB 00:00 2023-09-23T14:44:33.228 INFO:teuthology.orchestra.run.smithi027.stdout:(27/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.2 MB/s | 238 kB 00:00 2023-09-23T14:44:33.287 INFO:teuthology.orchestra.run.smithi027.stdout:(28/32): python3-pytz-2017.2-9.el8.noarch.rpm 341 kB/s | 54 kB 00:00 2023-09-23T14:44:33.362 INFO:teuthology.orchestra.run.smithi027.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 194 kB/s | 39 kB 00:00 2023-09-23T14:44:33.396 INFO:teuthology.orchestra.run.smithi027.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 939 kB/s | 157 kB 00:00 2023-09-23T14:44:33.429 INFO:teuthology.orchestra.run.smithi027.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.1 MB/s | 457 kB 00:00 2023-09-23T14:44:33.487 INFO:teuthology.orchestra.run.smithi027.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 827 kB/s | 103 kB 00:00 2023-09-23T14:44:33.489 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:44:33.489 INFO:teuthology.orchestra.run.smithi027.stdout:Total 9.9 MB/s | 28 MB 00:02 2023-09-23T14:44:33.585 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:44:33.625 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:44:33.625 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:44:33.843 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:44:33.847 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:44:34.919 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:44:35.113 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T14:44:35.384 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T14:44:35.574 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T14:44:35.784 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T14:44:36.000 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T14:44:36.158 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T14:44:36.494 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:44:36.508 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout: jq-1.6-3.el8.x86_64 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:36.689 INFO:teuthology.orchestra.run.smithi017.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T14:44:36.690 INFO:teuthology.orchestra.run.smithi017.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T14:44:36.690 INFO:teuthology.orchestra.run.smithi017.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T14:44:36.690 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:44:36.690 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:44:36.856 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T14:44:36.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph 2023-09-23T14:44:37.099 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T14:44:37.244 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T14:44:37.284 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:44:37.402 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T14:44:37.578 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T14:44:37.793 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T14:44:37.968 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T14:44:38.140 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T14:44:38.289 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T14:44:38.439 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T14:44:38.558 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T14:44:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T14:44:38.912 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:01:47 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:44:38.990 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T14:44:39.175 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T14:44:39.446 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T14:44:39.660 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T14:44:39.872 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T14:44:40.192 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T14:44:40.443 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================= 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================= 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T14:44:40.447 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T14:44:40.448 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T14:44:40.449 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T14:44:40.450 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:44:40.451 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:44:40.451 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================= 2023-09-23T14:44:40.451 INFO:teuthology.orchestra.run.smithi017.stdout:Install 32 Packages 2023-09-23T14:44:40.451 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:44:40.451 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 28 M 2023-09-23T14:44:40.452 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 110 M 2023-09-23T14:44:40.452 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:44:40.499 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T14:44:40.756 INFO:teuthology.orchestra.run.smithi017.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 39 kB/s | 6.4 kB 00:00 2023-09-23T14:44:40.841 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T14:44:41.010 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T14:44:41.748 INFO:teuthology.orchestra.run.smithi017.stdout:(2/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 2.0 MB/s | 2.3 MB 00:01 2023-09-23T14:44:41.856 INFO:teuthology.orchestra.run.smithi017.stdout:(3/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 1.2 MB/s | 1.6 MB 00:01 2023-09-23T14:44:42.006 INFO:teuthology.orchestra.run.smithi017.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 1.6 MB/s | 238 kB 00:00 2023-09-23T14:44:42.173 INFO:teuthology.orchestra.run.smithi017.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 262 kB/s | 44 kB 00:00 2023-09-23T14:44:42.307 INFO:teuthology.orchestra.run.smithi017.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.4 MB/s | 185 kB 00:00 2023-09-23T14:44:42.382 INFO:teuthology.orchestra.run.smithi017.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 2.3 MB/s | 173 kB 00:00 2023-09-23T14:44:42.506 INFO:teuthology.orchestra.run.smithi017.stdout:(8/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 3.1 MB/s | 384 kB 00:00 2023-09-23T14:44:42.566 INFO:teuthology.orchestra.run.smithi017.stdout:(9/32): python3-jaraco-6.2-6.el8.noarch.rpm 185 kB/s | 11 kB 00:00 2023-09-23T14:44:42.585 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T14:44:42.625 INFO:teuthology.orchestra.run.smithi017.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 303 kB/s | 18 kB 00:00 2023-09-23T14:44:42.684 INFO:teuthology.orchestra.run.smithi017.stdout:(11/32): python3-logutils-0.3.5-11.el8.noarch.r 834 kB/s | 49 kB 00:00 2023-09-23T14:44:42.742 INFO:teuthology.orchestra.run.smithi017.stdout:(12/32): python3-more-itertools-7.2.0-3.el8.noa 1.0 MB/s | 59 kB 00:00 2023-09-23T14:44:42.760 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T14:44:42.826 INFO:teuthology.orchestra.run.smithi017.stdout:(13/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86 1.9 MB/s | 3.9 MB 00:02 2023-09-23T14:44:42.851 INFO:teuthology.orchestra.run.smithi017.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.5 MB/s | 283 kB 00:00 2023-09-23T14:44:42.910 INFO:teuthology.orchestra.run.smithi017.stdout:(15/32): python3-simplegeneric-0.8.1-17.el8.noa 321 kB/s | 19 kB 00:00 2023-09-23T14:44:42.952 INFO:teuthology.orchestra.run.smithi017.stdout:(16/32): python3-portend-2.6-1.el8.noarch.rpm 124 kB/s | 16 kB 00:00 2023-09-23T14:44:42.977 INFO:teuthology.orchestra.run.smithi017.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 358 kB/s | 24 kB 00:00 2023-09-23T14:44:43.035 INFO:teuthology.orchestra.run.smithi017.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 351 kB/s | 29 kB 00:00 2023-09-23T14:44:43.061 INFO:teuthology.orchestra.run.smithi017.stdout:(19/32): python3-trustme-0.6.0-4.el8.noarch.rpm 329 kB/s | 27 kB 00:00 2023-09-23T14:44:43.128 INFO:teuthology.orchestra.run.smithi017.stdout:(20/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 3.7 MB/s | 251 kB 00:00 2023-09-23T14:44:43.177 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T14:44:43.217 INFO:teuthology.orchestra.run.smithi017.stdout:(21/32): python3-webtest-2.0.33-1.el8.noarch.rp 962 kB/s | 85 kB 00:00 2023-09-23T14:44:43.245 INFO:teuthology.orchestra.run.smithi017.stdout:(22/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.2 MB/s | 255 kB 00:00 2023-09-23T14:44:43.279 INFO:teuthology.orchestra.run.smithi017.stdout:(23/32): python3-zc-lockfile-2.0-2.el8.noarch.r 372 kB/s | 23 kB 00:00 2023-09-23T14:44:43.294 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T14:44:43.437 INFO:teuthology.orchestra.run.smithi017.stdout:(24/32): python3-pycparser-2.14-14.el8.noarch.r 569 kB/s | 109 kB 00:00 2023-09-23T14:44:43.488 INFO:teuthology.orchestra.run.smithi017.stdout:(25/32): python3-cryptography-3.2.1-5.el8.x86_6 2.7 MB/s | 559 kB 00:00 2023-09-23T14:44:43.523 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T14:44:43.571 INFO:teuthology.orchestra.run.smithi017.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.7 MB/s | 238 kB 00:00 2023-09-23T14:44:43.613 INFO:teuthology.orchestra.run.smithi017.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 432 kB/s | 54 kB 00:00 2023-09-23T14:44:43.652 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T14:44:43.671 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T14:44:43.747 INFO:teuthology.orchestra.run.smithi017.stdout:(28/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.2 MB/s | 157 kB 00:00 2023-09-23T14:44:43.767 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T14:44:43.772 INFO:teuthology.orchestra.run.smithi017.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 194 kB/s | 39 kB 00:00 2023-09-23T14:44:43.881 INFO:teuthology.orchestra.run.smithi017.stdout:(30/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.3 MB/s | 457 kB 00:00 2023-09-23T14:44:43.981 INFO:teuthology.orchestra.run.smithi017.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 496 kB/s | 103 kB 00:00 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T14:44:44.117 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T14:44:44.118 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T14:44:44.119 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T14:44:44.731 INFO:teuthology.orchestra.run.smithi017.stdout:(32/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86 5.6 MB/s | 17 MB 00:02 2023-09-23T14:44:44.734 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:44:44.734 INFO:teuthology.orchestra.run.smithi017.stdout:Total 6.6 MB/s | 28 MB 00:04 2023-09-23T14:44:44.828 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:44:44.867 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:44:44.867 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:44:45.085 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:44:45.090 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:44:45.696 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T14:44:45.696 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:44:46.184 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:44:46.399 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T14:44:46.672 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T14:44:46.920 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T14:44:47.148 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T14:44:47.354 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T14:44:47.598 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T14:44:47.831 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T14:44:48.200 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T14:44:48.511 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T14:44:48.807 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T14:44:49.023 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T14:44:49.199 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T14:44:49.418 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T14:44:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T14:44:49.803 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T14:44:49.969 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T14:44:50.118 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T14:44:50.246 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T14:44:50.500 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T14:44:50.729 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T14:44:50.904 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T14:44:51.167 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T14:44:51.558 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T14:44:51.761 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T14:44:52.076 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T14:44:52.394 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T14:44:52.728 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T14:44:52.880 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T14:44:54.276 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:44:54.453 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:44:54.481 INFO:teuthology.orchestra.run.smithi027.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T14:44:54.482 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T14:44:54.483 INFO:teuthology.orchestra.run.smithi027.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T14:44:54.484 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:44:54.484 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:44:54.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-base 2023-09-23T14:44:55.009 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:44:55.621 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T14:44:56.046 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T14:44:56.172 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T14:44:56.401 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T14:44:56.572 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T14:44:56.591 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T14:44:56.604 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:02 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:44:56.712 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T14:44:57.083 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T14:44:57.083 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T14:44:57.083 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T14:44:57.083 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T14:44:57.083 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T14:44:57.084 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T14:44:57.085 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T14:44:57.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T14:44:57.087 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T14:44:58.027 INFO:teuthology.orchestra.run.smithi027.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:44:58.149 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:44:58.150 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:44:58.150 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:44:58.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install cephadm 2023-09-23T14:44:58.554 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T14:44:58.554 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:44:58.620 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:00.238 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:06 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:01.764 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:45:01.765 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:01.766 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:45:01.766 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:01.766 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 80 k 2023-09-23T14:45:01.766 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 363 k 2023-09-23T14:45:01.766 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:45:02.203 INFO:teuthology.orchestra.run.smithi027.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 183 kB/s | 80 kB 00:00 2023-09-23T14:45:02.204 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:02.204 INFO:teuthology.orchestra.run.smithi027.stdout:Total 183 kB/s | 80 kB 00:00 2023-09-23T14:45:02.204 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:45:02.213 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:45:02.213 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:45:02.215 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:45:02.219 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:45:02.260 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:45:03.823 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:45:04.026 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:04.027 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T14:45:04.028 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T14:45:04.029 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T14:45:04.030 INFO:teuthology.orchestra.run.smithi017.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T14:45:04.031 INFO:teuthology.orchestra.run.smithi017.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T14:45:04.031 INFO:teuthology.orchestra.run.smithi017.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T14:45:04.031 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:04.031 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:45:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:04.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-base 2023-09-23T14:45:04.295 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:04.468 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:04.560 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:05.571 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:05.571 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:11 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:06.168 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:15 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:07.573 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:45:07.694 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:45:07.695 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:45:07.695 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:45:07.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install cephadm 2023-09-23T14:45:08.170 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:09.801 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:18 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:11.322 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:11.323 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 80 k 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 363 k 2023-09-23T14:45:11.324 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:45:11.486 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 491 kB/s | 80 kB 00:00 2023-09-23T14:45:11.487 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:11.488 INFO:teuthology.orchestra.run.smithi017.stdout:Total 487 kB/s | 80 kB 00:00 2023-09-23T14:45:11.488 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:45:11.497 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:45:11.497 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:45:11.498 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:45:11.502 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:45:11.547 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:45:12.070 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:45:12.268 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:12.268 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:45:12.268 INFO:teuthology.orchestra.run.smithi027.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:12.268 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:12.268 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:45:12.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T14:45:12.778 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:13.368 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:13.578 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:13.752 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:14.587 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:20 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:14.850 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T14:45:14.851 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:23 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:16.127 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:45:16.128 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:16.129 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 161 k 2023-09-23T14:45:16.129 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 490 k 2023-09-23T14:45:16.129 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:45:16.712 INFO:teuthology.orchestra.run.smithi027.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 277 kB/s | 161 kB 00:00 2023-09-23T14:45:16.712 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:16.713 INFO:teuthology.orchestra.run.smithi027.stdout:Total 276 kB/s | 161 kB 00:00 2023-09-23T14:45:16.713 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:45:16.725 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:45:16.725 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:45:16.782 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:45:16.787 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:45:16.905 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:45:17.090 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:17.480 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:18.569 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:18.570 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:24 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:23.096 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:45:23.294 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:23.294 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:45:23.294 INFO:teuthology.orchestra.run.smithi017.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:23.294 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:23.294 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:45:23.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T14:45:23.800 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:25.519 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:34 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:27.083 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:27.084 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 161 k 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 490 k 2023-09-23T14:45:27.085 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:45:27.278 INFO:teuthology.orchestra.run.smithi017.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 837 kB/s | 161 kB 00:00 2023-09-23T14:45:27.278 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:27.278 INFO:teuthology.orchestra.run.smithi017.stdout:Total 832 kB/s | 161 kB 00:00 2023-09-23T14:45:27.279 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:45:27.290 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:45:27.290 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:45:27.347 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:45:27.351 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:45:27.468 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:45:27.685 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:28.064 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:29.110 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T14:45:29.110 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:38 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:29.141 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:45:29.338 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:29.338 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:45:29.338 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:29.339 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:29.339 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:45:29.473 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr 2023-09-23T14:45:29.841 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:31.607 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:37 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:33.008 INFO:teuthology.orchestra.run.smithi027.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:45:33.128 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:45:33.129 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:45:33.130 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:45:33.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T14:45:33.599 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:35.230 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:41 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:36.775 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:45:36.776 INFO:teuthology.orchestra.run.smithi027.stdout:=================================================================================================== 2023-09-23T14:45:36.776 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:45:36.776 INFO:teuthology.orchestra.run.smithi027.stdout:=================================================================================================== 2023-09-23T14:45:36.776 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:45:36.776 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:36.777 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:45:36.778 INFO:teuthology.orchestra.run.smithi027.stdout:=================================================================================================== 2023-09-23T14:45:36.778 INFO:teuthology.orchestra.run.smithi027.stdout:Install 6 Packages 2023-09-23T14:45:36.778 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:36.779 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 2.0 M 2023-09-23T14:45:36.779 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 8.0 M 2023-09-23T14:45:36.779 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:45:37.158 INFO:teuthology.orchestra.run.smithi027.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 105 kB/s | 23 kB 00:00 2023-09-23T14:45:37.183 INFO:teuthology.orchestra.run.smithi027.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 59 kB/s | 14 kB 00:00 2023-09-23T14:45:37.262 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:45:37.384 INFO:teuthology.orchestra.run.smithi027.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 3.8 MB/s | 1.7 MB 00:00 2023-09-23T14:45:37.463 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:37.463 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:45:37.463 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:45:37.463 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:37.463 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:45:37.575 INFO:teuthology.orchestra.run.smithi027.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 224 kB/s | 43 kB 00:00 2023-09-23T14:45:37.651 INFO:teuthology.orchestra.run.smithi027.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 68 kB/s | 34 kB 00:00 2023-09-23T14:45:37.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr 2023-09-23T14:45:37.893 INFO:teuthology.orchestra.run.smithi027.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 277 kB/s | 196 kB 00:00 2023-09-23T14:45:37.896 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:37.897 INFO:teuthology.orchestra.run.smithi027.stdout:Total 1.8 MB/s | 2.0 MB 00:01 2023-09-23T14:45:37.910 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:45:37.921 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:45:37.921 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:45:37.987 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:45:37.991 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:45:38.091 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:38.274 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:45:38.447 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T14:45:38.645 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T14:45:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T14:45:38.927 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T14:45:39.256 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T14:45:39.438 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T14:45:39.684 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T14:45:39.773 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:40.770 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T14:45:40.770 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:46 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:41.184 INFO:teuthology.orchestra.run.smithi017.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:45:41.303 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:45:41.304 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:45:41.304 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:45:41.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T14:45:41.790 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:43.403 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:52 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:44.914 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:45:44.915 INFO:teuthology.orchestra.run.smithi017.stdout:=================================================================================================== 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout:=================================================================================================== 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T14:45:44.916 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout:=================================================================================================== 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout:Install 6 Packages 2023-09-23T14:45:44.917 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:44.918 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 2.0 M 2023-09-23T14:45:44.918 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 8.0 M 2023-09-23T14:45:44.919 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:45:45.447 INFO:teuthology.orchestra.run.smithi017.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 64 kB/s | 23 kB 00:00 2023-09-23T14:45:45.522 INFO:teuthology.orchestra.run.smithi017.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 33 kB/s | 14 kB 00:00 2023-09-23T14:45:45.547 INFO:teuthology.orchestra.run.smithi017.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 335 kB/s | 34 kB 00:00 2023-09-23T14:45:45.664 INFO:teuthology.orchestra.run.smithi017.stdout:(4/6): python3-routes-2.4.1-12.el8.noarch.rpm 1.3 MB/s | 196 kB 00:00 2023-09-23T14:45:45.756 INFO:teuthology.orchestra.run.smithi017.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 204 kB/s | 43 kB 00:00 2023-09-23T14:45:46.590 INFO:teuthology.orchestra.run.smithi017.stdout:(6/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 1.1 MB/s | 1.7 MB 00:01 2023-09-23T14:45:46.591 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:46.592 INFO:teuthology.orchestra.run.smithi017.stdout:Total 1.2 MB/s | 2.0 MB 00:01 2023-09-23T14:45:46.606 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:45:46.617 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:45:46.617 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:45:46.682 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:45:46.686 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:45:47.012 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:45:47.167 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T14:45:47.374 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T14:45:47.576 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T14:45:47.698 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T14:45:48.263 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T14:45:48.450 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T14:45:48.681 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T14:45:48.681 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T14:45:48.681 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T14:45:48.682 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T14:45:48.682 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T14:45:48.682 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T14:45:49.797 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T14:45:49.797 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:02:58 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:45:50.157 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:45:50.359 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:50.359 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:45:50.359 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:50.359 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:50.360 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:45:50.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T14:45:50.884 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:45:52.488 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:02:58 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:45:54.000 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:45:54.001 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================================= 2023-09-23T14:45:54.001 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:45:54.001 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================================= 2023-09-23T14:45:54.001 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:45:54.001 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T14:45:54.002 INFO:teuthology.orchestra.run.smithi027.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T14:45:54.003 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:54.003 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:45:54.003 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================================= 2023-09-23T14:45:54.003 INFO:teuthology.orchestra.run.smithi027.stdout:Install 8 Packages 2023-09-23T14:45:54.003 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:45:54.004 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 35 M 2023-09-23T14:45:54.004 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 210 M 2023-09-23T14:45:54.004 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:45:54.189 INFO:teuthology.orchestra.run.smithi027.stdout:(1/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 3.4 MB/s | 644 kB 00:00 2023-09-23T14:45:54.256 INFO:teuthology.orchestra.run.smithi027.stdout:(2/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 678 kB/s | 171 kB 00:00 2023-09-23T14:45:54.498 INFO:teuthology.orchestra.run.smithi027.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 15 MB/s | 3.7 MB 00:00 2023-09-23T14:45:54.657 INFO:teuthology.orchestra.run.smithi027.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 29 MB/s | 14 MB 00:00 2023-09-23T14:45:54.682 INFO:teuthology.orchestra.run.smithi027.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.2 MB/s | 225 kB 00:00 2023-09-23T14:45:54.874 INFO:teuthology.orchestra.run.smithi027.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 22 MB/s | 4.9 MB 00:00 2023-09-23T14:45:54.941 INFO:teuthology.orchestra.run.smithi027.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-09-23T14:45:55.158 INFO:teuthology.orchestra.run.smithi027.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 6.4 MB/s | 7.4 MB 00:01 2023-09-23T14:45:55.159 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:45:55.159 INFO:teuthology.orchestra.run.smithi027.stdout:Total 31 MB/s | 35 MB 00:01 2023-09-23T14:45:55.405 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:45:55.425 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:45:55.425 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:45:55.737 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:45:55.741 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:45:56.301 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:45:56.434 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T14:45:56.556 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T14:45:56.692 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:45:57.328 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:45:57.493 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T14:45:58.110 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T14:45:58.300 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T14:45:59.042 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T14:45:59.350 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T14:45:59.526 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:45:59.768 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:45:59.967 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:59.967 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:45:59.968 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:46:00.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T14:46:00.464 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:01.402 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:46:02.153 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:11 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:02.663 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T14:46:02.791 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:46:03.064 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T14:46:03.640 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:46:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================================= 2023-09-23T14:46:03.642 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:46:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================================= 2023-09-23T14:46:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:46:03.642 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T14:46:03.643 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:03.644 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:46:03.644 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================================= 2023-09-23T14:46:03.644 INFO:teuthology.orchestra.run.smithi017.stdout:Install 8 Packages 2023-09-23T14:46:03.644 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:03.644 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 35 M 2023-09-23T14:46:03.645 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 210 M 2023-09-23T14:46:03.645 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:46:03.828 INFO:teuthology.orchestra.run.smithi017.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 934 kB/s | 171 kB 00:00 2023-09-23T14:46:03.869 INFO:teuthology.orchestra.run.smithi017.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.8 MB/s | 644 kB 00:00 2023-09-23T14:46:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T14:46:04.148 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:10 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:04.203 INFO:teuthology.orchestra.run.smithi017.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 11 MB/s | 3.7 MB 00:00 2023-09-23T14:46:04.295 INFO:teuthology.orchestra.run.smithi017.stdout:(4/8): ceph-mgr-diskprediction-local-17.2.6-112 11 MB/s | 7.4 MB 00:00 2023-09-23T14:46:04.481 INFO:teuthology.orchestra.run.smithi017.stdout:(5/8): python3-scipy-1.0.0-21.module+el8.5.0+10 21 MB/s | 14 MB 00:00 2023-09-23T14:46:04.521 INFO:teuthology.orchestra.run.smithi017.stdout:(6/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 709 kB/s | 225 kB 00:00 2023-09-23T14:46:04.655 INFO:teuthology.orchestra.run.smithi017.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 14 MB/s | 4.9 MB 00:00 2023-09-23T14:46:04.746 INFO:teuthology.orchestra.run.smithi017.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 18 MB/s | 4.7 MB 00:00 2023-09-23T14:46:04.748 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:04.748 INFO:teuthology.orchestra.run.smithi017.stdout:Total 32 MB/s | 35 MB 00:01 2023-09-23T14:46:04.993 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:46:05.013 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:46:05.013 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:46:05.325 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:46:05.330 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:46:05.890 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:46:06.046 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T14:46:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T14:46:06.313 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:46:06.959 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:46:07.146 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T14:46:07.767 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T14:46:07.955 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T14:46:08.714 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T14:46:09.038 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T14:46:09.238 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:46:11.115 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:46:12.384 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T14:46:12.553 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T14:46:12.558 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:46:12.761 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:12.761 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:46:12.761 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:46:12.761 INFO:teuthology.orchestra.run.smithi027.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T14:46:12.761 INFO:teuthology.orchestra.run.smithi027.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:12.762 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:46:12.824 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T14:46:12.825 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T14:46:12.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-rook 2023-09-23T14:46:13.276 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:13.922 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T14:46:13.922 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:22 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:14.911 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:20 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:16.454 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:46:16.456 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================== 2023-09-23T14:46:16.456 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================== 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T14:46:16.457 INFO:teuthology.orchestra.run.smithi027.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T14:46:16.458 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout:======================================================================================================== 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout:Install 13 Packages 2023-09-23T14:46:16.459 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:16.460 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 1.9 M 2023-09-23T14:46:16.460 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 25 M 2023-09-23T14:46:16.460 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:46:16.882 INFO:teuthology.orchestra.run.smithi027.stdout:(1/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 181 kB/s | 47 kB 00:00 2023-09-23T14:46:16.907 INFO:teuthology.orchestra.run.smithi027.stdout:(2/13): python3-certifi-2018.10.15-7.el8.noarch 57 kB/s | 16 kB 00:00 2023-09-23T14:46:16.932 INFO:teuthology.orchestra.run.smithi027.stdout:(3/13): python3-cachetools-3.1.1-4.el8.noarch.r 106 kB/s | 33 kB 00:00 2023-09-23T14:46:17.074 INFO:teuthology.orchestra.run.smithi027.stdout:(4/13): python3-google-auth-1.1.1-10.el8.noarch 423 kB/s | 81 kB 00:00 2023-09-23T14:46:17.099 INFO:teuthology.orchestra.run.smithi027.stdout:(5/13): python3-rsa-4.9-2.el8.noarch.rpm 384 kB/s | 64 kB 00:00 2023-09-23T14:46:17.200 INFO:teuthology.orchestra.run.smithi027.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 489 kB/s | 61 kB 00:00 2023-09-23T14:46:17.275 INFO:teuthology.orchestra.run.smithi027.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 244 kB/s | 43 kB 00:00 2023-09-23T14:46:17.300 INFO:teuthology.orchestra.run.smithi027.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.5 MB/s | 155 kB 00:00 2023-09-23T14:46:17.426 INFO:teuthology.orchestra.run.smithi027.stdout:(9/13): python3-kubernetes-11.0.0-6.el8.noarch. 2.2 MB/s | 1.2 MB 00:00 2023-09-23T14:46:17.443 INFO:teuthology.orchestra.run.smithi027.stdout:(10/13): python3-jsonpatch-1.21-2.el8.noarch.rp 187 kB/s | 27 kB 00:00 2023-09-23T14:46:17.460 INFO:teuthology.orchestra.run.smithi027.stdout:(11/13): python3-pyasn1-modules-0.3.7-6.el8.noa 598 kB/s | 110 kB 00:00 2023-09-23T14:46:17.526 INFO:teuthology.orchestra.run.smithi027.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 239 kB/s | 20 kB 00:00 2023-09-23T14:46:17.618 INFO:teuthology.orchestra.run.smithi027.stdout:(13/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 658 kB/s | 126 kB 00:00 2023-09-23T14:46:17.622 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:17.622 INFO:teuthology.orchestra.run.smithi027.stdout:Total 1.7 MB/s | 1.9 MB 00:01 2023-09-23T14:46:17.693 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:46:17.711 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:46:17.712 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:46:17.796 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:46:17.801 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:46:18.287 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:46:18.509 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T14:46:18.712 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T14:46:18.872 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T14:46:19.005 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T14:46:19.228 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T14:46:19.448 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T14:46:19.627 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T14:46:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T14:46:19.954 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T14:46:20.128 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T14:46:21.260 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T14:46:21.474 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T14:46:21.589 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T14:46:21.827 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T14:46:21.828 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T14:46:22.958 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T14:46:22.959 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:28 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:23.252 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:46:23.456 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T14:46:23.457 INFO:teuthology.orchestra.run.smithi017.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T14:46:23.458 INFO:teuthology.orchestra.run.smithi017.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T14:46:23.458 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:23.458 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:46:23.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-rook 2023-09-23T14:46:23.973 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:25.566 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:34 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:27.122 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:46:27.124 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================== 2023-09-23T14:46:27.124 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:46:27.124 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================== 2023-09-23T14:46:27.124 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T14:46:27.125 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T14:46:27.126 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:27.127 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:46:27.127 INFO:teuthology.orchestra.run.smithi017.stdout:======================================================================================================== 2023-09-23T14:46:27.127 INFO:teuthology.orchestra.run.smithi017.stdout:Install 13 Packages 2023-09-23T14:46:27.127 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:27.128 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 1.9 M 2023-09-23T14:46:27.128 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 25 M 2023-09-23T14:46:27.128 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:46:27.376 INFO:teuthology.orchestra.run.smithi017.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 154 kB/s | 16 kB 00:00 2023-09-23T14:46:27.401 INFO:teuthology.orchestra.run.smithi017.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 252 kB/s | 33 kB 00:00 2023-09-23T14:46:27.468 INFO:teuthology.orchestra.run.smithi017.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 889 kB/s | 81 kB 00:00 2023-09-23T14:46:27.527 INFO:teuthology.orchestra.run.smithi017.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 1.1 MB/s | 64 kB 00:00 2023-09-23T14:46:27.552 INFO:teuthology.orchestra.run.smithi017.stdout:(5/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 168 kB/s | 47 kB 00:00 2023-09-23T14:46:27.586 INFO:teuthology.orchestra.run.smithi017.stdout:(6/13): python3-kubernetes-11.0.0-6.el8.noarch. 6.3 MB/s | 1.2 MB 00:00 2023-09-23T14:46:27.619 INFO:teuthology.orchestra.run.smithi017.stdout:(7/13): python3-websocket-client-0.56.0-5.el8.n 664 kB/s | 61 kB 00:00 2023-09-23T14:46:27.695 INFO:teuthology.orchestra.run.smithi017.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 300 kB/s | 43 kB 00:00 2023-09-23T14:46:27.720 INFO:teuthology.orchestra.run.smithi017.stdout:(9/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.2 MB/s | 155 kB 00:00 2023-09-23T14:46:27.745 INFO:teuthology.orchestra.run.smithi017.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 940 kB/s | 110 kB 00:00 2023-09-23T14:46:27.871 INFO:teuthology.orchestra.run.smithi017.stdout:(11/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 841 kB/s | 126 kB 00:00 2023-09-23T14:46:27.895 INFO:teuthology.orchestra.run.smithi017.stdout:(12/13): python3-jsonpointer-1.10-11.el8.noarch 133 kB/s | 20 kB 00:00 2023-09-23T14:46:27.921 INFO:teuthology.orchestra.run.smithi017.stdout:(13/13): python3-jsonpatch-1.21-2.el8.noarch.rp 117 kB/s | 27 kB 00:00 2023-09-23T14:46:27.922 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:27.922 INFO:teuthology.orchestra.run.smithi017.stdout:Total 2.4 MB/s | 1.9 MB 00:00 2023-09-23T14:46:27.989 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:46:28.006 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:46:28.007 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:46:28.084 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:46:28.088 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:46:28.569 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:46:28.888 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T14:46:29.116 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T14:46:29.317 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T14:46:29.474 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:46:29.485 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T14:46:29.679 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:29.679 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:46:29.679 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:46:29.679 INFO:teuthology.orchestra.run.smithi027.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T14:46:29.679 INFO:teuthology.orchestra.run.smithi027.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T14:46:29.680 INFO:teuthology.orchestra.run.smithi027.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T14:46:29.681 INFO:teuthology.orchestra.run.smithi027.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T14:46:29.681 INFO:teuthology.orchestra.run.smithi027.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T14:46:29.681 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:29.681 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:46:29.698 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T14:46:29.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T14:46:29.909 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T14:46:30.149 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T14:46:30.191 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:30.343 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T14:46:30.501 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T14:46:30.695 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T14:46:31.755 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:37 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:31.857 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T14:46:32.103 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T14:46:32.276 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T14:46:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T14:46:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T14:46:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T14:46:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T14:46:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T14:46:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T14:46:33.299 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================================== 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================================== 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T14:46:33.300 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================================== 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout:Install 5 Packages 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 6.0 M 2023-09-23T14:46:33.301 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 26 M 2023-09-23T14:46:33.302 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:46:33.715 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T14:46:33.715 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:42 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:33.725 INFO:teuthology.orchestra.run.smithi027.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 419 kB/s | 121 kB 00:00 2023-09-23T14:46:33.942 INFO:teuthology.orchestra.run.smithi027.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.4 MB/s | 538 kB 00:00 2023-09-23T14:46:34.242 INFO:teuthology.orchestra.run.smithi027.stdout:(3/5): python3-babel-2.5.1-7.el8.noarch.rpm 16 MB/s | 4.8 MB 00:00 2023-09-23T14:46:34.267 INFO:teuthology.orchestra.run.smithi027.stdout:(4/5): python3-natsort-7.1.1-2.el8.noarch.rpm 72 kB/s | 60 kB 00:00 2023-09-23T14:46:34.401 INFO:teuthology.orchestra.run.smithi027.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 493 kB/s | 476 kB 00:00 2023-09-23T14:46:34.403 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:34.403 INFO:teuthology.orchestra.run.smithi027.stdout:Total 5.4 MB/s | 6.0 MB 00:01 2023-09-23T14:46:34.457 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:46:34.470 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:46:34.470 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:46:34.554 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:46:34.558 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:46:35.579 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:46:35.909 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T14:46:36.174 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T14:46:36.509 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T14:46:36.758 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T14:46:36.912 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T14:46:37.165 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T14:46:37.166 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T14:46:37.166 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T14:46:37.166 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T14:46:37.166 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T14:46:38.256 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T14:46:38.256 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:42.214 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:46:42.418 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:42.418 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:46:42.418 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T14:46:42.419 INFO:teuthology.orchestra.run.smithi017.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T14:46:42.420 INFO:teuthology.orchestra.run.smithi017.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T14:46:42.420 INFO:teuthology.orchestra.run.smithi017.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T14:46:42.420 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:42.420 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:46:42.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T14:46:42.926 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:44.605 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:53 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:46.065 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:46:46.066 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================================== 2023-09-23T14:46:46.066 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:46:46.066 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================================== 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T14:46:46.067 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:46.068 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:46:46.068 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================================== 2023-09-23T14:46:46.068 INFO:teuthology.orchestra.run.smithi017.stdout:Install 5 Packages 2023-09-23T14:46:46.068 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:46:46.069 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 6.0 M 2023-09-23T14:46:46.069 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 26 M 2023-09-23T14:46:46.069 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:46:46.489 INFO:teuthology.orchestra.run.smithi017.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 235 kB/s | 60 kB 00:00 2023-09-23T14:46:46.514 INFO:teuthology.orchestra.run.smithi017.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 430 kB/s | 121 kB 00:00 2023-09-23T14:46:46.614 INFO:teuthology.orchestra.run.smithi017.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 1.2 MB/s | 476 kB 00:00 2023-09-23T14:46:46.656 INFO:teuthology.orchestra.run.smithi017.stdout:(4/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 3.1 MB/s | 538 kB 00:00 2023-09-23T14:46:46.748 INFO:teuthology.orchestra.run.smithi017.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 21 MB/s | 4.8 MB 00:00 2023-09-23T14:46:46.749 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:46.749 INFO:teuthology.orchestra.run.smithi017.stdout:Total 8.7 MB/s | 6.0 MB 00:00 2023-09-23T14:46:46.806 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:46:46.818 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:46:46.819 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:46:46.902 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:46:46.906 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:46:47.837 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:46:47.932 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:46:48.040 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:48.040 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:46:48.040 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:46:48.040 INFO:teuthology.orchestra.run.smithi027.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T14:46:48.040 INFO:teuthology.orchestra.run.smithi027.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T14:46:48.041 INFO:teuthology.orchestra.run.smithi027.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T14:46:48.041 INFO:teuthology.orchestra.run.smithi027.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T14:46:48.041 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:48.041 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:46:48.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install ceph-fuse 2023-09-23T14:46:48.270 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T14:46:48.528 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T14:46:48.575 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:46:48.849 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T14:46:49.128 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T14:46:49.333 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T14:46:49.577 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T14:46:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T14:46:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T14:46:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T14:46:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T14:46:50.322 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:03:56 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:46:50.663 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T14:46:50.663 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:03:59 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:46:51.852 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:46:51.853 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:46:51.853 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repository Size 2023-09-23T14:46:51.853 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:46:51.853 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:46:51.853 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-fuse 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout:Installing dependencies: 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout:Install 2 Packages 2023-09-23T14:46:51.854 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:46:51.855 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 957 k 2023-09-23T14:46:51.855 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 3.0 M 2023-09-23T14:46:51.855 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:46:52.026 INFO:teuthology.orchestra.run.smithi027.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 482 kB/s | 83 kB 00:00 2023-09-23T14:46:52.243 INFO:teuthology.orchestra.run.smithi027.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.2 MB/s | 874 kB 00:00 2023-09-23T14:46:52.244 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:46:52.244 INFO:teuthology.orchestra.run.smithi027.stdout:Total 2.4 MB/s | 957 kB 00:00 2023-09-23T14:46:52.256 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:46:52.278 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:46:52.278 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:46:52.356 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:46:52.361 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:46:52.522 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:46:52.775 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T14:46:52.935 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T14:46:53.302 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T14:46:53.302 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T14:46:54.381 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T14:46:54.381 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:00 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:00.685 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T14:47:00.890 INFO:teuthology.orchestra.run.smithi017.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T14:47:00.891 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:00.891 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:01.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install ceph-fuse 2023-09-23T14:47:01.407 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:03.154 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:12 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:04.069 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:47:04.267 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:04.267 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:47:04.267 INFO:teuthology.orchestra.run.smithi027.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T14:47:04.267 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:04.267 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:04.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librados-devel 2023-09-23T14:47:04.662 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:04.662 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:04.663 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repository Size 2023-09-23T14:47:04.663 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:04.663 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:47:04.663 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout:Installing dependencies: 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:47:04.664 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:04.665 INFO:teuthology.orchestra.run.smithi017.stdout:Install 2 Packages 2023-09-23T14:47:04.665 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:04.665 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 957 k 2023-09-23T14:47:04.666 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 3.0 M 2023-09-23T14:47:04.666 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:47:04.777 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:04.807 INFO:teuthology.orchestra.run.smithi017.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 578 kB/s | 83 kB 00:00 2023-09-23T14:47:04.916 INFO:teuthology.orchestra.run.smithi017.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 3.4 MB/s | 874 kB 00:00 2023-09-23T14:47:04.917 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:47:04.917 INFO:teuthology.orchestra.run.smithi017.stdout:Total 3.7 MB/s | 957 kB 00:00 2023-09-23T14:47:04.928 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:47:04.950 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:47:04.950 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:47:05.029 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:47:05.033 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:47:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:47:05.428 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T14:47:05.596 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T14:47:05.945 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T14:47:05.946 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T14:47:06.604 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:12 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:07.046 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T14:47:07.046 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:16 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:08.142 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:08.143 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:47:08.144 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:08.144 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 130 k 2023-09-23T14:47:08.144 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 464 k 2023-09-23T14:47:08.144 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:47:08.354 INFO:teuthology.orchestra.run.smithi027.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 618 kB/s | 130 kB 00:00 2023-09-23T14:47:08.354 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:47:08.355 INFO:teuthology.orchestra.run.smithi027.stdout:Total 615 kB/s | 130 kB 00:00 2023-09-23T14:47:08.355 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:47:08.360 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:47:08.361 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:47:08.420 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:47:08.424 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:47:08.544 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:47:08.690 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:08.921 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:10.022 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:10.022 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:16 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:15.238 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:47:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:15.437 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:47:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T14:47:15.437 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:15.437 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:15.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados-devel 2023-09-23T14:47:15.972 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:17.641 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:26 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:18.167 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:47:18.365 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:18.365 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:47:18.365 INFO:teuthology.orchestra.run.smithi027.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:47:18.365 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:18.365 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:18.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install libcephfs2 2023-09-23T14:47:18.872 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:19.173 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:19.174 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:19.175 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 130 k 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 464 k 2023-09-23T14:47:19.176 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:47:19.362 INFO:teuthology.orchestra.run.smithi017.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 697 kB/s | 130 kB 00:00 2023-09-23T14:47:19.362 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:47:19.362 INFO:teuthology.orchestra.run.smithi017.stdout:Total 692 kB/s | 130 kB 00:00 2023-09-23T14:47:19.363 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:47:19.369 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:47:19.369 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:47:19.427 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:47:19.431 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:47:19.563 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:47:19.726 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:19.966 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:20.445 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:26 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:21.086 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:21.086 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:30 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:21.856 INFO:teuthology.orchestra.run.smithi027.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:21.973 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:21.974 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:21.975 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:22.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install libcephfs-devel 2023-09-23T14:47:22.467 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:23.996 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:29 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:25.516 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:47:25.517 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:47:25.518 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:25.518 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 25 k 2023-09-23T14:47:25.518 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 109 k 2023-09-23T14:47:25.518 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:47:25.813 INFO:teuthology.orchestra.run.smithi027.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 84 kB/s | 25 kB 00:00 2023-09-23T14:47:25.813 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:47:25.814 INFO:teuthology.orchestra.run.smithi027.stdout:Total 84 kB/s | 25 kB 00:00 2023-09-23T14:47:25.814 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:47:25.818 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:47:25.818 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:47:25.820 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:47:25.824 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:47:25.878 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:47:26.054 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:26.875 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:47:27.077 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:27.077 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:47:27.077 INFO:teuthology.orchestra.run.smithi017.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:47:27.077 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:27.077 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:27.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs2 2023-09-23T14:47:27.335 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:27.335 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:33 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:27.575 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:29.412 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:38 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:30.817 INFO:teuthology.orchestra.run.smithi017.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:30.934 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:30.935 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:47:30.935 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:31.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install libcephfs-devel 2023-09-23T14:47:31.422 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:33.033 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:42 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:34.528 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:34.529 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:34.529 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:47:34.530 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:34.531 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 25 k 2023-09-23T14:47:34.531 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 109 k 2023-09-23T14:47:34.531 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:47:34.682 INFO:teuthology.orchestra.run.smithi017.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 164 kB/s | 25 kB 00:00 2023-09-23T14:47:34.682 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:47:34.682 INFO:teuthology.orchestra.run.smithi017.stdout:Total 162 kB/s | 25 kB 00:00 2023-09-23T14:47:34.683 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:47:34.686 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:47:34.687 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:47:34.688 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:47:34.692 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:47:34.741 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:47:34.916 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:35.146 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:36.198 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:47:36.198 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:45 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:36.618 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:47:36.817 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:36.818 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:47:36.818 INFO:teuthology.orchestra.run.smithi027.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:47:36.818 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:47:36.818 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:36.958 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librados2 2023-09-23T14:47:37.333 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:39.004 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:45 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:40.413 INFO:teuthology.orchestra.run.smithi027.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:40.534 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:40.536 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:40.536 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:40.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install librbd1 2023-09-23T14:47:41.025 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:42.806 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:48 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:44.190 INFO:teuthology.orchestra.run.smithi027.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:44.308 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:44.309 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:44.309 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:44.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rados 2023-09-23T14:47:44.799 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:46.266 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:52 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:46.365 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:47:46.562 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:46.562 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:47:46.562 INFO:teuthology.orchestra.run.smithi017.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:47:46.562 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:47:46.562 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:46.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librados2 2023-09-23T14:47:47.067 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:47.646 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:47.765 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:47.765 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:47.765 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:47.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rgw 2023-09-23T14:47:48.260 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:48.883 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:04:57 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:49.872 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:55 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:50.299 INFO:teuthology.orchestra.run.smithi017.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:50.418 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:50.419 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:47:50.419 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:50.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install librbd1 2023-09-23T14:47:50.897 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:51.271 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:51.391 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:51.392 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:51.393 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:51.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-cephfs 2023-09-23T14:47:51.890 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:52.666 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:01 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:53.550 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:04:59 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:54.072 INFO:teuthology.orchestra.run.smithi017.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:54.193 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:54.194 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:47:54.194 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:54.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rados 2023-09-23T14:47:54.808 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:54.918 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:55.036 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:55.037 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:55.037 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:55.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install python3-rbd 2023-09-23T14:47:55.522 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:56.424 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:05 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:47:57.120 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:03 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:47:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:57.934 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:47:57.935 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:47:57.935 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:47:58.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rgw 2023-09-23T14:47:58.529 INFO:teuthology.orchestra.run.smithi027.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:47:58.562 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:47:58.646 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:47:58.647 INFO:teuthology.orchestra.run.smithi027.stdout:Nothing to do. 2023-09-23T14:47:58.648 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:47:58.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-fuse 2023-09-23T14:47:59.124 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:00.272 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:09 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:00.836 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:06 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:01.689 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:48:01.809 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:01.810 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:48:01.810 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:01.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-cephfs 2023-09-23T14:48:02.342 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:48:02.343 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:02.343 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2023-09-23T14:48:02.343 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:48:02.344 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:02.345 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 90 k 2023-09-23T14:48:02.345 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 264 k 2023-09-23T14:48:02.345 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:48:02.346 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:02.656 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 289 kB/s | 90 kB 00:00 2023-09-23T14:48:02.657 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:02.657 INFO:teuthology.orchestra.run.smithi027.stdout:Total 288 kB/s | 90 kB 00:00 2023-09-23T14:48:02.657 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:48:02.668 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:48:02.668 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:48:02.726 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:48:02.730 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:48:02.843 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:48:02.983 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:03.209 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:04.098 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:13 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:04.271 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:04.271 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:10 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:05.515 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:48:05.631 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:05.633 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:48:05.633 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:05.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install python3-rbd 2023-09-23T14:48:06.127 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:07.722 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:16 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:09.127 INFO:teuthology.orchestra.run.smithi017.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T14:48:09.249 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:09.250 INFO:teuthology.orchestra.run.smithi017.stdout:Nothing to do. 2023-09-23T14:48:09.251 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:09.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-fuse 2023-09-23T14:48:09.738 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:11.423 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:20 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:12.943 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:12.943 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:12.943 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:48:12.944 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:12.945 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 90 k 2023-09-23T14:48:12.945 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 264 k 2023-09-23T14:48:12.945 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:48:13.108 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 554 kB/s | 90 kB 00:00 2023-09-23T14:48:13.108 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:13.108 INFO:teuthology.orchestra.run.smithi017.stdout:Total 550 kB/s | 90 kB 00:00 2023-09-23T14:48:13.109 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:48:13.119 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:48:13.119 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:48:13.167 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:48:13.176 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:48:13.181 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:48:13.288 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:48:13.368 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:13.369 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:48:13.369 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:13.369 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:13.369 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:48:13.453 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:13.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-mirror 2023-09-23T14:48:13.696 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:13.877 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:14.783 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:14.783 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:23 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:15.523 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:21 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:17.053 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout: Package Arch Version Repo Size 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T14:48:17.054 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 3.5 M 2023-09-23T14:48:17.055 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 16 M 2023-09-23T14:48:17.056 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:48:17.567 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.8 MB/s | 3.5 MB 00:00 2023-09-23T14:48:17.568 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:17.568 INFO:teuthology.orchestra.run.smithi027.stdout:Total 6.7 MB/s | 3.5 MB 00:00 2023-09-23T14:48:17.568 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:48:17.581 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:48:17.581 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:48:17.653 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:48:17.657 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:48:18.120 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:48:18.292 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:18.651 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:19.703 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:19.703 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:25 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:26.247 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:48:26.443 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:26.443 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:48:26.443 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:26.444 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:26.444 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:26.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-mirror 2023-09-23T14:48:26.954 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:28.491 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:37 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:30.012 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:30.012 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:30.012 INFO:teuthology.orchestra.run.smithi017.stdout: Package Arch Version Repo Size 2023-09-23T14:48:30.012 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:30.012 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:30.013 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 3.5 M 2023-09-23T14:48:30.014 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 16 M 2023-09-23T14:48:30.014 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:48:30.542 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.5 MB/s | 3.5 MB 00:00 2023-09-23T14:48:30.543 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:30.543 INFO:teuthology.orchestra.run.smithi017.stdout:Total 6.5 MB/s | 3.5 MB 00:00 2023-09-23T14:48:30.544 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:48:30.556 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:48:30.557 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:48:30.629 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:48:30.633 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:48:30.990 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:48:31.125 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:48:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:31.188 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:48:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:31.188 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:48:31.311 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:31.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo yum -y install rbd-nbd 2023-09-23T14:48:31.689 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:31.866 INFO:teuthology.orchestra.run.smithi027.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:32.767 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:32.768 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:41 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:33.421 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:39 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:34.926 INFO:teuthology.orchestra.run.smithi027.stdout:Dependencies resolved. 2023-09-23T14:48:34.926 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout: Package Architecture Version Repository Size 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout:Installing: 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction Summary 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout:================================================================================ 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout:Install 1 Package 2023-09-23T14:48:34.927 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:34.928 INFO:teuthology.orchestra.run.smithi027.stdout:Total download size: 173 k 2023-09-23T14:48:34.928 INFO:teuthology.orchestra.run.smithi027.stdout:Installed size: 552 k 2023-09-23T14:48:34.928 INFO:teuthology.orchestra.run.smithi027.stdout:Downloading Packages: 2023-09-23T14:48:35.145 INFO:teuthology.orchestra.run.smithi027.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 797 kB/s | 173 kB 00:00 2023-09-23T14:48:35.145 INFO:teuthology.orchestra.run.smithi027.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:35.146 INFO:teuthology.orchestra.run.smithi027.stdout:Total 792 kB/s | 173 kB 00:00 2023-09-23T14:48:35.146 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction check 2023-09-23T14:48:35.158 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction check succeeded. 2023-09-23T14:48:35.158 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction test 2023-09-23T14:48:35.215 INFO:teuthology.orchestra.run.smithi027.stdout:Transaction test succeeded. 2023-09-23T14:48:35.219 INFO:teuthology.orchestra.run.smithi027.stdout:Running transaction 2023-09-23T14:48:35.339 INFO:teuthology.orchestra.run.smithi027.stdout: Preparing : 1/1 2023-09-23T14:48:35.487 INFO:teuthology.orchestra.run.smithi027.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:35.726 INFO:teuthology.orchestra.run.smithi027.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:36.865 INFO:teuthology.orchestra.run.smithi027.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:36.865 INFO:teuthology.orchestra.run.smithi027.stdout:Last metadata expiration check: 0:05:42 ago on Sat 23 Sep 2023 02:42:54 PM UTC. 2023-09-23T14:48:42.366 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:48:42.563 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:42.563 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:48:42.563 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:42.563 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:42.563 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:42.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo yum -y install rbd-nbd 2023-09-23T14:48:43.070 INFO:teuthology.orchestra.run.smithi017.stdout:Updating Subscription Management repositories. 2023-09-23T14:48:44.621 INFO:teuthology.orchestra.run.smithi027.stdout:Installed products updated. 2023-09-23T14:48:44.730 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:53 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:44.821 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:44.821 INFO:teuthology.orchestra.run.smithi027.stdout:Installed: 2023-09-23T14:48:44.822 INFO:teuthology.orchestra.run.smithi027.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:44.822 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:48:44.822 INFO:teuthology.orchestra.run.smithi027.stdout:Complete! 2023-09-23T14:48:44.953 DEBUG:teuthology.parallel:result is None 2023-09-23T14:48:46.255 INFO:teuthology.orchestra.run.smithi017.stdout:Dependencies resolved. 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout: Package Architecture Version Repository Size 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:Installing: 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction Summary 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:================================================================================ 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout:Install 1 Package 2023-09-23T14:48:46.256 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:46.257 INFO:teuthology.orchestra.run.smithi017.stdout:Total download size: 173 k 2023-09-23T14:48:46.257 INFO:teuthology.orchestra.run.smithi017.stdout:Installed size: 552 k 2023-09-23T14:48:46.257 INFO:teuthology.orchestra.run.smithi017.stdout:Downloading Packages: 2023-09-23T14:48:46.445 INFO:teuthology.orchestra.run.smithi017.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 922 kB/s | 173 kB 00:00 2023-09-23T14:48:46.445 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:48:46.445 INFO:teuthology.orchestra.run.smithi017.stdout:Total 916 kB/s | 173 kB 00:00 2023-09-23T14:48:46.446 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction check 2023-09-23T14:48:46.458 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction check succeeded. 2023-09-23T14:48:46.458 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction test 2023-09-23T14:48:46.518 INFO:teuthology.orchestra.run.smithi017.stdout:Transaction test succeeded. 2023-09-23T14:48:46.522 INFO:teuthology.orchestra.run.smithi017.stdout:Running transaction 2023-09-23T14:48:46.645 INFO:teuthology.orchestra.run.smithi017.stdout: Preparing : 1/1 2023-09-23T14:48:46.807 INFO:teuthology.orchestra.run.smithi017.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:47.049 INFO:teuthology.orchestra.run.smithi017.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:48.130 INFO:teuthology.orchestra.run.smithi017.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T14:48:48.130 INFO:teuthology.orchestra.run.smithi017.stdout:Last metadata expiration check: 0:05:57 ago on Sat 23 Sep 2023 02:42:51 PM UTC. 2023-09-23T14:48:58.035 INFO:teuthology.orchestra.run.smithi017.stdout:Installed products updated. 2023-09-23T14:48:58.237 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:58.237 INFO:teuthology.orchestra.run.smithi017.stdout:Installed: 2023-09-23T14:48:58.237 INFO:teuthology.orchestra.run.smithi017.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T14:48:58.237 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:48:58.237 INFO:teuthology.orchestra.run.smithi017.stdout:Complete! 2023-09-23T14:48:58.366 DEBUG:teuthology.parallel:result is None 2023-09-23T14:48:58.366 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:48:58.367 INFO:teuthology.packaging:ref: None 2023-09-23T14:48:58.367 INFO:teuthology.packaging:tag: None 2023-09-23T14:48:58.367 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:48:58.367 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:48:58.367 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T14:48:58.587 DEBUG:teuthology.orchestra.run.smithi017:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T14:48:58.612 INFO:teuthology.orchestra.run.smithi017.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T14:48:58.612 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T14:48:58.612 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T14:48:58.613 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T14:48:58.613 INFO:teuthology.packaging:ref: None 2023-09-23T14:48:58.613 INFO:teuthology.packaging:tag: None 2023-09-23T14:48:58.613 INFO:teuthology.packaging:branch: quincy 2023-09-23T14:48:58.613 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:48:58.614 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T14:48:58.936 DEBUG:teuthology.orchestra.run.smithi027:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T14:48:58.964 INFO:teuthology.orchestra.run.smithi027.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T14:48:58.965 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T14:48:58.965 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T14:48:58.965 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T14:48:58.965 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:48:58.965 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T14:48:58.996 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:48:58.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T14:48:59.038 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T14:48:59.038 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:48:59.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T14:48:59.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T14:48:59.136 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:48:59.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T14:48:59.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T14:48:59.236 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T14:48:59.237 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:48:59.237 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T14:48:59.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T14:48:59.334 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:48:59.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T14:48:59.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T14:48:59.432 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T14:48:59.432 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:48:59.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T14:48:59.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T14:48:59.530 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:48:59.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T14:48:59.559 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T14:48:59.627 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T14:48:59.696 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T14:48:59.697 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T14:48:59.697 INFO:tasks.cephadm:Cluster fsid is 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:48:59.697 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T14:48:59.697 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.17', 'mon.c': '[v2:172.21.15.17:3301,v1:172.21.15.17:6790]', 'mon.b': '172.21.15.27'} 2023-09-23T14:48:59.697 INFO:tasks.cephadm:First mon is mon.a on smithi017 2023-09-23T14:48:59.697 INFO:tasks.cephadm:First mgr is y 2023-09-23T14:48:59.698 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T14:48:59.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo hostname $(hostname -s) 2023-09-23T14:48:59.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo hostname $(hostname -s) 2023-09-23T14:48:59.756 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T14:48:59.756 DEBUG:teuthology.orchestra.run.smithi017:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T14:48:59.902 INFO:teuthology.orchestra.run.smithi017.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 14:48 /home/ubuntu/cephtest/cephadm 2023-09-23T14:48:59.902 DEBUG:teuthology.orchestra.run.smithi027:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T14:49:00.038 INFO:teuthology.orchestra.run.smithi027.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 14:49 /home/ubuntu/cephtest/cephadm 2023-09-23T14:49:00.039 DEBUG:teuthology.orchestra.run.smithi017:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T14:49:00.060 DEBUG:teuthology.orchestra.run.smithi027:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T14:49:00.087 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T14:49:00.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T14:49:00.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T14:49:01.024 INFO:teuthology.orchestra.run.smithi027.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T14:49:01.211 INFO:teuthology.orchestra.run.smithi017.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T14:49:40.406 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T14:49:40.407 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T14:49:40.407 INFO:teuthology.orchestra.run.smithi017.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T14:49:40.407 INFO:teuthology.orchestra.run.smithi017.stdout: "repo_digests": [ 2023-09-23T14:49:40.407 INFO:teuthology.orchestra.run.smithi017.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T14:49:40.407 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T14:49:40.408 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T14:49:53.989 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T14:49:53.989 INFO:teuthology.orchestra.run.smithi027.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T14:49:53.989 INFO:teuthology.orchestra.run.smithi027.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T14:49:53.989 INFO:teuthology.orchestra.run.smithi027.stdout: "repo_digests": [ 2023-09-23T14:49:53.989 INFO:teuthology.orchestra.run.smithi027.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T14:49:53.990 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T14:49:53.990 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T14:49:54.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo mkdir -p /etc/ceph 2023-09-23T14:49:54.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo mkdir -p /etc/ceph 2023-09-23T14:49:54.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 777 /etc/ceph 2023-09-23T14:49:54.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 777 /etc/ceph 2023-09-23T14:49:54.159 INFO:tasks.cephadm:Writing seed config... 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2023-09-23T14:49:54.160 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T14:49:54.161 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T14:49:54.161 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T14:49:54.161 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T14:49:54.161 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T14:49:54.162 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:49:54.162 DEBUG:teuthology.orchestra.run.smithi017:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T14:49:54.187 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 546d6668-5a20-11ee-8db4-212e2dc638e7 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T14:49:54.187 DEBUG:teuthology.orchestra.run.smithi017:mon.a> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T14:49:54.229 DEBUG:teuthology.orchestra.run.smithi017:mgr.y> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T14:49:54.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:49:54.260 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T14:49:54.260 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.17 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T14:49:54.288 INFO:journalctl@ceph.mgr.y.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:49:54.473 INFO:teuthology.orchestra.run.smithi017.stdout:-------------------------------------------------------------------------------- 2023-09-23T14:49:54.473 INFO:teuthology.orchestra.run.smithi017.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '546d6668-5a20-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.17', '--skip-admin-label'] 2023-09-23T14:49:54.541 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 3.0.1 2023-09-23T14:49:54.542 INFO:teuthology.orchestra.run.smithi017.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T14:49:54.542 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying podman|docker is present... 2023-09-23T14:49:54.601 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 3.0.1 2023-09-23T14:49:54.601 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying lvm2 is present... 2023-09-23T14:49:54.601 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying time synchronization is in place... 2023-09-23T14:49:54.608 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T14:49:54.608 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T14:49:54.614 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T14:49:54.614 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:49:54.620 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2023-09-23T14:49:54.626 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2023-09-23T14:49:54.626 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2023-09-23T14:49:54.627 INFO:teuthology.orchestra.run.smithi017.stdout:Repeating the final host check... 2023-09-23T14:49:54.690 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 3.0.1 2023-09-23T14:49:54.690 INFO:teuthology.orchestra.run.smithi017.stdout:podman (/bin/podman) version 3.0.1 is present 2023-09-23T14:49:54.690 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl is present 2023-09-23T14:49:54.690 INFO:teuthology.orchestra.run.smithi017.stdout:lvcreate is present 2023-09-23T14:49:54.696 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T14:49:54.696 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T14:49:54.702 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T14:49:54.702 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:49:54.708 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout enabled 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout active 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:Unit chronyd.service is enabled and running 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:Host looks OK 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:Cluster fsid: 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:Acquiring lock 139865406413288 on /run/cephadm/546d6668-5a20-11ee-8db4-212e2dc638e7.lock 2023-09-23T14:49:54.714 INFO:teuthology.orchestra.run.smithi017.stdout:Lock 139865406413288 acquired on /run/cephadm/546d6668-5a20-11ee-8db4-212e2dc638e7.lock 2023-09-23T14:49:54.715 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 3300 ... 2023-09-23T14:49:54.715 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying IP 172.21.15.17 port 6789 ... 2023-09-23T14:49:54.715 INFO:teuthology.orchestra.run.smithi017.stdout:Base mon IP(s) is [172.21.15.17:3300, 172.21.15.17:6789], mon addrv is [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2023-09-23T14:49:54.718 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T14:49:54.719 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.17 metric 100 2023-09-23T14:49:54.721 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T14:49:54.721 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T14:49:54.721 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1722sec hoplimit 64 pref medium 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe59:d992/64 scope link 2023-09-23T14:49:54.724 INFO:teuthology.orchestra.run.smithi017.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T14:49:54.725 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2023-09-23T14:49:54.725 INFO:teuthology.orchestra.run.smithi017.stdout:Mon IP `172.21.15.17` is in CIDR network `172.21.0.0/20` 2023-09-23T14:49:54.725 INFO:teuthology.orchestra.run.smithi017.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T14:49:54.725 INFO:teuthology.orchestra.run.smithi017.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T14:49:54.726 INFO:teuthology.orchestra.run.smithi017.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T14:49:55.132 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T14:49:55.132 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T14:49:55.132 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T14:49:55.132 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T14:49:55.132 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T14:49:55.133 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T14:49:55.133 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T14:49:55.133 INFO:teuthology.orchestra.run.smithi017.stdout:/bin/podman: stderr Storing signatures 2023-09-23T14:49:56.195 INFO:teuthology.orchestra.run.smithi017.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T14:49:56.196 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T14:49:56.196 INFO:teuthology.orchestra.run.smithi017.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T14:49:57.067 INFO:teuthology.orchestra.run.smithi017.stdout:stat: stdout 167 167 2023-09-23T14:49:57.067 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial keys... 2023-09-23T14:49:57.975 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAV+w5lHlVIGhAA0z6BCYasDhtZlpaGJg1b6Q== 2023-09-23T14:49:58.838 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAW+w5lUz7WEhAAnxGQMTVDoZK0w7V2ppA4MQ== 2023-09-23T14:49:59.721 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph-authtool: stdout AQAX+w5lLydUDRAAy2v73kdeBWj0gD1+Qk28jQ== 2023-09-23T14:49:59.722 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial monmap... 2023-09-23T14:50:00.650 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T14:50:00.650 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T14:50:00.650 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:00.650 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T14:50:00.650 INFO:teuthology.orchestra.run.smithi017.stdout:monmaptool for a [v2:172.21.15.17:3300,v1:172.21.15.17:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T14:50:00.651 INFO:teuthology.orchestra.run.smithi017.stdout:setting min_mon_release = octopus 2023-09-23T14:50:00.651 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: set fsid to 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:00.651 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T14:50:00.651 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:50:00.651 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mon... 2023-09-23T14:50:01.627 INFO:teuthology.orchestra.run.smithi017.stdout:create mon.a on 2023-09-23T14:50:01.731 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T14:50:01.824 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T14:50:01.924 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target. 2023-09-23T14:50:01.925 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target. 2023-09-23T14:50:02.052 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T14:50:02.052 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T14:50:02.143 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target.wants/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@.service. 2023-09-23T14:50:02.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:02 smithi017 systemd[1]: Starting Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:50:02.712 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T14:50:02.712 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon to start... 2023-09-23T14:50:02.720 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mon... 2023-09-23T14:50:02.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:02 smithi017 bash[99171]: 7b4d687c7ef1ed19a0329d905e9a42b101d1af9db3501c0cb153a60560a8e125 2023-09-23T14:50:02.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:02 smithi017 systemd[1]: Started Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:50:02.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:02 smithi017 ceph-mon[99237]: mkfs 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:02.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:02 smithi017 ceph-mon[99237]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T14:50:04.074 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T14:50:04.074 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout id: 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:04.074 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T14:50:04.074 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout services: 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.624865s) 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout data: 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T14:50:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T14:50:04.076 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:04.076 INFO:teuthology.orchestra.run.smithi017.stdout:mon is available 2023-09-23T14:50:04.076 INFO:teuthology.orchestra.run.smithi017.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: fsmap 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: mgrmap e1: no daemons active 2023-09-23T14:50:04.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:03 smithi017 ceph-mon[99237]: from='client.? 172.21.15.17:0/2344629250' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T14:50:05.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:04 smithi017 ceph-mon[99237]: from='client.? 172.21.15.17:0/2961190617' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T14:50:05.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:04 smithi017 ceph-mon[99237]: from='client.? 172.21.15.17:0/2961190617' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.17:3300,v1:172.21.15.17:6789] 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T14:50:05.224 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T14:50:05.225 INFO:teuthology.orchestra.run.smithi017.stdout:Generating new minimal ceph.conf... 2023-09-23T14:50:06.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:05 smithi017 ceph-mon[99237]: from='client.? 172.21.15.17:0/3014161906' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:50:06.348 INFO:teuthology.orchestra.run.smithi017.stdout:Restarting the monitor... 2023-09-23T14:50:06.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:06 smithi017 systemd[1]: Stopping Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:50:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:06 smithi017 conmon[99213]: 2023-09-23T14:50:06.628+0000 7f10ff561700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T14:50:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:06 smithi017 conmon[99213]: 2023-09-23T14:50:06.628+0000 7f10ff561700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T14:50:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 bash[99795]: 7b4d687c7ef1ed19a0329d905e9a42b101d1af9db3501c0cb153a60560a8e125 2023-09-23T14:50:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T14:50:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 systemd[1]: Stopped Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:50:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 systemd[1]: Starting Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:50:07.851 INFO:teuthology.orchestra.run.smithi017.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T14:50:08.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: pidfile_write: ignore empty --pid-file 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: load: jerasure load: lrc 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: RocksDB version: 6.15.5 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Compile date Sep 22 2023 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: DB SUMMARY 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: DB Session ID: J5SOFDSSHG7WVM5G7OTL 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: CURRENT file: CURRENT 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T14:50:08.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83589 ; 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.error_if_exists: 0 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.create_if_missing: 0 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.paranoid_checks: 1 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.env: 0x55c3bb955a80 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.fs: Posix File System 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.info_log: 0x55c3bdf745a0 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T14:50:08.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.statistics: (nil) 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.use_fsync: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_log_file_size: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_fallocate: 1 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.use_direct_reads: 0 2023-09-23T14:50:08.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.db_log_dir: 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T14:50:08.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.write_buffer_manager: 0x55c3bece6960 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T14:50:08.100 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.unordered_write: 0 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 bash[100124]: 54ba4e8b4801d1a4f4db77268365ee916418be42aad1a8bcb363e076147935eb 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.row_cache: None 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.wal_filter: None 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.preserve_deletes: 0 2023-09-23T14:50:08.101 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.two_write_queues: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.atomic_flush: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.log_readahead_size: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T14:50:08.102 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_background_jobs: 2 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_background_compactions: -1 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_subcompactions: 1 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T14:50:08.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_open_files: -1 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_background_flushes: -1 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Compression algorithms supported: 2023-09-23T14:50:08.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kZSTD supported: 0 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kXpressCompression supported: 0 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kZlibCompression supported: 1 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kSnappyCompression supported: 1 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kLZ4Compression supported: 1 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: kBZip2Compression supported: 0 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T14:50:08.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.merge_operator: 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_filter: None 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c3bdfc6fb8) 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2023-09-23T14:50:08.107 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: hash_index_allow_collision: 1 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 1 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x55c3bdff9090 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2023-09-23T14:50:08.108 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T14:50:08.109 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 4 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression: NoCompression 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T14:50:08.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.num_levels: 7 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T14:50:08.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T14:50:08.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T14:50:08.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T14:50:08.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.table_properties_collectors: 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.inplace_update_support: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.bloom_locality: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.max_successive_merges: 0 2023-09-23T14:50:08.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.ttl: 2592000 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.enable_blob_files: false 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.min_blob_size: 0 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T14:50:08.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480607844746, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480607845562, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80136, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78404, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72612, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695480607, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "242a97b8-636c-478c-a7eb-30aa3083b328", "db_session_id": "J5SOFDSSHG7WVM5G7OTL"}} 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480607845910, "job": 1, "event": "recovery_finished"} 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c3bdf8d180 2023-09-23T14:50:08.117 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: DB pointer 0x55c3be062000 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T14:50:08.118 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.119 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 24.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 24.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.120 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 2/0 80.02 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 80.02 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 141.3 0.00 0.00 1 0.001 0 0 2023-09-23T14:50:08.121 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 24.35 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:08.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???) e1 preinit fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).mds e1 new map 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).mds e1 print_map 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: e1 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: -1 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout: No filesystems configured 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:50:08.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 systemd[1]: Started Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: monmap e1: 1 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0]} removed_ranks: {} 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: fsmap 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T14:50:08.124 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:07 smithi017 ceph-mon[100189]: mgrmap e1: no daemons active 2023-09-23T14:50:09.022 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T14:50:09.022 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T14:50:09.022 INFO:teuthology.orchestra.run.smithi017.stdout:Creating mgr... 2023-09-23T14:50:09.023 INFO:teuthology.orchestra.run.smithi017.stdout:Verifying port 9283 ... 2023-09-23T14:50:09.153 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T14:50:09.154 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Failed to reset failed state of unit ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T14:50:09.245 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7.target.wants/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@.service. 2023-09-23T14:50:09.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:09 smithi017 systemd[1]: Starting Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:50:09.852 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:09 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1125848888' entity='client.admin' 2023-09-23T14:50:09.871 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T14:50:09.871 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2023-09-23T14:50:09.879 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T14:50:09.879 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:50:09.879 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2023-09-23T14:50:09.879 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T14:50:09.888 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T14:50:09.888 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr to start... 2023-09-23T14:50:09.896 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr... 2023-09-23T14:50:10.105 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:09 smithi017 bash[100578]: 2de58bf12b45e54b232225eef2ec58c3b832da780aaacab04f0e0448c4ec8dc3 2023-09-23T14:50:10.105 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:09 smithi017 systemd[1]: Started Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:50:10.105 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:09 smithi017 conmon[100620]: 2023-09-23T14:50:09.997+0000 7f37a96491c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:50:10.105 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:10 smithi017 conmon[100620]: 2023-09-23T14:50:10.091+0000 7f37a96491c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:10.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:10 smithi017 conmon[100620]: 2023-09-23T14:50:10.201+0000 7f37a96491c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:50:10.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:10 smithi017 conmon[100620]: 2023-09-23T14:50:10.313+0000 7f37a96491c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:50:10.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:10 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/46517371' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T14:50:10.924 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:10 smithi017 conmon[100620]: 2023-09-23T14:50:10.516+0000 7f37a96491c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:50:11.166 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:11.166 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "546d6668-5a20-11ee-8db4-212e2dc638e7", 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T14:50:11.167 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.168 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T14:50:11.169 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:11.170 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:11.171 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T14:50:02.736362+0000", 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:11.172 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (1/15)... 2023-09-23T14:50:11.179 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:10 smithi017 conmon[100620]: 2023-09-23T14:50:10.923+0000 7f37a96491c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:50:11.179 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:11 smithi017 conmon[100620]: 2023-09-23T14:50:11.028+0000 7f37a96491c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:50:11.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:11 smithi017 conmon[100620]: 2023-09-23T14:50:11.178+0000 7f37a96491c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:50:11.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:11 smithi017 conmon[100620]: 2023-09-23T14:50:11.291+0000 7f37a96491c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:50:12.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:11 smithi017 conmon[100620]: 2023-09-23T14:50:11.795+0000 7f37a96491c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:50:12.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:12 smithi017 conmon[100620]: 2023-09-23T14:50:12.481+0000 7f37a96491c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:50:12.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:12 smithi017 conmon[100620]: 2023-09-23T14:50:12.670+0000 7f37a96491c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:50:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3332208408' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T14:50:13.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:13 smithi017 conmon[100620]: 2023-09-23T14:50:13.642+0000 7f37a96491c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:13.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:13 smithi017 conmon[100620]: 2023-09-23T14:50:13.751+0000 7f37a96491c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:50:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "546d6668-5a20-11ee-8db4-212e2dc638e7", 2023-09-23T14:50:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T14:50:14.299 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T14:50:14.300 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T14:50:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T14:50:14.302 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T14:50:14.306 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.306 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T14:50:14.306 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T14:50:14.306 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.307 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T14:50:14.308 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T14:50:02.736362+0000", 2023-09-23T14:50:14.309 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:14.310 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:14.310 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T14:50:14.310 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:14.310 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (2/15)... 2023-09-23T14:50:14.310 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:13 smithi017 conmon[100620]: 2023-09-23T14:50:13.974+0000 7f37a96491c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:14.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:14 smithi017 conmon[100620]: 2023-09-23T14:50:14.317+0000 7f37a96491c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:50:14.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:14 smithi017 conmon[100620]: 2023-09-23T14:50:14.420+0000 7f37a96491c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:50:15.066 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:14 smithi017 conmon[100620]: 2023-09-23T14:50:14.748+0000 7f37a96491c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:50:15.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:15 smithi017 conmon[100620]: 2023-09-23T14:50:15.065+0000 7f37a96491c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:50:16.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:15 smithi017 conmon[100620]: 2023-09-23T14:50:15.885+0000 7f37a96491c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:50:16.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:16 smithi017 conmon[100620]: 2023-09-23T14:50:16.040+0000 7f37a96491c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:50:16.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:16 smithi017 conmon[100620]: 2023-09-23T14:50:16.276+0000 7f37a96491c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:50:16.533 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:16 smithi017 conmon[100620]: 2023-09-23T14:50:16.419+0000 7f37a96491c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:16.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:16 smithi017 conmon[100620]: 2023-09-23T14:50:16.689+0000 7f37a96491c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:50:17.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:17 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1630779133' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T14:50:17.487 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "546d6668-5a20-11ee-8db4-212e2dc638e7", 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T14:50:17.490 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T14:50:17.491 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T14:50:17.492 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T14:50:17.492 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T14:50:17.492 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T14:50:17.492 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T14:50:17.495 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:17.496 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T14:50:02.736362+0000", 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:17.497 INFO:teuthology.orchestra.run.smithi017.stdout:mgr not available, waiting (3/15)... 2023-09-23T14:50:17.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:17 smithi017 conmon[100620]: 2023-09-23T14:50:17.736+0000 7f37a96491c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:50:18.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: Activating manager daemon y 2023-09-23T14:50:18.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: mgrmap e2: y(active, starting, since 0.00289445s) 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: Manager daemon y is now available 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:50:18.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' 2023-09-23T14:50:18.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' 2023-09-23T14:50:18.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:18 smithi017 ceph-mon[100189]: from='mgr.14100 172.21.15.17:0/366112454' entity='mgr.y' 2023-09-23T14:50:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:19 smithi017 ceph-mon[100189]: mgrmap e3: y(active, since 1.00652s) 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsid": "546d6668-5a20-11ee-8db4-212e2dc638e7", 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T14:50:21.015 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 0 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "quorum_age": 12, 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T14:50:21.020 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T14:50:21.020 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T14:50:21.020 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T14:50:21.020 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T14:50:21.020 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T14:50:21.021 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ], 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T14:50:02.736362+0000", 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T14:50:21.022 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout }, 2023-09-23T14:50:21.023 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T14:50:21.023 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:21.023 INFO:teuthology.orchestra.run.smithi017.stdout:mgr is available 2023-09-23T14:50:21.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:20 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4124087557' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T14:50:22.002 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:21 smithi017 ceph-mon[100189]: mgrmap e4: y(active, since 3s) 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout fsid = 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T14:50:22.347 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T14:50:22.348 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling cephadm module... 2023-09-23T14:50:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:22 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2371468673' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T14:50:24.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:23 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2342980726' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T14:50:24.141 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:23 smithi017 conmon[100620]: ignoring --setuser ceph since I am not root 2023-09-23T14:50:24.142 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:23 smithi017 conmon[100620]: ignoring --setgroup ceph since I am not root 2023-09-23T14:50:24.142 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.045+0000 7f27050e21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:50:24.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.140+0000 7f27050e21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:24.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.246+0000 7f27050e21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:50:24.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.361+0000 7f27050e21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:50:24.856 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.576+0000 7f27050e21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:50:25.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:24 smithi017 conmon[100620]: 2023-09-23T14:50:24.977+0000 7f27050e21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:50:25.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:25 smithi017 conmon[100620]: 2023-09-23T14:50:25.083+0000 7f27050e21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:50:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:24 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2342980726' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T14:50:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:24 smithi017 ceph-mon[100189]: mgrmap e5: y(active, since 6s) 2023-09-23T14:50:25.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:25 smithi017 conmon[100620]: 2023-09-23T14:50:25.232+0000 7f27050e21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:50:25.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:25 smithi017 conmon[100620]: 2023-09-23T14:50:25.349+0000 7f27050e21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2023-09-23T14:50:25.712 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 5... 2023-09-23T14:50:26.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:25 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1519956760' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T14:50:26.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:25 smithi017 conmon[100620]: 2023-09-23T14:50:25.866+0000 7f27050e21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:50:26.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:26 smithi017 conmon[100620]: 2023-09-23T14:50:26.563+0000 7f27050e21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:50:26.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:26 smithi017 conmon[100620]: 2023-09-23T14:50:26.752+0000 7f27050e21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:50:28.040 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:27 smithi017 conmon[100620]: 2023-09-23T14:50:27.718+0000 7f27050e21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:28.040 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:27 smithi017 conmon[100620]: 2023-09-23T14:50:27.826+0000 7f27050e21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:50:28.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:28 smithi017 conmon[100620]: 2023-09-23T14:50:28.039+0000 7f27050e21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:28.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:28 smithi017 conmon[100620]: 2023-09-23T14:50:28.380+0000 7f27050e21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:50:28.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:28 smithi017 conmon[100620]: 2023-09-23T14:50:28.483+0000 7f27050e21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:50:29.131 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:28 smithi017 conmon[100620]: 2023-09-23T14:50:28.812+0000 7f27050e21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:50:29.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:29 smithi017 conmon[100620]: 2023-09-23T14:50:29.130+0000 7f27050e21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:50:30.346 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:29 smithi017 conmon[100620]: 2023-09-23T14:50:29.949+0000 7f27050e21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:50:30.346 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:30 smithi017 conmon[100620]: 2023-09-23T14:50:30.107+0000 7f27050e21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:50:30.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:30 smithi017 conmon[100620]: 2023-09-23T14:50:30.345+0000 7f27050e21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:50:30.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:30 smithi017 conmon[100620]: 2023-09-23T14:50:30.481+0000 7f27050e21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:31.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:30 smithi017 conmon[100620]: 2023-09-23T14:50:30.751+0000 7f27050e21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:50:32.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:31 smithi017 conmon[100620]: 2023-09-23T14:50:31.813+0000 7f27050e21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: Active manager daemon y restarted 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: Activating manager daemon y 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: mgrmap e6: y(active, starting, since 0.00342406s) 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: Manager daemon y is now available 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:32.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:32.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:31 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:33.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: Found migration_current of "None". Setting to last migration. 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:32 smithi017 ceph-mon[100189]: mgrmap e7: y(active, since 1.00602s) 2023-09-23T14:50:33.377 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:33.378 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T14:50:33.378 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T14:50:33.378 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:33.378 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 5 is available 2023-09-23T14:50:33.378 INFO:teuthology.orchestra.run.smithi017.stdout:Setting orchestrator backend to cephadm... 2023-09-23T14:50:34.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:33 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:32] ENGINE Bus STARTING 2023-09-23T14:50:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:33 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:32] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:50:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:33 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:32] ENGINE Bus STARTED 2023-09-23T14:50:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:33 smithi017 ceph-mon[100189]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T14:50:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:33 smithi017 ceph-mon[100189]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T14:50:35.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:34 smithi017 ceph-mon[100189]: mgrmap e8: y(active, since 2s) 2023-09-23T14:50:35.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:34 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:35.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:34 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T14:50:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:Generating ssh key... 2023-09-23T14:50:36.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:35 smithi017 ceph-mon[100189]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:36 smithi017 ceph-mon[100189]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: Generating public/private rsa key pair. 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: Your identification has been saved in /tmp/tmp7vw4rmgz/key. 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: Your public key has been saved in /tmp/tmp7vw4rmgz/key.pub. 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: The key fingerprint is: 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: SHA256:mY2jvHCgKHtMsHb4tgHkodP6lD8jhi4FnSndUHeEHt4 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: The key's randomart image is: 2023-09-23T14:50:37.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: +---[RSA 3072]----+ 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: | .. .oo | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: | . .o. | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: | = = o o | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |B.* . o E= | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |oBo . S . | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |.*++ o . . | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |==*.. + | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |++=++o . | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: |++oooo. | 2023-09-23T14:50:37.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:37 smithi017 conmon[100620]: +----[SHA256]-----+ 2023-09-23T14:50:38.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:38 smithi017 ceph-mon[100189]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:38.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:38 smithi017 ceph-mon[100189]: Generating ssh key... 2023-09-23T14:50:38.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:38 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:38.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:38 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:39.135 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqcCG6HzCfARLFWREpPH3NCsiBPQnnXAme13twmfxwINCMZNkGVxrHJuaFPsAS+OAZXX0wvzNcTVDldvoejygtR6I4jh3CyAyAMgkbnRkWMhvzXgxuUdOrOWKJDf0BGcNVuuddjl9sV8yZ/72uBZwWIFxz4mA8YxuOASoivw/kswEgKPEpNHWaG9xH7sysEapqbk/5ufmfCgCDQig5+go1drzo/4T9UWWzoVgayOFRUZ5q1cwsLNNqkMaU7vW8kX38JwnQqB3dIlzsayRImD+LjyYKtaA69FR9fgcgP/xKkdyl3H7nBvByAlbodxLy1LO/Q9jtGCpwzQk2ztRadNSj+yz1ImXTa/Bg5iaPkXpsi+T2svfWaS+S52gpaU6Q5nj7Uj8HQ1MgRsqZ1toKAIsKP05SCeJgpi1pFCImjzwXxBN85eM79oYjbv5W+U7NbMXQqqPeiVfxMKtUQ5bFV7OMKkFT/X7fiGwLq1kRIss+2tgnUkXJVKNoIsDaIHbdgKk= ceph-546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:50:39.135 INFO:teuthology.orchestra.run.smithi017.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T14:50:39.135 INFO:teuthology.orchestra.run.smithi017.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T14:50:39.137 INFO:teuthology.orchestra.run.smithi017.stdout:Adding host smithi017... 2023-09-23T14:50:39.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:39 smithi017 ceph-mon[100189]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:41.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:40 smithi017 ceph-mon[100189]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi017", "addr": "172.21.15.17", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:41.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:40 smithi017 ceph-mon[100189]: Deploying cephadm binary to smithi017 2023-09-23T14:50:42.518 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Added host 'smithi017' with addr '172.21.15.17' 2023-09-23T14:50:42.518 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mon service... 2023-09-23T14:50:43.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:42 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:43.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:42 smithi017 ceph-mon[100189]: Added host smithi017 2023-09-23T14:50:43.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:42 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:44.304 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T14:50:44.305 INFO:teuthology.orchestra.run.smithi017.stdout:Deploying unmanaged mgr service... 2023-09-23T14:50:44.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:44 smithi017 ceph-mon[100189]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:44.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:44 smithi017 ceph-mon[100189]: Saving service mon spec with placement count:5 2023-09-23T14:50:44.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:44 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:44.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:44 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:45.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:45 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:45.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:45 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:46.103 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T14:50:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:46 smithi017 ceph-mon[100189]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:50:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:46 smithi017 ceph-mon[100189]: Saving service mgr spec with placement count:2 2023-09-23T14:50:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:46 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:46 smithi017 ceph-mon[100189]: from='mgr.14120 172.21.15.17:0/3053934850' entity='mgr.y' 2023-09-23T14:50:48.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:47 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1315297056' entity='client.admin' 2023-09-23T14:50:49.433 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling the dashboard module... 2023-09-23T14:50:50.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:49 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1946849766' entity='client.admin' 2023-09-23T14:50:51.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:50 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1733103963' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T14:50:51.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:50 smithi017 conmon[100620]: ignoring --setuser ceph since I am not root 2023-09-23T14:50:51.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:50 smithi017 conmon[100620]: ignoring --setgroup ceph since I am not root 2023-09-23T14:50:51.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:51 smithi017 conmon[100620]: 2023-09-23T14:50:51.138+0000 7fed60ffc1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:50:51.616 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:51 smithi017 conmon[100620]: 2023-09-23T14:50:51.234+0000 7fed60ffc1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:51.616 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:51 smithi017 conmon[100620]: 2023-09-23T14:50:51.340+0000 7fed60ffc1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:50:51.617 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:51 smithi017 conmon[100620]: 2023-09-23T14:50:51.454+0000 7fed60ffc1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:50:51.932 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:51 smithi017 conmon[100620]: 2023-09-23T14:50:51.673+0000 7fed60ffc1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:50:52.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:51 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1733103963' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T14:50:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:51 smithi017 ceph-mon[100189]: mgrmap e9: y(active, since 19s) 2023-09-23T14:50:52.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:52 smithi017 conmon[100620]: 2023-09-23T14:50:52.075+0000 7fed60ffc1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:50:52.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:52 smithi017 conmon[100620]: 2023-09-23T14:50:52.181+0000 7fed60ffc1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:50:52.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:52 smithi017 conmon[100620]: 2023-09-23T14:50:52.330+0000 7fed60ffc1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:50:52.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:52 smithi017 conmon[100620]: 2023-09-23T14:50:52.444+0000 7fed60ffc1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:50:52.876 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:50:52.876 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T14:50:52.876 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T14:50:52.876 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T14:50:52.876 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T14:50:52.877 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:50:52.877 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for the mgr to restart... 2023-09-23T14:50:52.877 INFO:teuthology.orchestra.run.smithi017.stdout:Waiting for mgr epoch 9... 2023-09-23T14:50:53.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:52 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2058378300' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T14:50:53.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:52 smithi017 conmon[100620]: 2023-09-23T14:50:52.974+0000 7fed60ffc1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:50:53.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:53 smithi017 conmon[100620]: 2023-09-23T14:50:53.663+0000 7fed60ffc1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:50:53.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:53 smithi017 conmon[100620]: 2023-09-23T14:50:53.853+0000 7fed60ffc1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:50:55.150 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:54 smithi017 conmon[100620]: 2023-09-23T14:50:54.824+0000 7fed60ffc1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:55.150 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:54 smithi017 conmon[100620]: 2023-09-23T14:50:54.932+0000 7fed60ffc1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:50:55.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:55 smithi017 conmon[100620]: 2023-09-23T14:50:55.149+0000 7fed60ffc1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:50:55.926 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:55 smithi017 conmon[100620]: 2023-09-23T14:50:55.493+0000 7fed60ffc1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:50:55.926 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:55 smithi017 conmon[100620]: 2023-09-23T14:50:55.598+0000 7fed60ffc1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:50:56.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:55 smithi017 conmon[100620]: 2023-09-23T14:50:55.925+0000 7fed60ffc1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:50:56.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:56 smithi017 conmon[100620]: 2023-09-23T14:50:56.245+0000 7fed60ffc1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:50:57.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:57 smithi017 conmon[100620]: 2023-09-23T14:50:57.065+0000 7fed60ffc1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:50:57.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:57 smithi017 conmon[100620]: 2023-09-23T14:50:57.221+0000 7fed60ffc1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:50:57.861 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:57 smithi017 conmon[100620]: 2023-09-23T14:50:57.458+0000 7fed60ffc1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:50:57.861 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:57 smithi017 conmon[100620]: 2023-09-23T14:50:57.593+0000 7fed60ffc1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:50:58.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:57 smithi017 conmon[100620]: 2023-09-23T14:50:57.860+0000 7fed60ffc1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:50:59.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:50:58 smithi017 conmon[100620]: 2023-09-23T14:50:58.886+0000 7fed60ffc1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:50:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: Active manager daemon y restarted 2023-09-23T14:50:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: Activating manager daemon y 2023-09-23T14:50:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: mgrmap e10: y(active, starting, since 0.00293387s) 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: Manager daemon y is now available 2023-09-23T14:50:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:50:59.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:58 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:00.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:51:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:51:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:50:59 smithi017 ceph-mon[100189]: mgrmap e11: y(active, since 1.00587s) 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout { 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout } 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:mgr epoch 9 is available 2023-09-23T14:51:00.527 INFO:teuthology.orchestra.run.smithi017.stdout:Generating a dashboard self-signed certificate... 2023-09-23T14:51:01.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:59] ENGINE Bus STARTING 2023-09-23T14:51:01.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:59] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:51:01.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: [23/Sep/2023:14:50:59] ENGINE Bus STARTED 2023-09-23T14:51:01.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T14:51:01.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T14:51:01.382 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:01.382 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:00 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:02.445 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T14:51:02.445 INFO:teuthology.orchestra.run.smithi017.stdout:Creating initial admin user... 2023-09-23T14:51:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:02 smithi017 ceph-mon[100189]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:02 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:02 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:02 smithi017 ceph-mon[100189]: mgrmap e12: y(active, since 2s) 2023-09-23T14:51:04.207 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$jkErrWezP4mUEqq9jcE3PeSleqKB8PgOXPO.E.T9PDyOC6axu13sq", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695480663, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T14:51:04.207 INFO:teuthology.orchestra.run.smithi017.stdout:Fetching dashboard port number... 2023-09-23T14:51:04.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:04 smithi017 ceph-mon[100189]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:04.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:04 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:05.607 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T14:51:05.617 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T14:51:05.618 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout disabled 2023-09-23T14:51:05.626 INFO:teuthology.orchestra.run.smithi017.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T14:51:05.626 INFO:teuthology.orchestra.run.smithi017.stdout:systemctl: stdout inactive 2023-09-23T14:51:05.627 INFO:teuthology.orchestra.run.smithi017.stdout:firewalld.service is not enabled 2023-09-23T14:51:05.627 INFO:teuthology.orchestra.run.smithi017.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T14:51:05.628 INFO:teuthology.orchestra.run.smithi017.stdout:Ceph Dashboard is now available at: 2023-09-23T14:51:05.628 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:05.628 INFO:teuthology.orchestra.run.smithi017.stdout: URL: https://smithi017:8443/ 2023-09-23T14:51:05.629 INFO:teuthology.orchestra.run.smithi017.stdout: User: admin 2023-09-23T14:51:05.629 INFO:teuthology.orchestra.run.smithi017.stdout: Password: s6q5y4j9p4 2023-09-23T14:51:05.629 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:05.629 INFO:teuthology.orchestra.run.smithi017.stdout:Saving cluster configuration to /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config directory 2023-09-23T14:51:05.630 INFO:teuthology.orchestra.run.smithi017.stdout:Enabling autotune for osd_memory_target 2023-09-23T14:51:05.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:05 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3369865388' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T14:51:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:06 smithi017 ceph-mon[100189]: mgrmap e13: y(active, since 6s) 2023-09-23T14:51:08.649 INFO:teuthology.orchestra.run.smithi017.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T14:51:08.649 INFO:teuthology.orchestra.run.smithi017.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T14:51:08.649 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.649 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:08.649 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T14:51:08.650 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout: ceph telemetry on 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout:For more information see: 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:08.651 INFO:teuthology.orchestra.run.smithi017.stdout:Bootstrap complete. 2023-09-23T14:51:08.702 INFO:tasks.cephadm:Fetching config... 2023-09-23T14:51:08.702 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:08.702 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T14:51:08.721 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T14:51:08.721 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:08.721 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T14:51:08.780 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T14:51:08.781 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:08.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T14:51:08.850 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T14:51:08.850 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:08.850 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T14:51:08.907 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T14:51:08.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqcCG6HzCfARLFWREpPH3NCsiBPQnnXAme13twmfxwINCMZNkGVxrHJuaFPsAS+OAZXX0wvzNcTVDldvoejygtR6I4jh3CyAyAMgkbnRkWMhvzXgxuUdOrOWKJDf0BGcNVuuddjl9sV8yZ/72uBZwWIFxz4mA8YxuOASoivw/kswEgKPEpNHWaG9xH7sysEapqbk/5ufmfCgCDQig5+go1drzo/4T9UWWzoVgayOFRUZ5q1cwsLNNqkMaU7vW8kX38JwnQqB3dIlzsayRImD+LjyYKtaA69FR9fgcgP/xKkdyl3H7nBvByAlbodxLy1LO/Q9jtGCpwzQk2ztRadNSj+yz1ImXTa/Bg5iaPkXpsi+T2svfWaS+S52gpaU6Q5nj7Uj8HQ1MgRsqZ1toKAIsKP05SCeJgpi1pFCImjzwXxBN85eM79oYjbv5W+U7NbMXQqqPeiVfxMKtUQ5bFV7OMKkFT/X7fiGwLq1kRIss+2tgnUkXJVKNoIsDaIHbdgKk= ceph-546d6668-5a20-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T14:51:08.993 INFO:teuthology.orchestra.run.smithi017.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqcCG6HzCfARLFWREpPH3NCsiBPQnnXAme13twmfxwINCMZNkGVxrHJuaFPsAS+OAZXX0wvzNcTVDldvoejygtR6I4jh3CyAyAMgkbnRkWMhvzXgxuUdOrOWKJDf0BGcNVuuddjl9sV8yZ/72uBZwWIFxz4mA8YxuOASoivw/kswEgKPEpNHWaG9xH7sysEapqbk/5ufmfCgCDQig5+go1drzo/4T9UWWzoVgayOFRUZ5q1cwsLNNqkMaU7vW8kX38JwnQqB3dIlzsayRImD+LjyYKtaA69FR9fgcgP/xKkdyl3H7nBvByAlbodxLy1LO/Q9jtGCpwzQk2ztRadNSj+yz1ImXTa/Bg5iaPkXpsi+T2svfWaS+S52gpaU6Q5nj7Uj8HQ1MgRsqZ1toKAIsKP05SCeJgpi1pFCImjzwXxBN85eM79oYjbv5W+U7NbMXQqqPeiVfxMKtUQ5bFV7OMKkFT/X7fiGwLq1kRIss+2tgnUkXJVKNoIsDaIHbdgKk= ceph-546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:09.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqcCG6HzCfARLFWREpPH3NCsiBPQnnXAme13twmfxwINCMZNkGVxrHJuaFPsAS+OAZXX0wvzNcTVDldvoejygtR6I4jh3CyAyAMgkbnRkWMhvzXgxuUdOrOWKJDf0BGcNVuuddjl9sV8yZ/72uBZwWIFxz4mA8YxuOASoivw/kswEgKPEpNHWaG9xH7sysEapqbk/5ufmfCgCDQig5+go1drzo/4T9UWWzoVgayOFRUZ5q1cwsLNNqkMaU7vW8kX38JwnQqB3dIlzsayRImD+LjyYKtaA69FR9fgcgP/xKkdyl3H7nBvByAlbodxLy1LO/Q9jtGCpwzQk2ztRadNSj+yz1ImXTa/Bg5iaPkXpsi+T2svfWaS+S52gpaU6Q5nj7Uj8HQ1MgRsqZ1toKAIsKP05SCeJgpi1pFCImjzwXxBN85eM79oYjbv5W+U7NbMXQqqPeiVfxMKtUQ5bFV7OMKkFT/X7fiGwLq1kRIss+2tgnUkXJVKNoIsDaIHbdgKk= ceph-546d6668-5a20-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T14:51:09.068 INFO:teuthology.orchestra.run.smithi027.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDqcCG6HzCfARLFWREpPH3NCsiBPQnnXAme13twmfxwINCMZNkGVxrHJuaFPsAS+OAZXX0wvzNcTVDldvoejygtR6I4jh3CyAyAMgkbnRkWMhvzXgxuUdOrOWKJDf0BGcNVuuddjl9sV8yZ/72uBZwWIFxz4mA8YxuOASoivw/kswEgKPEpNHWaG9xH7sysEapqbk/5ufmfCgCDQig5+go1drzo/4T9UWWzoVgayOFRUZ5q1cwsLNNqkMaU7vW8kX38JwnQqB3dIlzsayRImD+LjyYKtaA69FR9fgcgP/xKkdyl3H7nBvByAlbodxLy1LO/Q9jtGCpwzQk2ztRadNSj+yz1ImXTa/Bg5iaPkXpsi+T2svfWaS+S52gpaU6Q5nj7Uj8HQ1MgRsqZ1toKAIsKP05SCeJgpi1pFCImjzwXxBN85eM79oYjbv5W+U7NbMXQqqPeiVfxMKtUQ5bFV7OMKkFT/X7fiGwLq1kRIss+2tgnUkXJVKNoIsDaIHbdgKk= ceph-546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:09.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T14:51:09.429 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:09.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:09 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/783879313' entity='client.admin' 2023-09-23T14:51:11.996 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T14:51:11.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T14:51:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:12 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3875885222' entity='client.admin' 2023-09-23T14:51:12.890 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:14.130 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:14.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:14.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi017", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:51:14.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:14.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:14.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:15.223 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi027 2023-09-23T14:51:15.223 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:51:15.223 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/etc/ceph/ceph.conf 2023-09-23T14:51:15.245 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:51:15.245 DEBUG:teuthology.orchestra.run.smithi027:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:15.305 INFO:tasks.cephadm:Adding host smithi027 to orchestrator... 2023-09-23T14:51:15.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi027 2023-09-23T14:51:15.683 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:15.886 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:15.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:16 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:16 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:51:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:18 smithi017 ceph-mon[100189]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi027", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:18 smithi017 ceph-mon[100189]: Deploying cephadm binary to smithi027 2023-09-23T14:51:19.421 INFO:teuthology.orchestra.run.smithi017.stdout:Added host 'smithi027' with addr '172.21.15.27' 2023-09-23T14:51:19.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T14:51:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:19 smithi017 ceph-mon[100189]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:19 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:19 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:19 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:20.315 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:21.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:20 smithi017 ceph-mon[100189]: Added host smithi027 2023-09-23T14:51:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:21.935 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:51:21.936 INFO:teuthology.orchestra.run.smithi017.stdout:[{"addr": "172.21.15.17", "hostname": "smithi017", "labels": [], "status": ""}, {"addr": "172.21.15.27", "hostname": "smithi027", "labels": [], "status": ""}] 2023-09-23T14:51:22.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:21 smithi017 ceph-mon[100189]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:21 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:22.478 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T14:51:22.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T14:51:22.824 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:22 smithi017 ceph-mon[100189]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:51:24.008 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:23 smithi017 ceph-mon[100189]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:24.917 INFO:teuthology.orchestra.run.smithi017.stderr:adjusted tunables profile to default 2023-09-23T14:51:25.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/595463522' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:25.444 INFO:tasks.cephadm:Adding mon.a on smithi017 2023-09-23T14:51:25.444 INFO:tasks.cephadm:Adding mon.c on smithi017 2023-09-23T14:51:25.444 INFO:tasks.cephadm:Adding mon.b on smithi027 2023-09-23T14:51:25.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi027:172.21.15.27=b' 2023-09-23T14:51:25.825 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/595463522' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:26.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:26.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:26.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:27.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:26 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:51:27.483 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled mon update... 2023-09-23T14:51:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:28.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:28.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:28.220 DEBUG:teuthology.orchestra.run.smithi017:mon.c> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T14:51:28.223 DEBUG:teuthology.orchestra.run.smithi027:mon.b> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T14:51:28.227 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T14:51:28.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T14:51:28.257 INFO:journalctl@ceph.mon.b.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:51:28.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:51:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:28 smithi017 ceph-mon[100189]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi027:172.21.15.27=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:28 smithi017 ceph-mon[100189]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi027:172.21.15.27=b;count:3 2023-09-23T14:51:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:28 smithi017 ceph-mon[100189]: Deploying daemon mon.b on smithi027 2023-09-23T14:51:29.239 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:30.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:29 smithi017 ceph-mon[100189]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:31.572 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:51:31.572 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":1,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","modified":"2023-09-23T14:50:00.112882Z","created":"2023-09-23T14:50:00.112882Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T14:51:31.575 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 1 2023-09-23T14:51:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:31 smithi027 ceph-mon[105164]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T14:51:33.333 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T14:51:33.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T14:51:33.691 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: mon.a calling monitor election 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: mon.b calling monitor election 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} removed_ranks: {} 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: fsmap 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: mgrmap e13: y(active, since 37s) 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: overall HEALTH_OK 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:37.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:37.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:37.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: mon.a calling monitor election 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: mon.b calling monitor election 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} removed_ranks: {} 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: fsmap 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: mgrmap e13: y(active, since 37s) 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: overall HEALTH_OK 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:37.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:37.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:37 smithi017 ceph-mon[100189]: Deploying daemon mon.c on smithi017 2023-09-23T14:51:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:37 smithi017 ceph-mon[100189]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:37 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:37 smithi027 ceph-mon[105164]: Deploying daemon mon.c on smithi017 2023-09-23T14:51:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:37 smithi027 ceph-mon[105164]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:37 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:38.291 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:51:38.291 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":2,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","modified":"2023-09-23T14:51:31.591673Z","created":"2023-09-23T14:50:00.112882Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T14:51:38.293 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 2 2023-09-23T14:51:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:38 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/3387182942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:39.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:38 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/3387182942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:39.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 systemd[1]: Starting Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:51:39.866 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T14:51:39.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pidfile_write: ignore empty --pid-file 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 bash[118063]: 4d08e21f44ed510bf43c3f09b79d99b9a67912c5a83ae9fa30b0d7f9f3c0da23 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: load: jerasure load: lrc 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: RocksDB version: 6.15.5 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T14:51:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Compile date Sep 22 2023 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: DB SUMMARY 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: DB Session ID: Q6Y8W3NF2TBIG6MYMY83 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: CURRENT file: CURRENT 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.error_if_exists: 0 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.create_if_missing: 0 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.paranoid_checks: 1 2023-09-23T14:51:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.env: 0x563936379a80 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.fs: Posix File System 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.info_log: 0x56393823c5a0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.statistics: (nil) 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.use_fsync: 0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_log_file_size: 0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_fallocate: 1 2023-09-23T14:51:40.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.use_direct_reads: 0 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.db_log_dir: 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T14:51:40.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T14:51:40.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T14:51:40.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.write_buffer_manager: 0x563938fae930 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T14:51:40.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.unordered_write: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.row_cache: None 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.wal_filter: None 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T14:51:40.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.preserve_deletes: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.two_write_queues: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.atomic_flush: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.log_readahead_size: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T14:51:40.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_background_jobs: 2 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_background_compactions: -1 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_subcompactions: 1 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_open_files: -1 2023-09-23T14:51:40.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_background_flushes: -1 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Compression algorithms supported: 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kZSTD supported: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kXpressCompression supported: 0 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kZlibCompression supported: 1 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kSnappyCompression supported: 1 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kLZ4Compression supported: 1 2023-09-23T14:51:40.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: kBZip2Compression supported: 0 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.merge_operator: 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_filter: None 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56393828efb8) 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2023-09-23T14:51:40.203 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: hash_index_allow_collision: 1 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 1 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x5639382c1090 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2023-09-23T14:51:40.204 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2023-09-23T14:51:40.205 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 4 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T14:51:40.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression: NoCompression 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.num_levels: 7 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T14:51:40.207 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T14:51:40.208 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T14:51:40.209 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T14:51:40.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.table_properties_collectors: 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.inplace_update_support: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.bloom_locality: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.max_successive_merges: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T14:51:40.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.ttl: 2592000 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.enable_blob_files: false 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.min_blob_size: 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T14:51:40.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480699812106, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480699812377, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695480699, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "fceac21a-a61e-4cf1-b12d-eae3a55e2da8", "db_session_id": "Q6Y8W3NF2TBIG6MYMY83"}} 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695480699812718, "job": 1, "event": "recovery_finished"} 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563938255180 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: DB pointer 0x56393832a000 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:51:40.213 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.214 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T14:51:40.215 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.63 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.216 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.0 0.00 0.00 1 0.000 0 0 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T14:51:40.217 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: using public_addrv [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: starting mon.c rank -1 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(???) e0 preinit fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 systemd[1]: Started Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: e1 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: -1 2023-09-23T14:51:40.218 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout: No filesystems configured 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/595463522' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi027", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:40.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/595463522' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi027:172.21.15.27=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Saving service mon spec with placement smithi017:172.21.15.17=a;smithi017:[v2:172.21.15.17:3301,v1:172.21.15.17:6790]=c;smithi027:172.21.15.27=b;count:3 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Deploying daemon mon.b on smithi027 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.a calling monitor election 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.b calling monitor election 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: monmap e2: 2 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0]} removed_ranks: {} 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: fsmap 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mgrmap e13: y(active, since 37s) 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: overall HEALTH_OK 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:40.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: Deploying daemon mon.c on smithi017 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/3387182942' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T14:51:40.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:39 smithi017 ceph-mon[118128]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T14:51:40.230 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: mon.a calling monitor election 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: mon.b calling monitor election 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: mon.c calling monitor election 2023-09-23T14:51:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: fsmap 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: mgrmap e13: y(active, since 45s) 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: overall HEALTH_OK 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: mon.a calling monitor election 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: mon.b calling monitor election 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: mon.c calling monitor election 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: fsmap 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: mgrmap e13: y(active, since 45s) 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: overall HEALTH_OK 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:51:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: mon.a calling monitor election 2023-09-23T14:51:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: mon.b calling monitor election 2023-09-23T14:51:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:51:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: mon.c calling monitor election 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: fsmap 2023-09-23T14:51:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: mgrmap e13: y(active, since 45s) 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: overall HEALTH_OK 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: 2023-09-23T14:51:45.492 INFO:teuthology.orchestra.run.smithi027.stdout:{"epoch":3,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","modified":"2023-09-23T14:51:39.837627Z","created":"2023-09-23T14:50:00.112882Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3300","nonce":0},{"type":"v1","addr":"172.21.15.17:6789","nonce":0}]},"addr":"172.21.15.17:6789/0","public_addr":"172.21.15.17:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:3300","nonce":0},{"type":"v1","addr":"172.21.15.27:6789","nonce":0}]},"addr":"172.21.15.27:6789/0","public_addr":"172.21.15.27:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:3301","nonce":0},{"type":"v1","addr":"172.21.15.17:6790","nonce":0}]},"addr":"172.21.15.17:6790/0","public_addr":"172.21.15.17:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-09-23T14:51:45.495 INFO:teuthology.orchestra.run.smithi027.stderr:dumped monmap epoch 3 2023-09-23T14:51:46.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[100189]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/1283999365' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[118128]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/1283999365' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:45 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:46.339 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:45 smithi027 ceph-mon[105164]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:46.339 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:45 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.339 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:45 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/1283999365' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T14:51:46.339 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:45 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:51:46.357 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T14:51:46.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T14:51:46.736 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[100189]: Reconfiguring daemon mon.a on smithi017 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:47 smithi017 ceph-mon[118128]: Reconfiguring daemon mon.a on smithi017 2023-09-23T14:51:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:47 smithi027 ceph-mon[105164]: Reconfiguring daemon mon.a on smithi017 2023-09-23T14:51:48.866 INFO:teuthology.orchestra.run.smithi017.stdout:# minimal ceph.conf for 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:48.866 INFO:teuthology.orchestra.run.smithi017.stdout:[global] 2023-09-23T14:51:48.867 INFO:teuthology.orchestra.run.smithi017.stdout: fsid = 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T14:51:48.867 INFO:teuthology.orchestra.run.smithi017.stdout: mon_host = [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] 2023-09-23T14:51:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4289038360' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:49.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4289038360' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4289038360' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:49.386 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T14:51:49.386 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:49.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T14:51:49.419 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:49.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:49.487 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:51:49.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T14:51:49.525 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:51:49.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:51:49.598 INFO:tasks.cephadm:Adding mgr.y on smithi017 2023-09-23T14:51:49.599 INFO:tasks.cephadm:Adding mgr.x on smithi027 2023-09-23T14:51:49.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi017=y;smithi027=x' 2023-09-23T14:51:49.986 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:51:50.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:50.189 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.189 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.189 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: Reconfiguring mon.c (monmap changed)... 2023-09-23T14:51:50.189 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:50.189 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:50.190 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.190 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:50.190 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:49 smithi027 ceph-mon[105164]: Reconfiguring daemon mon.c on smithi017 2023-09-23T14:51:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: Reconfiguring mon.c (monmap changed)... 2023-09-23T14:51:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:50.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:50.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:50.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[118128]: Reconfiguring daemon mon.c on smithi017 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: Reconfiguring mon.c (monmap changed)... 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:50.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:50.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:50.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:50.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:49 smithi017 ceph-mon[100189]: Reconfiguring daemon mon.c on smithi017 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: Reconfiguring mon.b (monmap changed)... 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: Reconfiguring daemon mon.b on smithi027 2023-09-23T14:51:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:51 smithi027 ceph-mon[105164]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:51.663 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled mgr update... 2023-09-23T14:51:51.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: Reconfiguring mon.b (monmap changed)... 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: Reconfiguring daemon mon.b on smithi027 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[100189]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: Reconfiguring mon.b (monmap changed)... 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: Reconfiguring daemon mon.b on smithi027 2023-09-23T14:51:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:51 smithi017 ceph-mon[118128]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:52.500 DEBUG:teuthology.orchestra.run.smithi027:mgr.x> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T14:51:52.502 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T14:51:52.502 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:51:52.502 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T14:51:52.523 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T14:51:52.524 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_1 2023-09-23T14:51:52.530 INFO:journalctl@ceph.mgr.x.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:51:52.584 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T14:51:52.584 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 6h/6d Inode: 387592 Links: 1 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2023-09-23 14:51:39.202031736 +0000 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2023-09-23 14:40:54.891856933 +0000 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2023-09-23 14:40:54.891856933 +0000 2023-09-23T14:51:52.585 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: - 2023-09-23T14:51:52.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T14:51:52.657 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2023-09-23T14:51:52.657 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2023-09-23T14:51:52.657 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000131844 s, 3.9 MB/s 2023-09-23T14:51:52.658 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T14:51:52.719 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_2 2023-09-23T14:51:52.777 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T14:51:52.777 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 6h/6d Inode: 383569 Links: 1 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2023-09-23 14:51:39.202031736 +0000 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2023-09-23 14:40:55.424851945 +0000 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2023-09-23 14:40:55.424851945 +0000 2023-09-23T14:51:52.778 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: - 2023-09-23T14:51:52.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T14:51:52.844 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2023-09-23T14:51:52.845 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2023-09-23T14:51:52.845 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.00010534 s, 4.9 MB/s 2023-09-23T14:51:52.846 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T14:51:52.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi027=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: Saving service mgr spec with placement smithi017=y;smithi027=x;count:2 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:52.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T14:51:52.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:52.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.905 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_3 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi027=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: Saving service mgr spec with placement smithi017=y;smithi027=x;count:2 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi017=y;smithi027=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: Saving service mgr spec with placement smithi017=y;smithi027=x;count:2 2023-09-23T14:51:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:52.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 6h/6d Inode: 390914 Links: 1 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2023-09-23 14:51:39.202031736 +0000 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2023-09-23 14:40:55.954846985 +0000 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2023-09-23 14:40:55.954846985 +0000 2023-09-23T14:51:52.963 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: - 2023-09-23T14:51:52.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T14:51:53.030 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2023-09-23T14:51:53.030 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2023-09-23T14:51:53.030 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 0.000120393 s, 4.3 MB/s 2023-09-23T14:51:53.031 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T14:51:53.090 DEBUG:teuthology.orchestra.run.smithi017:> stat /dev/vg_nvme/lv_4 2023-09-23T14:51:53.148 INFO:teuthology.orchestra.run.smithi017.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T14:51:53.148 INFO:teuthology.orchestra.run.smithi017.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:53.148 INFO:teuthology.orchestra.run.smithi017.stdout:Device: 6h/6d Inode: 383645 Links: 1 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout:Access: 2023-09-23 14:51:39.202031736 +0000 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout:Modify: 2023-09-23 14:40:56.648840491 +0000 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout:Change: 2023-09-23 14:40:56.648840491 +0000 2023-09-23T14:51:53.149 INFO:teuthology.orchestra.run.smithi017.stdout: Birth: - 2023-09-23T14:51:53.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T14:51:53.214 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records in 2023-09-23T14:51:53.214 INFO:teuthology.orchestra.run.smithi017.stderr:1+0 records out 2023-09-23T14:51:53.215 INFO:teuthology.orchestra.run.smithi017.stderr:512 bytes copied, 9.8987e-05 s, 5.2 MB/s 2023-09-23T14:51:53.216 DEBUG:teuthology.orchestra.run.smithi017:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T14:51:53.274 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:51:53.274 DEBUG:teuthology.orchestra.run.smithi027:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T14:51:53.292 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T14:51:53.292 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_1 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 6h/6d Inode: 390218 Links: 1 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2023-09-23 14:51:30.861063802 +0000 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2023-09-23 14:40:54.895062715 +0000 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2023-09-23 14:40:54.895062715 +0000 2023-09-23T14:51:53.349 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: - 2023-09-23T14:51:53.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T14:51:53.415 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2023-09-23T14:51:53.416 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2023-09-23T14:51:53.416 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000240368 s, 2.1 MB/s 2023-09-23T14:51:53.417 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T14:51:53.474 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_2 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 6h/6d Inode: 390289 Links: 1 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2023-09-23 14:51:30.861063802 +0000 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2023-09-23 14:40:55.423049335 +0000 2023-09-23T14:51:53.531 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2023-09-23 14:40:55.423049335 +0000 2023-09-23T14:51:53.532 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: - 2023-09-23T14:51:53.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T14:51:53.597 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2023-09-23T14:51:53.597 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2023-09-23T14:51:53.597 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000216808 s, 2.4 MB/s 2023-09-23T14:51:53.598 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T14:51:53.657 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_3 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 6h/6d Inode: 387021 Links: 1 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2023-09-23 14:51:53.697482489 +0000 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2023-09-23 14:40:55.939036259 +0000 2023-09-23T14:51:53.715 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2023-09-23 14:40:55.939036259 +0000 2023-09-23T14:51:53.716 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: - 2023-09-23T14:51:53.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T14:51:53.784 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2023-09-23T14:51:53.784 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2023-09-23T14:51:53.784 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000226924 s, 2.3 MB/s 2023-09-23T14:51:53.786 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T14:51:53.844 DEBUG:teuthology.orchestra.run.smithi027:> stat /dev/vg_nvme/lv_4 2023-09-23T14:51:53.902 INFO:teuthology.orchestra.run.smithi027.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T14:51:53.902 INFO:teuthology.orchestra.run.smithi027.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Device: 6h/6d Inode: 391737 Links: 1 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Access: 2023-09-23 14:51:53.834479001 +0000 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Modify: 2023-09-23 14:40:56.467022879 +0000 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout:Change: 2023-09-23 14:40:56.467022879 +0000 2023-09-23T14:51:53.903 INFO:teuthology.orchestra.run.smithi027.stdout: Birth: - 2023-09-23T14:51:53.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T14:51:53.971 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records in 2023-09-23T14:51:53.972 INFO:teuthology.orchestra.run.smithi027.stderr:1+0 records out 2023-09-23T14:51:53.972 INFO:teuthology.orchestra.run.smithi027.stderr:512 bytes copied, 0.000196499 s, 2.6 MB/s 2023-09-23T14:51:53.973 DEBUG:teuthology.orchestra.run.smithi027:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T14:51:54.034 INFO:tasks.cephadm:Deploying osd.0 on smithi017 with /dev/vg_nvme/lv_4... 2023-09-23T14:51:54.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T14:51:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:53 smithi017 ceph-mon[100189]: Deploying daemon mgr.x on smithi027 2023-09-23T14:51:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:53 smithi017 ceph-mon[100189]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:53 smithi017 ceph-mon[118128]: Deploying daemon mgr.x on smithi027 2023-09-23T14:51:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:53 smithi017 ceph-mon[118128]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:54.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:53 smithi027 ceph-mon[105164]: Deploying daemon mgr.x on smithi027 2023-09-23T14:51:54.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:53 smithi027 ceph-mon[105164]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:54.330 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:53 smithi027 systemd[1]: Starting Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:51:54.393 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 bash[108342]: 02c28602755772df4af84b754166f3d6a03a32882a5c09b0081e4a62939f8df3 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 ceph-mgr[108407]: -- 172.21.15.27:0/3527498816 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562b2ff609c0 con 0x562b2ff38800 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 systemd[1]: Started Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 conmon[108383]: 2023-09-23T14:51:54.587+0000 7f5f731741c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 conmon[108383]: 2023-09-23T14:51:54.682+0000 7f5f731741c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:51:54.795 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 conmon[108383]: 2023-09-23T14:51:54.794+0000 7f5f731741c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:51:55.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:54 smithi027 conmon[108383]: 2023-09-23T14:51:54.905+0000 7f5f731741c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:51:55.510 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:55 smithi027 conmon[108383]: 2023-09-23T14:51:55.137+0000 7f5f731741c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:55.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[100189]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:55.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:55 smithi017 ceph-mon[118128]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:55 smithi027 conmon[108383]: 2023-09-23T14:51:55.555+0000 7f5f731741c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:55 smithi027 conmon[108383]: 2023-09-23T14:51:55.675+0000 7f5f731741c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:55.826 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:55 smithi027 ceph-mon[105164]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:56.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:55 smithi027 conmon[108383]: 2023-09-23T14:51:55.825+0000 7f5f731741c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:51:56.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:55 smithi027 conmon[108383]: 2023-09-23T14:51:55.940+0000 7f5f731741c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10+0 records in 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr:10+0 records out 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.00825314 s, 1.3 GB/s 2023-09-23T14:51:56.195 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping successful for: 2023-09-23T14:51:56.741 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:56 smithi027 conmon[108383]: 2023-09-23T14:51:56.454+0000 7f5f731741c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:51:56.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi017:vg_nvme/lv_4 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[100189]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:56 smithi017 ceph-mon[118128]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T14:51:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T14:51:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:51:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:51:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:56 smithi027 ceph-mon[105164]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T14:51:57.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:57 smithi027 conmon[108383]: 2023-09-23T14:51:57.141+0000 7f5f731741c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:51:57.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:57 smithi027 conmon[108383]: 2023-09-23T14:51:57.330+0000 7f5f731741c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:51:57.768 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:51:57.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:57.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:57 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:51:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:57 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:51:58.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:58 smithi027 conmon[108383]: 2023-09-23T14:51:58.283+0000 7f5f731741c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:51:58.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:58 smithi027 conmon[108383]: 2023-09-23T14:51:58.391+0000 7f5f731741c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:51:58.944 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:58 smithi027 conmon[108383]: 2023-09-23T14:51:58.603+0000 7f5f731741c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:51:59.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:58 smithi027 conmon[108383]: 2023-09-23T14:51:58.943+0000 7f5f731741c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:51:59.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:59 smithi027 conmon[108383]: 2023-09-23T14:51:59.046+0000 7f5f731741c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:51:59.692 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:59 smithi027 conmon[108383]: 2023-09-23T14:51:59.374+0000 7f5f731741c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:51:59.968 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:51:59 smithi027 conmon[108383]: 2023-09-23T14:51:59.691+0000 7f5f731741c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:52:00.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[100189]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:00.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:00.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:00.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[118128]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:00.234 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:51:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:59 smithi027 ceph-mon[105164]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:51:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:00.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:00 smithi027 conmon[108383]: 2023-09-23T14:52:00.518+0000 7f5f731741c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:52:00.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:00 smithi027 conmon[108383]: 2023-09-23T14:52:00.677+0000 7f5f731741c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:52:01.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:00 smithi017 ceph-mon[100189]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:01.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:00 smithi017 ceph-mon[118128]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:01.321 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:00 smithi027 ceph-mon[105164]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:01.322 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:00 smithi027 conmon[108383]: 2023-09-23T14:52:00.916+0000 7f5f731741c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:52:01.322 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:01 smithi027 conmon[108383]: 2023-09-23T14:52:01.052+0000 7f5f731741c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:52:02.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:01 smithi027 conmon[108383]: 2023-09-23T14:52:01.320+0000 7f5f731741c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2000494613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]': finished 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T14:52:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2000494613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]': finished 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T14:52:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2000494613' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]: dispatch 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0ab537e0-788a-404a-aee3-04d5c85f3ea8"}]': finished 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T14:52:02.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:02.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:52:02 smithi027 conmon[108383]: 2023-09-23T14:52:02.377+0000 7f5f731741c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:52:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: Standby manager daemon x started 2023-09-23T14:52:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:52:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:52:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:52:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:52:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:02 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3963265376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:03.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: Standby manager daemon x started 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3963265376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: Standby manager daemon x started 2023-09-23T14:52:03.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:52:03.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:52:03.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:52:03.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/2302528538' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:52:03.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:02 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3963265376' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:03 smithi027 ceph-mon[105164]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:03 smithi027 ceph-mon[105164]: mgrmap e14: y(active, since 64s), standbys: x 2023-09-23T14:52:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:03 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[100189]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[100189]: mgrmap e14: y(active, since 64s), standbys: x 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[118128]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[118128]: mgrmap e14: y(active, since 64s), standbys: x 2023-09-23T14:52:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:03 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:52:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:05 smithi027 ceph-mon[105164]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:06.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:05 smithi017 ceph-mon[100189]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:06.398 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:05 smithi017 ceph-mon[118128]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:07 smithi027 ceph-mon[105164]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:07 smithi017 ceph-mon[100189]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:07 smithi017 ceph-mon[118128]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:09.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T14:52:09.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:09.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:08 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T14:52:09.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:08 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:09.410 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:08 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T14:52:09.410 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:08 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:09 smithi027 ceph-mon[105164]: Deploying daemon osd.0 on smithi017 2023-09-23T14:52:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:09 smithi027 ceph-mon[105164]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:09 smithi017 ceph-mon[100189]: Deploying daemon osd.0 on smithi017 2023-09-23T14:52:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:09 smithi017 ceph-mon[100189]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:09 smithi017 ceph-mon[118128]: Deploying daemon osd.0 on smithi017 2023-09-23T14:52:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:09 smithi017 ceph-mon[118128]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:11 smithi017 ceph-mon[100189]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:12.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:11 smithi017 ceph-mon[118128]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:12.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:11 smithi027 ceph-mon[105164]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:14.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[100189]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:14.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[118128]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:13 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:14.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:13 smithi027 ceph-mon[105164]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:14.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:13 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:13 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:14.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:13 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:16.104 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 0 on host 'smithi017' 2023-09-23T14:52:16.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:15 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:16.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:15 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.711 DEBUG:teuthology.orchestra.run.smithi017:osd.0> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T14:52:16.713 INFO:tasks.cephadm:Deploying osd.1 on smithi017 with /dev/vg_nvme/lv_3... 2023-09-23T14:52:16.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T14:52:16.742 INFO:journalctl@ceph.osd.0.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:52:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[100189]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[100189]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[118128]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[118128]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:16 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:17.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:16 smithi027 ceph-mon[105164]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:16 smithi027 ceph-mon[105164]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T14:52:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:16 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:16 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:16 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:17.767 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:52:18.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[100189]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T14:52:18.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T14:52:18.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:18.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.085 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:17 smithi017 ceph-mon[118128]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:18.085 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:52:17 smithi017 conmon[124373]: 2023-09-23T14:52:17.919+0000 7fa0d2e4f700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T14:52:18.085 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:52:17 smithi017 conmon[124373]: 2023-09-23T14:52:17.921+0000 7fa0cbc46700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:52:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T14:52:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T14:52:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: from='osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:17 smithi027 ceph-mon[105164]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[100189]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[100189]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[118128]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:19.181 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[118128]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T14:52:19.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.182 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:18 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:18 smithi027 ceph-mon[105164]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:18 smithi027 ceph-mon[105164]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T14:52:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:18 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:18 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10+0 records in 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr:10+0 records out 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00796818 s, 1.3 GB/s 2023-09-23T14:52:19.745 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping successful for: 2023-09-23T14:52:19.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:52:19.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:52:19.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:19.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106] boot 2023-09-23T14:52:19.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T14:52:19.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:52:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:52:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106] boot 2023-09-23T14:52:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T14:52:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:19 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:52:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:52:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T14:52:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: osd.0 [v2:172.21.15.17:6802/2828543106,v1:172.21.15.17:6803/2828543106] boot 2023-09-23T14:52:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T14:52:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:19 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:52:20.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi017:vg_nvme/lv_3 2023-09-23T14:52:20.887 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:52:21.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:21.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:21.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:21.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:21.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:22.055 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi017 2023-09-23T14:52:22.056 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi017 to 16149M 2023-09-23T14:52:22.056 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[100189]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:22.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi017 2023-09-23T14:52:22.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi017 to 16149M 2023-09-23T14:52:22.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:21 smithi017 ceph-mon[118128]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:21 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi017 2023-09-23T14:52:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:21 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi017 to 16149M 2023-09-23T14:52:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:21 smithi027 ceph-mon[105164]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:23.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:22 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:22 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:22 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:22 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:24.150 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:23 smithi017 ceph-mon[100189]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:24.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:23 smithi017 ceph-mon[100189]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:24.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:23 smithi017 ceph-mon[118128]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:24.151 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:23 smithi017 ceph-mon[118128]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:23 smithi027 ceph-mon[105164]: from='client.24140 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:23 smithi027 ceph-mon[105164]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/345276394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]': finished 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[100189]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/345276394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]': finished 2023-09-23T14:52:25.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[118128]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T14:52:25.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:24 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:25.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:24 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/345276394' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:24 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]: dispatch 2023-09-23T14:52:25.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:24 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d26e6ec-8564-4b09-a9e8-f7fe77e36a65"}]': finished 2023-09-23T14:52:25.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:24 smithi027 ceph-mon[105164]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T14:52:25.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:24 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:26.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:25 smithi017 ceph-mon[100189]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:25 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1797890324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:25 smithi017 ceph-mon[118128]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:25 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1797890324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:26.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:25 smithi027 ceph-mon[105164]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:26.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:25 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1797890324' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:27 smithi017 ceph-mon[100189]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:28.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:27 smithi017 ceph-mon[118128]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:27 smithi027 ceph-mon[105164]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:30.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:29 smithi017 ceph-mon[100189]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:30.113 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:29 smithi017 ceph-mon[118128]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:29 smithi027 ceph-mon[105164]: pgmap v45: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:32.028 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[118128]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:32.028 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T14:52:32.028 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:32.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[100189]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:32.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T14:52:32.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:31 smithi027 ceph-mon[105164]: pgmap v46: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T14:52:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:33.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:32 smithi017 ceph-mon[100189]: Deploying daemon osd.1 on smithi017 2023-09-23T14:52:33.070 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:32 smithi017 ceph-mon[118128]: Deploying daemon osd.1 on smithi017 2023-09-23T14:52:33.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:32 smithi027 ceph-mon[105164]: Deploying daemon osd.1 on smithi017 2023-09-23T14:52:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:33 smithi017 ceph-mon[100189]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:33 smithi017 ceph-mon[118128]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:33 smithi027 ceph-mon[105164]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:36.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:35 smithi017 ceph-mon[100189]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:35 smithi017 ceph-mon[118128]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:36.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:35 smithi027 ceph-mon[105164]: pgmap v48: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:37.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:37 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:37 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:37 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:37.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:37 smithi027 ceph-mon[105164]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:37.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[100189]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:37.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:37 smithi017 ceph-mon[118128]: pgmap v49: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:39.266 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 1 on host 'smithi017' 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[100189]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:39 smithi017 ceph-mon[118128]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:39 smithi027 ceph-mon[105164]: pgmap v50: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:39.936 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T14:52:39.938 INFO:tasks.cephadm:Deploying osd.2 on smithi017 with /dev/vg_nvme/lv_2... 2023-09-23T14:52:39.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T14:52:39.971 INFO:journalctl@ceph.osd.1.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[100189]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:40.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:40 smithi017 ceph-mon[118128]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T14:52:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:40 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:40 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:40 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:52:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:40 smithi027 ceph-mon[105164]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T14:52:40.978 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:52:41.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[100189]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T14:52:41.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[100189]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T14:52:41.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:41.362 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[100189]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[100189]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[118128]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[118128]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[118128]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:41.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:41 smithi017 ceph-mon[118128]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:41.363 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:52:41 smithi017 conmon[134049]: 2023-09-23T14:52:41.233+0000 7f72da0ed700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T14:52:41.364 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:52:41 smithi017 conmon[134049]: 2023-09-23T14:52:41.237+0000 7f72d56e9700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:52:41.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:41 smithi027 ceph-mon[105164]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T14:52:41.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:41 smithi027 ceph-mon[105164]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T14:52:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:41 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:41 smithi027 ceph-mon[105164]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:52:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:41 smithi027 ceph-mon[105164]: pgmap v52: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[100189]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[100189]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[118128]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[118128]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T14:52:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:42 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:42 smithi027 ceph-mon[105164]: from='osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:52:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:42 smithi027 ceph-mon[105164]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T14:52:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:42 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:42 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10+0 records in 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr:10+0 records out 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.008093 s, 1.3 GB/s 2023-09-23T14:52:42.991 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping successful for: 2023-09-23T14:52:43.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:52:43.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:52:43.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339] boot 2023-09-23T14:52:43.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T14:52:43.410 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:43.410 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[118128]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:43.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:52:43.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:52:43.410 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339] boot 2023-09-23T14:52:43.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T14:52:43.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:43.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:43 smithi017 ceph-mon[100189]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:52:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:52:43.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: osd.1 [v2:172.21.15.17:6810/1437136339,v1:172.21.15.17:6811/1437136339] boot 2023-09-23T14:52:43.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T14:52:43.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:52:43.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:43 smithi027 ceph-mon[105164]: pgmap v55: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T14:52:43.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi017:vg_nvme/lv_2 2023-09-23T14:52:43.971 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi017 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi017 to 8074M 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi017 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi017 to 8074M 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi017 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi017 to 8074M 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:44.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:52:44.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:52:45.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:45 smithi027 ceph-mon[105164]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:45.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:45 smithi017 ceph-mon[100189]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:45 smithi017 ceph-mon[118128]: pgmap v57: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[100189]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[118128]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:46.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:46 smithi027 ceph-mon[105164]: from='client.24167 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:52:46.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:52:46.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:52:46.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:47 smithi027 ceph-mon[105164]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:47.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:47 smithi017 ceph-mon[100189]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:47.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:47 smithi017 ceph-mon[118128]: pgmap v58: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]: dispatch 2023-09-23T14:52:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]': finished 2023-09-23T14:52:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[100189]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T14:52:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:52:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3321988715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:49.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]: dispatch 2023-09-23T14:52:49.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]': finished 2023-09-23T14:52:49.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[118128]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T14:52:49.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:52:49.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3321988715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]: dispatch 2023-09-23T14:52:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2186733530' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "509065ed-f964-449d-8ed3-bb7c0b529dd8"}]': finished 2023-09-23T14:52:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:48 smithi027 ceph-mon[105164]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T14:52:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:48 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:52:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3321988715' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:52:50.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:49 smithi017 ceph-mon[100189]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:49 smithi017 ceph-mon[118128]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:50.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:49 smithi027 ceph-mon[105164]: pgmap v60: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:52.275 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:51 smithi017 ceph-mon[100189]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:52.276 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:51 smithi017 ceph-mon[118128]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:51 smithi027 ceph-mon[105164]: pgmap v61: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:53 smithi017 ceph-mon[100189]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:53 smithi017 ceph-mon[118128]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:53 smithi027 ceph-mon[105164]: pgmap v62: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:54 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T14:52:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:54 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:54 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T14:52:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:54 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:54 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T14:52:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:54 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:52:55.993 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:55 smithi017 ceph-mon[100189]: Deploying daemon osd.2 on smithi017 2023-09-23T14:52:55.993 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:55 smithi017 ceph-mon[100189]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:55.993 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:55 smithi017 ceph-mon[118128]: Deploying daemon osd.2 on smithi017 2023-09-23T14:52:55.993 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:55 smithi017 ceph-mon[118128]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:55 smithi027 ceph-mon[105164]: Deploying daemon osd.2 on smithi017 2023-09-23T14:52:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:55 smithi027 ceph-mon[105164]: pgmap v63: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:57 smithi027 ceph-mon[105164]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:58.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:57 smithi017 ceph-mon[100189]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:52:58.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:57 smithi017 ceph-mon[118128]: pgmap v64: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:00.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[100189]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[118128]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:52:59 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:59 smithi027 ceph-mon[105164]: pgmap v65: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:52:59 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:02.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:02.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: pgmap v66: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:02.366 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 2 on host 'smithi017' 2023-09-23T14:53:02.953 DEBUG:teuthology.orchestra.run.smithi017:osd.2> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T14:53:02.956 INFO:tasks.cephadm:Deploying osd.3 on smithi017 with /dev/vg_nvme/lv_1... 2023-09-23T14:53:02.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T14:53:02.990 INFO:journalctl@ceph.osd.2.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:53:03.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:03.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[100189]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T14:53:03.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:03.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:02 smithi017 ceph-mon[118128]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T14:53:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:02 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:02 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:02 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:02 smithi027 ceph-mon[105164]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T14:53:03.619 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:53:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[100189]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[100189]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T14:53:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[100189]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T14:53:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[100189]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:04.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[118128]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:04.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[118128]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T14:53:04.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[118128]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T14:53:04.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:04.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:03 smithi017 ceph-mon[118128]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:03 smithi027 ceph-mon[105164]: pgmap v67: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:03 smithi027 ceph-mon[105164]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T14:53:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:03 smithi027 ceph-mon[105164]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T14:53:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:03 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:03 smithi027 ceph-mon[105164]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:04.591 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:04 smithi017 conmon[142966]: 2023-09-23T14:53:04.336+0000 7fda53d3b700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T14:53:04.591 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:04 smithi017 conmon[142966]: 2023-09-23T14:53:04.340+0000 7fda4fb38700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:53:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:05 smithi027 ceph-mon[105164]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:05 smithi027 ceph-mon[105164]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T14:53:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:05 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:05 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:05 smithi027 ceph-mon[105164]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[100189]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[100189]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T14:53:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[100189]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[118128]: from='osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[118128]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T14:53:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:05 smithi017 ceph-mon[118128]: pgmap v70: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T14:53:05.838 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T14:53:05.838 INFO:teuthology.orchestra.run.smithi017.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T14:53:05.838 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10+0 records in 2023-09-23T14:53:05.838 INFO:teuthology.orchestra.run.smithi017.stderr:10+0 records out 2023-09-23T14:53:05.838 INFO:teuthology.orchestra.run.smithi017.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0133016 s, 788 MB/s 2023-09-23T14:53:05.839 INFO:teuthology.orchestra.run.smithi017.stderr:--> Zapping successful for: 2023-09-23T14:53:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:06 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:53:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:06 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:53:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:06 smithi027 ceph-mon[105164]: osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690] boot 2023-09-23T14:53:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:06 smithi027 ceph-mon[105164]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T14:53:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:06 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:53:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:53:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[100189]: osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690] boot 2023-09-23T14:53:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[100189]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T14:53:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:53:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:53:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[118128]: osd.2 [v2:172.21.15.17:6818/542107690,v1:172.21.15.17:6819/542107690] boot 2023-09-23T14:53:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[118128]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T14:53:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:06 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:53:06.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi017:vg_nvme/lv_1 2023-09-23T14:53:07.148 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T14:53:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:07.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:07.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:07.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi017 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi017 to 5383M 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T14:53:08.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi017 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi017 to 5383M 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T14:53:08.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T14:53:08.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:08 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:53:08.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: pgmap v73: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi017 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi017 to 5383M 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T14:53:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:53:09.781 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[149964]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:09.782 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[149964]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:09.782 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[149964]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:09.782 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[149964]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:09.782 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[100189]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T14:53:09.783 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:53:09.784 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:09 smithi017 ceph-mon[118128]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: from='client.24194 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi017:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: pgmap v75: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:53:09.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:09 smithi027 ceph-mon[105164]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T14:53:10.192 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[150020]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:10.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[150020]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:10.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[150020]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:10.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 14:53:09 smithi017 sudo[150020]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:10.592 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150066]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:10.593 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150066]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:10.593 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150066]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:10.593 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150066]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[100189]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T14:53:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[100189]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T14:53:11.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[118128]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T14:53:11.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:11.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:10 smithi017 ceph-mon[118128]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T14:53:11.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150114]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:11.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150114]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:11.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150114]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:11.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 14:53:10 smithi017 sudo[150114]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:11.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:10 smithi027 ceph-mon[105164]: osdmap e22: 3 total, 3 up, 3 in 2023-09-23T14:53:11.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:10 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:11.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:10 smithi027 ceph-mon[105164]: mgrmap e15: y(active, since 2m), standbys: x 2023-09-23T14:53:11.692 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150149]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:11.692 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150149]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:11.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150149]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:11.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150149]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:12.192 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150187]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:12.193 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150187]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:12.193 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150187]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:12.193 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 14:53:11 smithi017 sudo[150187]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:12.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150191]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:12.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150191]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:12.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150191]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:12.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150191]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]: dispatch 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]': finished 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150194]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150194]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150194]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 sudo[150194]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]: dispatch 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]': finished 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:12 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: pgmap v78: 1 pgs: 1 creating+peering; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]: dispatch 2023-09-23T14:53:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/796119895' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "1430d849-5f40-43a2-9ebd-c82ddaaa262b"}]': finished 2023-09-23T14:53:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T14:53:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:12 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:13.594 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109283]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:13.594 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109283]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:13.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109283]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:13.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109283]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109286]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109286]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109286]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:13 smithi027 sudo[109286]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:14.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[150810]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T14:53:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[150810]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[150810]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[150810]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/368299137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/368299137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[151006]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[151006]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[151006]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 sudo[151006]: pam_unix(sudo:session): session closed for user root 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/368299137' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: pgmap v80: 1 pgs: 1 creating+peering; 0 B data, 16 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:53:14.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:53:15.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:15 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:15 smithi027 ceph-mon[105164]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:15 smithi027 ceph-mon[105164]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:15.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:15.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[100189]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:15.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[100189]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T14:53:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[118128]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T14:53:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:15 smithi017 ceph-mon[118128]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:17 smithi027 ceph-mon[105164]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:18.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:17 smithi017 ceph-mon[100189]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:18.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:17 smithi017 ceph-mon[118128]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:18 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T14:53:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:18 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:19.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:18 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T14:53:19.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:18 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:19.390 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:18 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T14:53:19.390 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:18 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:19 smithi027 ceph-mon[105164]: Deploying daemon osd.3 on smithi017 2023-09-23T14:53:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:19 smithi027 ceph-mon[105164]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T14:53:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:19 smithi017 ceph-mon[100189]: Deploying daemon osd.3 on smithi017 2023-09-23T14:53:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:19 smithi017 ceph-mon[100189]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T14:53:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:19 smithi017 ceph-mon[118128]: Deploying daemon osd.3 on smithi017 2023-09-23T14:53:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:19 smithi017 ceph-mon[118128]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T14:53:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:21 smithi017 ceph-mon[100189]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T14:53:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:21 smithi017 ceph-mon[118128]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T14:53:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:21 smithi027 ceph-mon[105164]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T14:53:24.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[100189]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T14:53:24.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:24.147 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.147 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[118128]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T14:53:24.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:23 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:23 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:23 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:24.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:23 smithi027 ceph-mon[105164]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T14:53:24.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:23 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:25.782 INFO:teuthology.orchestra.run.smithi017.stdout:Created osd(s) 3 on host 'smithi017' 2023-09-23T14:53:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:25 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:26.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:26.113 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:26.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:26.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:26.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:25 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:26.459 DEBUG:teuthology.orchestra.run.smithi017:osd.3> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T14:53:26.461 INFO:tasks.cephadm:Deploying osd.4 on smithi027 with /dev/vg_nvme/lv_4... 2023-09-23T14:53:26.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T14:53:26.493 INFO:journalctl@ceph.osd.3.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:53:26.819 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:53:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:26 smithi027 ceph-mon[105164]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:26 smithi027 ceph-mon[105164]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:26 smithi017 ceph-mon[100189]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:26 smithi017 ceph-mon[100189]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:26 smithi017 ceph-mon[118128]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.137 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:26 smithi017 ceph-mon[118128]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:27.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:27 smithi027 ceph-mon[105164]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[100189]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: from='osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T14:53:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:27 smithi017 ceph-mon[118128]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T14:53:28.195 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 14:53:27 smithi017 conmon[152758]: 2023-09-23T14:53:27.835+0000 7ff306195700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T14:53:28.195 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 14:53:27 smithi017 conmon[152758]: 2023-09-23T14:53:27.839+0000 7ff302f94700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0107877 s, 972 MB/s 2023-09-23T14:53:28.471 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2023-09-23T14:53:29.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi027:vg_nvme/lv_4 2023-09-23T14:53:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:28 smithi027 ceph-mon[105164]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:28 smithi027 ceph-mon[105164]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T14:53:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:28 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:28 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[100189]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[100189]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[118128]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]': finished 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[118128]: osdmap e25: 4 total, 3 up, 4 in 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:28 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.429 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:53:29.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:53:29.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:53:29.843 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025] boot 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:29.845 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.918 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:53:29.918 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:53:29.918 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025] boot 2023-09-23T14:53:29.918 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T14:53:29.918 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:29.919 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:29.920 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:29 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:30.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: osd.3 [v2:172.21.15.17:6826/4002411025,v1:172.21.15.17:6827/4002411025] boot 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:30.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:30.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:30.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:30.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:29 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:30 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi017 2023-09-23T14:53:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:30 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi017 to 4037M 2023-09-23T14:53:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:30 smithi027 ceph-mon[105164]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T14:53:31.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi017 2023-09-23T14:53:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi017 to 4037M 2023-09-23T14:53:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[100189]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T14:53:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi017 2023-09-23T14:53:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi017 to 4037M 2023-09-23T14:53:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:30 smithi017 ceph-mon[118128]: osdmap e27: 4 total, 4 up, 4 in 2023-09-23T14:53:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:31 smithi027 ceph-mon[105164]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:31 smithi027 ceph-mon[105164]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:32.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[100189]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[100189]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[118128]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[118128]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/3787499896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]': finished 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T14:53:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/3787499896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]': finished 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T14:53:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:33 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/3787499896' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]: dispatch 2023-09-23T14:53:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5b9137a6-fdb7-4150-813f-c72707df3509"}]': finished 2023-09-23T14:53:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T14:53:34.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:33 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:35.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:34 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/1905908201' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:35.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:34 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/1905908201' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:35.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:34 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/1905908201' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:36.170 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:35 smithi027 ceph-mon[105164]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:36.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:35 smithi017 ceph-mon[100189]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:35 smithi017 ceph-mon[118128]: pgmap v96: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:38.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:37 smithi027 ceph-mon[105164]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:38.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:37 smithi017 ceph-mon[100189]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:38.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:37 smithi017 ceph-mon[118128]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:39 smithi027 ceph-mon[105164]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T14:53:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:39 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:40.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[100189]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T14:53:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[118128]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T14:53:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:39 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:41.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:40 smithi027 ceph-mon[105164]: Deploying daemon osd.4 on smithi027 2023-09-23T14:53:41.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:40 smithi017 ceph-mon[100189]: Deploying daemon osd.4 on smithi027 2023-09-23T14:53:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:40 smithi017 ceph-mon[118128]: Deploying daemon osd.4 on smithi027 2023-09-23T14:53:42.287 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:41 smithi027 ceph-mon[105164]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:42.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:41 smithi017 ceph-mon[118128]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:41 smithi017 ceph-mon[100189]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:44.158 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:43 smithi027 ceph-mon[105164]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:44.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:43 smithi017 ceph-mon[100189]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:43 smithi017 ceph-mon[118128]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:45.541 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:45 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.542 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:45 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.542 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:45 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:45.542 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:45 smithi027 ceph-mon[105164]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[100189]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:45 smithi017 ceph-mon[118128]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:47.108 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 4 on host 'smithi027' 2023-09-23T14:53:47.282 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.283 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.283 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:47.283 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:47.283 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:46 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:47.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:46 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:47.682 DEBUG:teuthology.orchestra.run.smithi027:osd.4> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T14:53:47.724 INFO:tasks.cephadm:Deploying osd.5 on smithi027 with /dev/vg_nvme/lv_3... 2023-09-23T14:53:47.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T14:53:47.758 INFO:journalctl@ceph.osd.4.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:47 smithi027 ceph-mon[105164]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[100189]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:48.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:53:48.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:48.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:47 smithi017 ceph-mon[118128]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T14:53:48.571 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:49 smithi027 ceph-mon[105164]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:49.850 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 14:53:49 smithi027 conmon[113211]: 2023-09-23T14:53:49.477+0000 7fae64af8700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T14:53:49.851 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 14:53:49 smithi027 conmon[113211]: 2023-09-23T14:53:49.481+0000 7fae610f6700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:53:49.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[118128]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: from='osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:53:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:49 smithi017 ceph-mon[100189]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T14:53:50.468 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T14:53:50.468 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T14:53:50.469 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2023-09-23T14:53:50.469 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2023-09-23T14:53:50.469 INFO:teuthology.orchestra.run.smithi027.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0108846 s, 963 MB/s 2023-09-23T14:53:50.469 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2023-09-23T14:53:50.791 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:50 smithi027 ceph-mon[105164]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:53:50.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:50 smithi027 ceph-mon[105164]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T14:53:50.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:50 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:50 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:50 smithi027 ceph-mon[105164]: osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574] boot 2023-09-23T14:53:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[100189]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:53:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[100189]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[100189]: osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574] boot 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[118128]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[118128]: osdmap e30: 5 total, 4 up, 5 in 2023-09-23T14:53:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:50 smithi017 ceph-mon[118128]: osd.4 [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574] boot 2023-09-23T14:53:50.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi027:vg_nvme/lv_3 2023-09-23T14:53:51.311 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi027 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:51.807 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 17168M 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:51.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:51 smithi027 ceph-mon[105164]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi027 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 17168M 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[100189]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:53:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi027 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 17168M 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:53:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:53:51.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:51.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:51 smithi017 ceph-mon[118128]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T14:53:53.789 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:53.790 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:53 smithi027 ceph-mon[105164]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:53.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[100189]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T14:53:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:53:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T14:53:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:53:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:53:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:53:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:53 smithi017 ceph-mon[118128]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:53:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:54 smithi017 ceph-mon[100189]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T14:53:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:54 smithi017 ceph-mon[118128]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T14:53:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:54 smithi027 ceph-mon[105164]: osdmap e35: 5 total, 5 up, 5 in 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/1831765471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]': finished 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:53:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/2102827035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/1831765471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]': finished 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:53:56.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:55 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/2102827035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: pgmap v113: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2023-09-23T14:53:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/1831765471' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]: dispatch 2023-09-23T14:53:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06d868f4-810f-45ef-be6a-0c6edcea5ad7"}]': finished 2023-09-23T14:53:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T14:53:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:53:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:55 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/2102827035' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:53:58.264 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:57 smithi027 ceph-mon[105164]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2023-09-23T14:53:58.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:57 smithi017 ceph-mon[100189]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2023-09-23T14:53:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:57 smithi017 ceph-mon[118128]: pgmap v115: 1 pgs: 1 peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 129 KiB/s, 0 objects/s recovering 2023-09-23T14:54:00.144 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:53:59 smithi027 ceph-mon[105164]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:53:59 smithi017 ceph-mon[100189]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:00.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:53:59 smithi017 ceph-mon[118128]: pgmap v116: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:02.235 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:01 smithi027 ceph-mon[105164]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:02.235 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T14:54:02.235 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:01 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[100189]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[118128]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T14:54:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:01 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:03.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:02 smithi027 ceph-mon[105164]: Deploying daemon osd.5 on smithi027 2023-09-23T14:54:03.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:02 smithi017 ceph-mon[100189]: Deploying daemon osd.5 on smithi027 2023-09-23T14:54:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:02 smithi017 ceph-mon[118128]: Deploying daemon osd.5 on smithi027 2023-09-23T14:54:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:03 smithi027 ceph-mon[105164]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:03 smithi017 ceph-mon[100189]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:03 smithi017 ceph-mon[118128]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:06 smithi027 ceph-mon[105164]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:06 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:06 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:06 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:06.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[100189]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[118128]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:05 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:08.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:08 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.403 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 5 on host 'smithi027' 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:08.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:08.936 DEBUG:teuthology.orchestra.run.smithi027:osd.5> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T14:54:08.939 INFO:tasks.cephadm:Deploying osd.6 on smithi027 with /dev/vg_nvme/lv_2... 2023-09-23T14:54:08.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T14:54:08.970 INFO:journalctl@ceph.osd.5.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:54:09.471 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.471 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.471 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:09.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:09 smithi027 ceph-mon[105164]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[100189]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T14:54:09.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:09 smithi017 ceph-mon[118128]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:09.885 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:54:10.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:10 smithi027 ceph-mon[105164]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T14:54:10.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:10 smithi027 ceph-mon[105164]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T14:54:10.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:10 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:10.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:10 smithi027 ceph-mon[105164]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:10.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:10 smithi027 ceph-mon[105164]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:10.531 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 14:54:10 smithi027 conmon[122090]: 2023-09-23T14:54:10.387+0000 7ffb26ddb700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T14:54:10.531 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 14:54:10 smithi027 conmon[122090]: 2023-09-23T14:54:10.392+0000 7ffb20bd4700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:54:10.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[100189]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[100189]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[100189]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[100189]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[118128]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T14:54:10.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[118128]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T14:54:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[118128]: from='osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:10 smithi017 ceph-mon[118128]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:11.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:11 smithi027 ceph-mon[105164]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:11.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:11 smithi027 ceph-mon[105164]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T14:54:11.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:11 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:11 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:11 smithi027 ceph-mon[105164]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[100189]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[100189]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[100189]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[118128]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[118128]: osdmap e38: 6 total, 5 up, 6 in 2023-09-23T14:54:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:11 smithi017 ceph-mon[118128]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T14:54:11.794 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T14:54:11.795 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T14:54:11.795 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2023-09-23T14:54:11.795 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2023-09-23T14:54:11.795 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0108316 s, 968 MB/s 2023-09-23T14:54:11.795 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2023-09-23T14:54:12.403 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi027:vg_nvme/lv_2 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946] boot 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.430 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:12.430 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:12.430 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.430 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:12 smithi027 ceph-mon[105164]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946] boot 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[100189]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:54:12.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: osd.5 [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946] boot 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:12.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:12 smithi017 ceph-mon[118128]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T14:54:12.752 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:54:13.943 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:13 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi027 2023-09-23T14:54:13.943 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:13 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 8584M 2023-09-23T14:54:13.943 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:13 smithi027 ceph-mon[105164]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:13.943 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:13 smithi027 ceph-mon[105164]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi027 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 8584M 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[100189]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[100189]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi027 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 8584M 2023-09-23T14:54:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[118128]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:14.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:13 smithi017 ceph-mon[118128]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T14:54:15.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:15.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:15.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:14 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:15.212 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:15.212 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:15.212 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:14 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:16.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[100189]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[100189]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T14:54:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[100189]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T14:54:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[118128]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[118128]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T14:54:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:15 smithi017 ceph-mon[118128]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T14:54:16.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:15 smithi027 ceph-mon[105164]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:16.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:15 smithi027 ceph-mon[105164]: osdmap e42: 6 total, 6 up, 6 in 2023-09-23T14:54:16.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:15 smithi027 ceph-mon[105164]: pgmap v130: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 2/6 objects degraded (33.333%) 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/157779766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]': finished 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/157779766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]': finished 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T14:54:17.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:16 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:17.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-09-23T14:54:17.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: osdmap e43: 6 total, 6 up, 6 in 2023-09-23T14:54:17.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/157779766' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]: dispatch 2023-09-23T14:54:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "06f02713-faf7-456c-bd0d-52eb13680742"}]': finished 2023-09-23T14:54:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T14:54:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:16 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:18.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:17 smithi017 ceph-mon[100189]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:18.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:17 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/1177788192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:18.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:17 smithi017 ceph-mon[118128]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:18.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:17 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/1177788192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:17 smithi027 ceph-mon[105164]: pgmap v133: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:17 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/1177788192' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:19.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:18 smithi017 ceph-mon[100189]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T14:54:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:18 smithi017 ceph-mon[100189]: Cluster is now healthy 2023-09-23T14:54:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:18 smithi017 ceph-mon[118128]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T14:54:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:18 smithi017 ceph-mon[118128]: Cluster is now healthy 2023-09-23T14:54:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:18 smithi027 ceph-mon[105164]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-09-23T14:54:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:18 smithi027 ceph-mon[105164]: Cluster is now healthy 2023-09-23T14:54:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:19 smithi017 ceph-mon[100189]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:19 smithi017 ceph-mon[118128]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:19 smithi027 ceph-mon[105164]: pgmap v134: 1 pgs: 1 active+clean; 577 KiB data, 36 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:21 smithi027 ceph-mon[105164]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:21 smithi017 ceph-mon[100189]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:21 smithi017 ceph-mon[118128]: pgmap v135: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 95 KiB/s, 0 objects/s recovering 2023-09-23T14:54:23.247 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:22 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T14:54:23.247 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:22 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:23.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:22 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T14:54:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:22 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:22 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T14:54:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:22 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:23 smithi027 ceph-mon[105164]: Deploying daemon osd.6 on smithi027 2023-09-23T14:54:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:23 smithi027 ceph-mon[105164]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:24.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:23 smithi017 ceph-mon[100189]: Deploying daemon osd.6 on smithi027 2023-09-23T14:54:24.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:23 smithi017 ceph-mon[100189]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:23 smithi017 ceph-mon[118128]: Deploying daemon osd.6 on smithi027 2023-09-23T14:54:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:23 smithi017 ceph-mon[118128]: pgmap v136: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:26.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:25 smithi027 ceph-mon[105164]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:54:26.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:25 smithi017 ceph-mon[100189]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:54:26.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:25 smithi017 ceph-mon[118128]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:54:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:28 smithi027 ceph-mon[105164]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:28.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:28 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:28 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:28 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:28.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[100189]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[118128]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:28 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:30.083 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 6 on host 'smithi027' 2023-09-23T14:54:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:30 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:30 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:30.631 DEBUG:teuthology.orchestra.run.smithi027:osd.6> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T14:54:30.673 INFO:tasks.cephadm:Deploying osd.7 on smithi027 with /dev/vg_nvme/lv_1... 2023-09-23T14:54:30.674 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T14:54:30.707 INFO:journalctl@ceph.osd.6.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:54:31.339 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:54:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:31 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:31 smithi027 ceph-mon[105164]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:31 smithi027 ceph-mon[105164]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[118128]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[118128]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[100189]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:31 smithi017 ceph-mon[100189]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T14:54:32.302 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:32.302 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T14:54:32.303 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T14:54:32.303 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:32.303 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:32.303 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:32 smithi027 ceph-mon[105164]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:32.303 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 14:54:32 smithi027 conmon[131674]: 2023-09-23T14:54:32.043+0000 7fb08f991700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T14:54:32.303 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 14:54:32 smithi027 conmon[131674]: 2023-09-23T14:54:32.051+0000 7fb08bf8f700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:54:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[100189]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T14:54:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: osdmap e45: 7 total, 6 up, 7 in 2023-09-23T14:54:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: from='osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:32 smithi017 ceph-mon[118128]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010] boot 2023-09-23T14:54:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T14:54:33.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:33 smithi027 ceph-mon[105164]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010] boot 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[100189]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: osdmap e46: 7 total, 6 up, 7 in 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: osd.6 [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010] boot 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:54:33.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:33 smithi017 ceph-mon[118128]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10+0 records in 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr:10+0 records out 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0155294 s, 675 MB/s 2023-09-23T14:54:33.451 INFO:teuthology.orchestra.run.smithi027.stderr:--> Zapping successful for: 2023-09-23T14:54:34.331 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:34 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:54:34.331 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:34 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:54:34.332 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:34 smithi027 ceph-mon[105164]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T14:54:34.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi027:vg_nvme/lv_1 2023-09-23T14:54:34.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:54:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:54:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[118128]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T14:54:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:54:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:54:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:34 smithi017 ceph-mon[100189]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T14:54:34.729 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:54:35.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi027 2023-09-23T14:54:35.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 5722M 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:35.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:35.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T14:54:35.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:35 smithi027 ceph-mon[105164]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi027 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 5722M 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T14:54:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[100189]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi027 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 5722M 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T14:54:35.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:35 smithi017 ceph-mon[118128]: pgmap v147: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[100189]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T14:54:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[118128]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T14:54:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:36 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:37.209 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:36 smithi027 ceph-mon[105164]: osdmap e50: 7 total, 7 up, 7 in 2023-09-23T14:54:37.209 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T14:54:37.209 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T14:54:37.209 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:36 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:38.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[100189]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:38.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[100189]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T14:54:38.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[100189]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:38.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[118128]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:38.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[118128]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T14:54:38.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:37 smithi017 ceph-mon[118128]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:38.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:37 smithi027 ceph-mon[105164]: from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi027:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:54:38.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:37 smithi027 ceph-mon[105164]: osdmap e51: 7 total, 7 up, 7 in 2023-09-23T14:54:38.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:37 smithi027 ceph-mon[105164]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:39.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/3178885667' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[100189]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]': finished 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[100189]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/3178885667' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[118128]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]': finished 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[118128]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T14:54:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:38 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:38 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/3178885667' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:38 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]: dispatch 2023-09-23T14:54:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:38 smithi027 ceph-mon[105164]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4"}]': finished 2023-09-23T14:54:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:38 smithi027 ceph-mon[105164]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T14:54:39.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:38 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:39 smithi017 ceph-mon[100189]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T14:54:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:39 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/4020925868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:39 smithi017 ceph-mon[118128]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T14:54:40.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:39 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/4020925868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:39 smithi027 ceph-mon[105164]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 143 KiB/s, 0 objects/s recovering 2023-09-23T14:54:40.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:39 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/4020925868' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T14:54:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:41 smithi027 ceph-mon[105164]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:54:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:41 smithi017 ceph-mon[100189]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:54:42.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:41 smithi017 ceph-mon[118128]: pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:54:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:43 smithi017 ceph-mon[100189]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T14:54:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:43 smithi017 ceph-mon[118128]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T14:54:44.329 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:43 smithi027 ceph-mon[105164]: pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 82 KiB/s, 0 objects/s recovering 2023-09-23T14:54:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T14:54:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:44 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T14:54:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:44 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:45.250 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T14:54:45.250 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:44 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:45 smithi017 ceph-mon[100189]: Deploying daemon osd.7 on smithi027 2023-09-23T14:54:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:45 smithi017 ceph-mon[100189]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:45 smithi017 ceph-mon[118128]: Deploying daemon osd.7 on smithi027 2023-09-23T14:54:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:45 smithi017 ceph-mon[118128]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:46.243 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:45 smithi027 ceph-mon[105164]: Deploying daemon osd.7 on smithi027 2023-09-23T14:54:46.244 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:45 smithi027 ceph-mon[105164]: pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T14:54:48.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:48 smithi027 ceph-mon[105164]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T14:54:48.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:47 smithi017 ceph-mon[100189]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T14:54:48.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:47 smithi017 ceph-mon[118128]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T14:54:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:50 smithi027 ceph-mon[105164]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:50 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:50 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:50 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[100189]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[118128]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:50.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:50 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:52.145 INFO:teuthology.orchestra.run.smithi027.stdout:Created osd(s) 7 on host 'smithi027' 2023-09-23T14:54:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:52 smithi027 ceph-mon[105164]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:52 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[100189]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[118128]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:52 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:52.703 DEBUG:teuthology.orchestra.run.smithi027:osd.7> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T14:54:52.706 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T14:54:52.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T14:54:52.737 INFO:journalctl@ceph.osd.7.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:54:53.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:53.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:53.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[118128]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:53.081 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:54:53.249 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:53.249 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:53.249 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.249 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.249 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.250 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:53.250 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:53.250 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:53 smithi027 ceph-mon[105164]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:53.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:53.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:53.294 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:53 smithi017 ceph-mon[100189]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T14:54:54.350 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 14:54:54 smithi027 conmon[141165]: 2023-09-23T14:54:54.129+0000 7f30b24f2700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T14:54:54.350 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 14:54:54 smithi027 conmon[141165]: 2023-09-23T14:54:54.137+0000 7f30ab2e9700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T14:54:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T14:54:54.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T14:54:54.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:54.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:54 smithi027 ceph-mon[105164]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:54.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T14:54:54.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T14:54:54.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:54.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[100189]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:54.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T14:54:54.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: osdmap e53: 8 total, 7 up, 8 in 2023-09-23T14:54:54.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:54.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: from='osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:54 smithi017 ceph-mon[118128]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T14:54:54.743 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:54:55.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:55.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T14:54:55.371 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":54,"num_osds":8,"num_up_osds":7,"osd_up_since":1695480872,"num_in_osds":8,"osd_in_since":1695480878,"num_remapped_pgs":0} 2023-09-23T14:54:55.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3930792650' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[118128]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3930792650' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:55 smithi017 ceph-mon[100189]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]': finished 2023-09-23T14:54:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: osdmap e54: 8 total, 7 up, 8 in 2023-09-23T14:54:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:55.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3930792650' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:55.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:55 smithi027 ceph-mon[105164]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T14:54:56.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: purged_snaps scrub starts 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: purged_snaps scrub ok 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591] boot 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: Detected new or changed devices on smithi027 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 4292M 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:56.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: purged_snaps scrub starts 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: purged_snaps scrub ok 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591] boot 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: Detected new or changed devices on smithi027 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 4292M 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:56.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:56 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: purged_snaps scrub starts 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: purged_snaps scrub ok 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: osd.7 [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591] boot 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: Detected new or changed devices on smithi027 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 4292M 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:54:56.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:56 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:54:56.744 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:54:57.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[100189]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T14:54:57.301 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[100189]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:54:57.302 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[100189]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T14:54:57.302 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[118128]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T14:54:57.302 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[118128]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:54:57.302 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:57 smithi017 ceph-mon[118128]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T14:54:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:57 smithi027 ceph-mon[105164]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T14:54:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:57 smithi027 ceph-mon[105164]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:54:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:57 smithi027 ceph-mon[105164]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T14:54:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:54:58.999 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":58,"num_osds":8,"num_up_osds":8,"osd_up_since":1695480895,"num_in_osds":8,"osd_in_since":1695480878,"num_remapped_pgs":0} 2023-09-23T14:54:59.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T14:54:59.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:58 smithi017 ceph-mon[100189]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T14:54:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:58 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3413277193' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:58 smithi017 ceph-mon[118128]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T14:54:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:58 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3413277193' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:58 smithi027 ceph-mon[105164]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T14:54:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:58 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3413277193' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T14:54:59.392 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:54:59.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:59 smithi017 ceph-mon[118128]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T14:54:59.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:54:59 smithi017 ceph-mon[118128]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:00.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:59 smithi017 ceph-mon[100189]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T14:55:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:54:59 smithi017 ceph-mon[100189]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:59 smithi027 ceph-mon[105164]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T14:55:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:54:59 smithi027 ceph-mon[105164]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:01.045 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:55:01.045 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":59,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:54:58.911189+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:55:01.667 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T14:53:07.440399+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T14:55:01.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T14:55:02.020 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:02.261 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:02 smithi017 ceph-mon[100189]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:02.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:02 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3490107981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:02.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:02 smithi017 ceph-mon[118128]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:02.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:02 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3490107981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:02 smithi027 ceph-mon[105164]: pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:02 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3490107981' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:03.657 INFO:teuthology.orchestra.run.smithi017.stdout:pg_num: 1 2023-09-23T14:55:03.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:03 smithi017 ceph-mon[100189]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:55:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:03 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1945618096' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T14:55:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:03 smithi017 ceph-mon[118128]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:55:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:03 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1945618096' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T14:55:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:03 smithi027 ceph-mon[105164]: pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T14:55:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:03 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1945618096' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T14:55:04.366 INFO:tasks.cephadm:Adding prometheus.a on smithi027 2023-09-23T14:55:04.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi027=a' 2023-09-23T14:55:04.736 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:55:06.281 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled prometheus update... 2023-09-23T14:55:06.306 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:06 smithi027 ceph-mon[105164]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T14:55:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:06 smithi017 ceph-mon[100189]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T14:55:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:06 smithi017 ceph-mon[118128]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T14:55:06.958 DEBUG:teuthology.orchestra.run.smithi027:prometheus.a> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T14:55:06.960 INFO:tasks.cephadm:Adding node-exporter.a on smithi017 2023-09-23T14:55:06.961 INFO:tasks.cephadm:Adding node-exporter.b on smithi027 2023-09-23T14:55:06.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi017=a;smithi027=b' 2023-09-23T14:55:06.994 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:55:07.336 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:55:07.547 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:07.547 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: Saving service prometheus spec with placement smithi027=a;count:1 2023-09-23T14:55:07.547 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: Deploying daemon prometheus.a on smithi027 2023-09-23T14:55:07.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:07 smithi027 ceph-mon[105164]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: Saving service prometheus spec with placement smithi027=a;count:1 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: Deploying daemon prometheus.a on smithi027 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[118128]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: Saving service prometheus spec with placement smithi027=a;count:1 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:07.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:07.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:07.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:07.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: Deploying daemon prometheus.a on smithi027 2023-09-23T14:55:07.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:07 smithi017 ceph-mon[100189]: pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 64 KiB/s, 0 objects/s recovering 2023-09-23T14:55:09.462 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled node-exporter update... 2023-09-23T14:55:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:10 smithi027 ceph-mon[105164]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T14:55:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:10 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:10.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:10 smithi017 ceph-mon[100189]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T14:55:10.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:10 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:10 smithi017 ceph-mon[118128]: pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2023-09-23T14:55:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:10 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:10.800 DEBUG:teuthology.orchestra.run.smithi017:node-exporter.a> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T14:55:10.802 DEBUG:teuthology.orchestra.run.smithi027:node-exporter.b> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T14:55:10.804 INFO:tasks.cephadm:Adding alertmanager.a on smithi017 2023-09-23T14:55:10.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi017=a' 2023-09-23T14:55:10.834 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:55:10.836 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:55:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:11 smithi027 ceph-mon[105164]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi027=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:11 smithi027 ceph-mon[105164]: Saving service node-exporter spec with placement smithi017=a;smithi027=b;count:2 2023-09-23T14:55:11.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:11 smithi017 ceph-mon[100189]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi027=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:11 smithi017 ceph-mon[100189]: Saving service node-exporter spec with placement smithi017=a;smithi027=b;count:2 2023-09-23T14:55:11.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:11 smithi017 ceph-mon[118128]: from='client.24341 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi017=a;smithi027=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:11.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:11 smithi017 ceph-mon[118128]: Saving service node-exporter spec with placement smithi017=a;smithi027=b;count:2 2023-09-23T14:55:12.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:12 smithi027 ceph-mon[105164]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:12.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:12 smithi017 ceph-mon[100189]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:12.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:12 smithi017 ceph-mon[118128]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:13.311 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:55:14.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:13 smithi017 ceph-mon[100189]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:13 smithi017 ceph-mon[118128]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:14.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:13 smithi027 ceph-mon[105164]: pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T14:55:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:16 smithi027 ceph-mon[105164]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:16.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:16 smithi017 ceph-mon[100189]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:16 smithi017 ceph-mon[118128]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:18.198 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled alertmanager update... 2023-09-23T14:55:18.228 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:18 smithi027 ceph-mon[105164]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:18.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:18 smithi017 ceph-mon[100189]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:18.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:18 smithi017 ceph-mon[118128]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:18.845 DEBUG:teuthology.orchestra.run.smithi017:alertmanager.a> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T14:55:18.848 INFO:tasks.cephadm:Adding grafana.a on smithi027 2023-09-23T14:55:18.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi027=a' 2023-09-23T14:55:18.876 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T14:55:19.200 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:18 smithi027 systemd[1]: Starting Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:55:19.201 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:19 smithi027 ceph-mon[105164]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:19.201 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:19 smithi027 ceph-mon[105164]: Saving service alertmanager spec with placement smithi017=a;count:1 2023-09-23T14:55:19.201 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:19 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:19.201 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:19 smithi027 ceph-mon[105164]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:19.553 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[100189]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[100189]: Saving service alertmanager spec with placement smithi017=a;count:1 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[100189]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[118128]: from='client.24347 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi017=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[118128]: Saving service alertmanager spec with placement smithi017=a;count:1 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:19.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:19 smithi017 ceph-mon[118128]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 bash[148499]: 3ec20d6247fb4072ff406348df60c065be1f0a4366d4c35fcd2ebb0e75935b85 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 systemd[1]: Started Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.558Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.558Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.558Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi027 (none))" 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.558Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.558Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.560Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.560Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T14:55:19.821 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.561Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.561Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.563Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.563Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.756µs 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.563Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.565Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.565Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.77µs wal_replay_duration=1.56361ms wbl_replay_duration=186ns total_replay_duration=1.63641ms 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.566Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.566Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.566Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.588Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.215965ms db_storage=891ns remote_storage=1.211µs web_handler=586ns query_engine=865ns scrape=735.656µs scrape_sd=28.699µs notify=973ns notify_sd=2.4µs rules=20.182773ms tracing=7.089µs 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.588Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T14:55:19.822 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:55:19 smithi027 conmon[148619]: ts=2023-09-23T14:55:19.588Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:20 smithi027 conmon[108383]: ignoring --setuser ceph since I am not root 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:20 smithi027 conmon[108383]: ignoring --setgroup ceph since I am not root 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:20 smithi027 ceph-mgr[108407]: -- 172.21.15.27:0/1470141474 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5588e40ae9c0 con 0x5588e4086c00 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:20 smithi027 conmon[108383]: 2023-09-23T14:55:20.856+0000 7f1dcc1371c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.874 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.875 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.875 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:20 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T14:55:20.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' 2023-09-23T14:55:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T14:55:20.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:20 smithi017 conmon[100620]: ignoring --setuser ceph since I am not root 2023-09-23T14:55:20.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:20 smithi017 conmon[100620]: ignoring --setgroup ceph since I am not root 2023-09-23T14:55:20.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:20 smithi017 ceph-mgr[100644]: -- 172.21.15.17:0/1101448495 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c7404589c0 con 0x55c740431000 2023-09-23T14:55:20.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:20 smithi017 conmon[100620]: 2023-09-23T14:55:20.850+0000 7f730401c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:55:21.182 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:20 smithi027 conmon[108383]: 2023-09-23T14:55:20.956+0000 7f1dcc1371c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:55:21.182 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:21 smithi027 conmon[108383]: 2023-09-23T14:55:21.065+0000 7f1dcc1371c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:55:21.182 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:21 smithi027 conmon[108383]: 2023-09-23T14:55:21.181+0000 7f1dcc1371c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:55:21.361 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:20 smithi017 conmon[100620]: 2023-09-23T14:55:20.943+0000 7f730401c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:55:21.361 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:21 smithi017 conmon[100620]: 2023-09-23T14:55:21.049+0000 7f730401c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:55:21.362 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:21 smithi017 conmon[100620]: 2023-09-23T14:55:21.158+0000 7f730401c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:55:21.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:21 smithi027 conmon[108383]: 2023-09-23T14:55:21.384+0000 7f1dcc1371c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:55:21.660 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:21 smithi017 conmon[100620]: 2023-09-23T14:55:21.360+0000 7f730401c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:55:21.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:21 smithi017 ceph-mon[100189]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T14:55:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:21 smithi017 ceph-mon[100189]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T14:55:21.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:21 smithi017 conmon[100620]: 2023-09-23T14:55:21.754+0000 7f730401c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:55:21.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:21 smithi017 conmon[100620]: 2023-09-23T14:55:21.861+0000 7f730401c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:55:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:21 smithi017 ceph-mon[118128]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T14:55:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:21 smithi017 ceph-mon[118128]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T14:55:22.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:21 smithi027 ceph-mon[105164]: from='mgr.14152 172.21.15.17:0/1597123351' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T14:55:22.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:21 smithi027 ceph-mon[105164]: mgrmap e16: y(active, since 4m), standbys: x 2023-09-23T14:55:22.033 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:21 smithi027 conmon[108383]: 2023-09-23T14:55:21.778+0000 7f1dcc1371c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:55:22.033 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:21 smithi027 conmon[108383]: 2023-09-23T14:55:21.884+0000 7f1dcc1371c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:55:22.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:22 smithi027 conmon[108383]: 2023-09-23T14:55:22.032+0000 7f1dcc1371c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:55:22.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:22 smithi027 conmon[108383]: 2023-09-23T14:55:22.145+0000 7f1dcc1371c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:55:22.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:22 smithi017 conmon[100620]: 2023-09-23T14:55:22.020+0000 7f730401c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:55:22.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:22 smithi017 conmon[100620]: 2023-09-23T14:55:22.133+0000 7f730401c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:55:22.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:22 smithi017 conmon[100620]: 2023-09-23T14:55:22.637+0000 7f730401c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:55:23.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:22 smithi027 conmon[108383]: 2023-09-23T14:55:22.648+0000 7f1dcc1371c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:55:23.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:23 smithi027 conmon[108383]: 2023-09-23T14:55:23.320+0000 7f1dcc1371c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:55:23.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:23 smithi027 conmon[108383]: 2023-09-23T14:55:23.508+0000 7f1dcc1371c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:55:23.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:23 smithi017 conmon[100620]: 2023-09-23T14:55:23.313+0000 7f730401c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:55:23.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:23 smithi017 conmon[100620]: 2023-09-23T14:55:23.503+0000 7f730401c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:55:24.784 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:24 smithi027 conmon[108383]: 2023-09-23T14:55:24.461+0000 7f1dcc1371c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:55:24.784 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:24 smithi027 conmon[108383]: 2023-09-23T14:55:24.570+0000 7f1dcc1371c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:55:24.785 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:24 smithi017 conmon[100620]: 2023-09-23T14:55:24.459+0000 7f730401c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:55:24.785 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:24 smithi017 conmon[100620]: 2023-09-23T14:55:24.569+0000 7f730401c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:55:25.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:24 smithi027 conmon[108383]: 2023-09-23T14:55:24.784+0000 7f1dcc1371c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:55:25.129 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:24 smithi017 conmon[100620]: 2023-09-23T14:55:24.784+0000 7f730401c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:55:25.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:25 smithi017 conmon[100620]: 2023-09-23T14:55:25.128+0000 7f730401c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:55:25.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:25 smithi017 conmon[100620]: 2023-09-23T14:55:25.232+0000 7f730401c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:55:25.557 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:25 smithi027 conmon[108383]: 2023-09-23T14:55:25.126+0000 7f1dcc1371c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:55:25.558 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:25 smithi027 conmon[108383]: 2023-09-23T14:55:25.230+0000 7f1dcc1371c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:55:25.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:25 smithi027 conmon[108383]: 2023-09-23T14:55:25.557+0000 7f1dcc1371c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:55:25.887 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:25 smithi017 conmon[100620]: 2023-09-23T14:55:25.561+0000 7f730401c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:55:26.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:25 smithi017 conmon[100620]: 2023-09-23T14:55:25.886+0000 7f730401c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:55:26.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:25 smithi027 conmon[108383]: 2023-09-23T14:55:25.880+0000 7f1dcc1371c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:55:27.093 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:26 smithi027 conmon[108383]: 2023-09-23T14:55:26.699+0000 7f1dcc1371c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:55:27.094 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:26 smithi027 conmon[108383]: 2023-09-23T14:55:26.855+0000 7f1dcc1371c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:55:27.116 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:26 smithi017 conmon[100620]: 2023-09-23T14:55:26.715+0000 7f730401c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:55:27.116 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:26 smithi017 conmon[100620]: 2023-09-23T14:55:26.873+0000 7f730401c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:55:27.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:27 smithi027 conmon[108383]: 2023-09-23T14:55:27.094+0000 7f1dcc1371c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:55:27.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:27 smithi027 conmon[108383]: 2023-09-23T14:55:27.230+0000 7f1dcc1371c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:55:27.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:27 smithi017 conmon[100620]: 2023-09-23T14:55:27.115+0000 7f730401c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:55:27.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:27 smithi017 conmon[100620]: 2023-09-23T14:55:27.253+0000 7f730401c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:55:27.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:27 smithi027 conmon[108383]: 2023-09-23T14:55:27.497+0000 7f1dcc1371c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:55:27.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:27 smithi017 conmon[100620]: 2023-09-23T14:55:27.522+0000 7f730401c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: 2023-09-23T14:55:28.527+0000 7f1dcc1371c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: [23/Sep/2023:14:55:28] ENGINE Bus STARTING 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: CherryPy Checker: 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: The Application mounted at '' has an empty config. 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: [23/Sep/2023:14:55:28] ENGINE Serving on http://:::9283 2023-09-23T14:55:28.797 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:28 smithi027 conmon[108383]: [23/Sep/2023:14:55:28] ENGINE Bus STARTED 2023-09-23T14:55:28.911 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: 2023-09-23T14:55:28.566+0000 7f730401c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:55:28.911 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: [23/Sep/2023:14:55:28] ENGINE Bus STARTING 2023-09-23T14:55:28.911 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: CherryPy Checker: 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: The Application mounted at '' has an empty config. 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: [23/Sep/2023:14:55:28] ENGINE Serving on http://:::9283 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:28 smithi017 conmon[100620]: [23/Sep/2023:14:55:28] ENGINE Bus STARTED 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: Standby manager daemon x restarted 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: Standby manager daemon x started 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:55:28.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: Active manager daemon y restarted 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: Activating manager daemon y 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: mgrmap e17: y(active, starting, since 0.00697412s), standbys: x 2023-09-23T14:55:28.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: Standby manager daemon x restarted 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: Standby manager daemon x started 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:55:28.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: Active manager daemon y restarted 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: Activating manager daemon y 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: mgrmap e17: y(active, starting, since 0.00697412s), standbys: x 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:55:28.915 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: Manager daemon y is now available 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:28.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:28.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: Standby manager daemon x restarted 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: Standby manager daemon x started 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/3340575212' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: Active manager daemon y restarted 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: Activating manager daemon y 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: mgrmap e17: y(active, starting, since 0.00697412s), standbys: x 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:55:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:55:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: Manager daemon y is now available 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:55:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: Manager daemon y is now available 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:29.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:55:29.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:55:29.605 INFO:teuthology.orchestra.run.smithi027.stdout:Scheduled grafana update... 2023-09-23T14:55:30.367 DEBUG:teuthology.orchestra.run.smithi027:grafana.a> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T14:55:30.370 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T14:55:30.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T14:55:30.402 INFO:journalctl@ceph.grafana.a.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T14:55:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: [23/Sep/2023:14:55:29] ENGINE Bus STARTING 2023-09-23T14:55:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: [23/Sep/2023:14:55:29] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: [23/Sep/2023:14:55:29] ENGINE Bus STARTED 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: mgrmap e18: y(active, since 1.01067s), standbys: x 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: Saving service grafana spec with placement smithi027=a;count:1 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:30 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: [23/Sep/2023:14:55:29] ENGINE Bus STARTING 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: [23/Sep/2023:14:55:29] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: [23/Sep/2023:14:55:29] ENGINE Bus STARTED 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: mgrmap e18: y(active, since 1.01067s), standbys: x 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: Saving service grafana spec with placement smithi027=a;count:1 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: [23/Sep/2023:14:55:29] ENGINE Bus STARTING 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: [23/Sep/2023:14:55:29] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: [23/Sep/2023:14:55:29] ENGINE Bus STARTED 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: mgrmap e18: y(active, since 1.01067s), standbys: x 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: from='client.24349 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi027=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: Saving service grafana spec with placement smithi027=a;count:1 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:30 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:30.748 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[100189]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[118128]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:31 smithi027 ceph-mon[105164]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.629 INFO:teuthology.orchestra.run.smithi017.stdout:[client.0] 2023-09-23T14:55:32.629 INFO:teuthology.orchestra.run.smithi017.stdout: key = AQBk/A5lW547JRAASq3pkt2jH5f7T1b9bY9hPQ== 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:32.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T14:55:32.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: mgrmap e19: y(active, since 3s), standbys: x 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:55:32.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:32.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:55:32.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:55:33.382 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:55:33.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T14:55:33.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T14:55:33.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T14:55:33.456 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:33 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:55:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 4036M 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3808422480' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 4036M 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3808422480' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:33 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.784 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T14:55:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 4036M 2023-09-23T14:55:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:55:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:55:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3808422480' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:33.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:33 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:34 smithi017 ceph-mon[100189]: Deploying daemon alertmanager.a on smithi017 2023-09-23T14:55:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:34 smithi017 ceph-mon[118128]: Deploying daemon alertmanager.a on smithi017 2023-09-23T14:55:34.988 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:34 smithi027 ceph-mon[105164]: Deploying daemon alertmanager.a on smithi017 2023-09-23T14:55:35.473 INFO:teuthology.orchestra.run.smithi027.stdout:[client.1] 2023-09-23T14:55:35.473 INFO:teuthology.orchestra.run.smithi027.stdout: key = AQBn/A5lrSX/GxAAszZq6l0crccZKrCe8AFm6w== 2023-09-23T14:55:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:35 smithi027 ceph-mon[105164]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:35 smithi027 ceph-mon[105164]: from='client.? 172.21.15.27:0/3322574117' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:35 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:35 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[100189]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[100189]: from='client.? 172.21.15.27:0/3322574117' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[118128]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[118128]: from='client.? 172.21.15.27:0/3322574117' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T14:55:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:35 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T14:55:35.999 DEBUG:teuthology.orchestra.run.smithi027:> set -ex 2023-09-23T14:55:36.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T14:55:36.000 DEBUG:teuthology.orchestra.run.smithi027:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T14:55:36.081 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T14:55:36.081 DEBUG:teuthology.orchestra.run.smithi017:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T14:55:36.421 INFO:teuthology.misc.health.smithi017.stdout: 2023-09-23T14:55:36.421 INFO:teuthology.misc.health.smithi017.stdout:{"epoch":60,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:55:28.581654+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:55:36.434 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T14:55:36.434 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T14:55:36.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T14:55:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:36 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/625963517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:36 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/625963517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:36 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/625963517' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:37.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:37.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:37 smithi017 ceph-mon[100189]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:37 smithi017 ceph-mon[118128]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:37 smithi027 ceph-mon[105164]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:38.917 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:38 smithi017 ceph-mon[118128]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:38 smithi017 ceph-mon[100189]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:38 smithi027 ceph-mon[105164]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:41.526 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:41 smithi017 systemd[1]: Starting Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:55:41.662 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'rbd' created 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[100189]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3084921295' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[118128]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3084921295' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:41 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:41 smithi027 ceph-mon[105164]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:41 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3084921295' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:41 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T14:55:42.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T14:55:42.751 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:42.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:42 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T14:55:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:42 smithi017 ceph-mon[100189]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T14:55:42.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:42 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T14:55:42.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:42 smithi017 ceph-mon[118128]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T14:55:42.943 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.635Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T14:55:42.943 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.635Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T14:55:42.943 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.636Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.637Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 bash[164444]: a2571ef49f8f1800c4ba604d5135cfa387f0ad2fec1b9cf999f3c65fd806dbea 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 systemd[1]: Started Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.669Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.669Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.673Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T14:55:42.944 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:42 smithi017 conmon[164514]: ts=2023-09-23T14:55:42.673Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T14:55:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:42 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T14:55:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:42 smithi027 ceph-mon[105164]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T14:55:43.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:43 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:55:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:43 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: pgmap v11: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T14:55:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:43 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.443 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:44 smithi017 bash[165225]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T14:55:44.677 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T14:55:44.835 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:44 smithi017 conmon[164514]: ts=2023-09-23T14:55:44.638Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.001029092s 2023-09-23T14:55:44.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[100189]: Deploying daemon node-exporter.a on smithi017 2023-09-23T14:55:44.835 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:44.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[100189]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T14:55:44.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:55:44.836 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:44 smithi017 bash[165225]: Getting image source signatures 2023-09-23T14:55:44.836 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:44 smithi017 bash[165225]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T14:55:44.836 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:44 smithi017 bash[165225]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T14:55:44.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[118128]: Deploying daemon node-exporter.a on smithi017 2023-09-23T14:55:44.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:44.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[118128]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T14:55:44.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:44.837 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:44 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:55:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:44 smithi027 ceph-mon[105164]: Deploying daemon node-exporter.a on smithi017 2023-09-23T14:55:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:44 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:44 smithi027 ceph-mon[105164]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T14:55:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:44 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:44 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T14:55:45.192 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:44 smithi017 bash[165225]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T14:55:45.595 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T14:55:45.595 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T14:55:45.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T14:55:45.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[100189]: pgmap v14: 9 pgs: 5 unknown, 4 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:45.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:55:45.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[100189]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T14:55:45.943 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:45 smithi017 bash[165225]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T14:55:45.943 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:45 smithi017 bash[165225]: Writing manifest to image destination 2023-09-23T14:55:45.944 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:45 smithi017 bash[165225]: Storing signatures 2023-09-23T14:55:45.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[118128]: pgmap v14: 9 pgs: 5 unknown, 4 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:45.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:55:45.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:45 smithi017 ceph-mon[118128]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T14:55:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:45 smithi027 ceph-mon[105164]: pgmap v14: 9 pgs: 5 unknown, 4 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:45 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1289743208' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T14:55:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:45 smithi027 ceph-mon[105164]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T14:55:46.474 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:47.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.975Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T14:55:47.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.975Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T14:55:47.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.976Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.976Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.976Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.976Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T14:55:47.194 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T14:55:47.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T14:55:47.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 conmon[165585]: ts=2023-09-23T14:55:46.977Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T14:55:47.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 bash[165225]: af3d2ae00db6e819f00ca79cba753cfb702212344d6824572e86a26aac08fc1d 2023-09-23T14:55:47.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:55:46 smithi017 systemd[1]: Started Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:55:47.745 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:47 smithi027 systemd[1]: Starting Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:55:47.746 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:47 smithi027 ceph-mon[105164]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:47.746 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:47 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.746 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:47 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.746 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:47 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[118128]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[100189]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:47 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:48.350 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T14:55:48.551 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:55:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:48 smithi027 ceph-mon[105164]: Deploying daemon node-exporter.b on smithi027 2023-09-23T14:55:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1163625373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:55:48.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Getting image source signatures 2023-09-23T14:55:48.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T14:55:48.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T14:55:48.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T14:55:48.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:48 smithi017 ceph-mon[100189]: Deploying daemon node-exporter.b on smithi027 2023-09-23T14:55:48.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1163625373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:55:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:48 smithi017 ceph-mon[118128]: Deploying daemon node-exporter.b on smithi027 2023-09-23T14:55:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1163625373' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:55:49.169 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":19,"active_gid":14463,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":625940860},{"type":"v1","addr":"172.21.15.17:6801","nonce":625940860}]},"active_addr":"172.21.15.17:6801/625940860","active_change":"2023-09-23T14:55:28.581764+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24343,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":774095134}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1851970177}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2135416630}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2670521468}]}]} 2023-09-23T14:55:49.173 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T14:55:49.174 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T14:55:49.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T14:55:49.349 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:48 smithi027 bash[151646]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T14:55:49.349 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:49 smithi027 bash[151646]: Writing manifest to image destination 2023-09-23T14:55:49.349 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:49 smithi027 bash[151646]: Storing signatures 2023-09-23T14:55:49.532 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:49.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:49 smithi017 ceph-mon[100189]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:49 smithi017 ceph-mon[118128]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:49 smithi027 ceph-mon[105164]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.426Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.426Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.427Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.427Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.427Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.427Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T14:55:50.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T14:55:50.851 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T14:55:50.852 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T14:55:50.853 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.428Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.429Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 conmon[151722]: ts=2023-09-23T14:55:50.429Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 bash[151646]: 67db5f1f10b695fbe8a6afd31e313e5f6dbdf2c904b46d84a3055adfe89be2e8 2023-09-23T14:55:50.854 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:55:50 smithi027 systemd[1]: Started Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:55:50.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE Bus STOPPING 2023-09-23T14:55:50.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:55:50.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE Bus STOPPED 2023-09-23T14:55:50.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE Bus STARTING 2023-09-23T14:55:50.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE Serving on http://:::9283 2023-09-23T14:55:50.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:55:50 smithi017 conmon[100620]: [23/Sep/2023:14:55:50] ENGINE Bus STARTED 2023-09-23T14:55:51.033 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:55:51.034 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":64,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:55:44.669191+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:55:51.619 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T14:55:51.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T14:55:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: Deploying daemon grafana.a on smithi027 2023-09-23T14:55:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:51 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/131678722' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: Deploying daemon grafana.a on smithi027 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/131678722' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:51.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: Deploying daemon grafana.a on smithi027 2023-09-23T14:55:51.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:51 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/131678722' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:51.982 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:52.877 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:55:52 smithi017 conmon[164514]: ts=2023-09-23T14:55:52.642Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.005116175s 2023-09-23T14:55:53.502 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:55:53.502 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":64,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:55:44.669191+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:55:53.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:55:53 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:55:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:55:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:53 smithi017 ceph-mon[100189]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:53 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2579871116' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:53 smithi017 ceph-mon[118128]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:53 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2579871116' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:54.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T14:55:54.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T14:55:54.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T14:55:54.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T14:55:54.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T14:55:54.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T14:55:54.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T14:55:54.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T14:55:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:53 smithi027 ceph-mon[105164]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:53 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2579871116' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:55:54.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:54 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:54.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:54 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:54 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:55.540 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:55.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:55.653 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:55.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:55.815 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:55.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:55 smithi017 ceph-mon[100189]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:55.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:55 smithi017 ceph-mon[118128]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:56.016 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:56.018 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:56.022 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:55:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:55 smithi027 ceph-mon[105164]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:57.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:57 smithi017 ceph-mon[100189]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:57 smithi017 ceph-mon[118128]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:57 smithi027 ceph-mon[105164]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:59.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[100189]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:59.047 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:59.047 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:59.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[118128]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:59.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:59.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:55:58 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:55:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:58 smithi027 ceph-mon[105164]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:55:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:58 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:55:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:55:58 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:01.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:01 smithi017 ceph-mon[100189]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:01.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:01 smithi017 ceph-mon[118128]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:01.987 INFO:teuthology.orchestra.run.smithi017.stdout:77309411364 2023-09-23T14:56:01.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T14:56:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:01 smithi027 ceph-mon[105164]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:03.326 INFO:teuthology.orchestra.run.smithi017.stdout:55834574889 2023-09-23T14:56:03.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T14:56:03.470 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:56:03 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:56:03.552 INFO:teuthology.orchestra.run.smithi017.stdout:236223201295 2023-09-23T14:56:03.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T14:56:03.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:03 smithi027 ceph-mon[105164]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:03.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:03 smithi017 ceph-mon[100189]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:03 smithi017 ceph-mon[118128]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:05.008 INFO:teuthology.orchestra.run.smithi017.stdout:111669149728 2023-09-23T14:56:05.008 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T14:56:05.047 INFO:teuthology.orchestra.run.smithi017.stdout:201863462931 2023-09-23T14:56:05.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T14:56:05.078 INFO:teuthology.orchestra.run.smithi017.stdout:167503724567 2023-09-23T14:56:05.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T14:56:05.519 INFO:teuthology.orchestra.run.smithi017.stdout:133143986204 2023-09-23T14:56:05.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T14:56:05.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:05 smithi017 ceph-mon[100189]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:05.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:05 smithi017 ceph-mon[118128]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:05.791 INFO:teuthology.orchestra.run.smithi017.stdout:34359738414 2023-09-23T14:56:05.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T14:56:05.909 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:05.980 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:06.028 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:05 smithi027 ceph-mon[105164]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:06.108 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:06.335 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:06.336 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:06.593 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:07 smithi017 ceph-mon[100189]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:07 smithi017 ceph-mon[118128]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:07 smithi027 ceph-mon[105164]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:08.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:08 smithi017 ceph-mon[100189]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:08.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:08 smithi017 ceph-mon[118128]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:09.010 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:09.348 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:08 smithi027 ceph-mon[105164]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:10.102 INFO:teuthology.orchestra.run.smithi017.stdout:55834574890 2023-09-23T14:56:10.431 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:10 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3144974132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:56:10.432 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:10 smithi027 systemd[1]: Starting Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:56:10.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:10 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3144974132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:56:10.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:10 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3144974132' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:56:10.878 INFO:teuthology.orchestra.run.smithi017.stdout:236223201296 2023-09-23T14:56:11.265 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:11.265 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3469351422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:56:11.266 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.266 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.266 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.266 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:11 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.266 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:10 smithi027 bash[152393]: dec828213828f457ad587ecd0199a0fcf8a5af9922e54a095be3c5ff67830dce 2023-09-23T14:56:11.266 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:10 smithi027 systemd[1]: Started Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:56:11.266 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135205963Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135441189Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135466321Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135477429Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135487357Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.13549728Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135506356Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135515934Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.13552529Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135534438Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135543359Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T14:56:11.267 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135552392Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135561423Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=settings t=2023-09-23T14:56:11.135570832Z level=info msg="App mode production" 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=sqlstore t=2023-09-23T14:56:11.135633323Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=sqlstore t=2023-09-23T14:56:11.135656659Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.136658861Z level=info msg="Starting DB migrations" 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.136754074Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.181954883Z level=info msg="Executing migration" id="create user table" 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.223862832Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T14:56:11.268 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.265797998Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T14:56:11.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3469351422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3469351422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:11 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.333198287Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.3583166Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.400216506Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.417009796Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.450611426Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.467370284Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T14:56:11.543 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.509454997Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T14:56:11.544 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.54301772Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T14:56:11.811 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.585165931Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T14:56:11.811 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.627083213Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T14:56:11.812 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.65227304Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T14:56:11.812 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.694162488Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T14:56:11.812 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.736197524Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T14:56:11.812 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.778056278Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T14:56:11.812 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.811861082Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T14:56:12.080 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.870556199Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T14:56:12.080 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.946604223Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T14:56:12.080 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:11 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:11.979649401Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T14:56:12.217 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:12 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:12.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:12 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:12.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:12 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:12.349 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.08025788Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T14:56:12.349 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.097148927Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T14:56:12.350 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.172326589Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T14:56:12.350 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.247801723Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T14:56:12.350 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.306413125Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T14:56:12.549 INFO:teuthology.orchestra.run.smithi017.stdout:201863462933 2023-09-23T14:56:12.634 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.356692217Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T14:56:12.634 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.398867051Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T14:56:12.634 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.415658615Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T14:56:12.634 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.432496306Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.449378616Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.466223343Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.49978691Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.516625052Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.541939709Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.567139567Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.583942106Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.60074357Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T14:56:12.635 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.634354574Z level=info msg="Executing migration" id="create star table" 2023-09-23T14:56:12.725 INFO:teuthology.orchestra.run.smithi017.stdout:167503724569 2023-09-23T14:56:12.919 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.668249115Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T14:56:12.919 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.701797446Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T14:56:12.919 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.718611485Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T14:56:12.920 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.760621068Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T14:56:12.920 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.785820219Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T14:56:12.920 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.855006672Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T14:56:13.230 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.919617806Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T14:56:13.230 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:12 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:12.962137678Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.004276773Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.071328185Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.09647217Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.163086582Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:56:13.231 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3342479470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:56:13.231 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:13 smithi027 ceph-mon[105164]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:13.232 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2540900951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:56:13.232 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/138578732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:56:13.440 INFO:teuthology.orchestra.run.smithi017.stdout:111669149730 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3342479470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[100189]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2540900951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/138578732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3342479470' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[118128]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2540900951' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:56:13.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/138578732' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:56:13.469 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574889 got 55834574890 for osd.1 2023-09-23T14:56:13.469 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:13.540 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.230754901Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.280949177Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.297880207Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.314629645Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.348237229Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.373503378Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.398696519Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.423870311Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.474077249Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T14:56:13.541 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.541005779Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T14:56:13.666 INFO:teuthology.orchestra.run.smithi017.stdout:133143986206 2023-09-23T14:56:13.809 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.566559774Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T14:56:13.809 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.616964384Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T14:56:13.809 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.704904256Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T14:56:13.810 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.759237252Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T14:56:13.810 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.80952017Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T14:56:14.099 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.83479518Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T14:56:14.099 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:13 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:13.876710733Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T14:56:14.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.043691399Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T14:56:14.363 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:14 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:14.363 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:14 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/713135344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:56:14.363 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:14 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:14.363 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.110973967Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.127961235Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.169966579Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.237282427Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.254103728Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.271475451Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.287813056Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.304650523Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T14:56:14.364 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.363491364Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/713135344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/713135344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:56:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:14 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:14.615 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.437411319Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T14:56:14.615 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.481257544Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T14:56:14.615 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.514855765Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T14:56:14.615 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.548275323Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T14:56:14.615 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.57355237Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T14:56:14.616 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.615645519Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T14:56:14.841 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724567 got 167503724569 for osd.5 2023-09-23T14:56:14.841 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:14.883 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.657527902Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T14:56:14.883 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.699300132Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T14:56:14.883 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.724576073Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T14:56:14.884 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.766481621Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T14:56:14.884 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.791563554Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T14:56:14.884 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.833586803Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T14:56:14.884 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.858733618Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T14:56:14.884 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.883894911Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T14:56:15.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.926096198Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.942916952Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.976429745Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:14 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:14.993340853Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.010187155Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.043942371Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.060601724Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.077593275Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.094433635Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.11135644Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.128049723Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T14:56:15.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.144969092Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T14:56:15.182 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201296 for osd.7 2023-09-23T14:56:15.182 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:15.423 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.23084311Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.256105127Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.289575178Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.30663943Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.323314031Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.340237905Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.357060709Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.373953081Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.390894183Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T14:56:15.424 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.4242451Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T14:56:15.477 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462933 for osd.6 2023-09-23T14:56:15.477 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:15.618 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149728 got 111669149730 for osd.3 2023-09-23T14:56:15.618 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:15.680 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986204 got 133143986206 for osd.4 2023-09-23T14:56:15.680 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:15.692 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:15 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.692 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:15 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.692 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:15 smithi027 ceph-mon[105164]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:15.692 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.466369112Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T14:56:15.692 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.49173166Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T14:56:15.692 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.516890521Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T14:56:15.693 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.558625021Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T14:56:15.693 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.583766288Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T14:56:15.693 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.609018792Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T14:56:15.693 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.650839466Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T14:56:15.693 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.692805141Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T14:56:15.713 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.713 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.713 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[100189]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:15.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:15.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:15 smithi017 ceph-mon[118128]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:15.869 INFO:teuthology.orchestra.run.smithi017.stdout:77309411367 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.73488534Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.760296637Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.785309329Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.827361266Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.869262339Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.894339776Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.91944433Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T14:56:15.961 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.961380415Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T14:56:16.238 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:15 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:15.995017115Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T14:56:16.238 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.020177212Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T14:56:16.238 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.053651802Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T14:56:16.238 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.070619767Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.087384885Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.10426843Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.121128333Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.137963588Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.154795667Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.188444346Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.205468192Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T14:56:16.239 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.238769386Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T14:56:16.433 INFO:teuthology.orchestra.run.smithi017.stdout:34359738417 2023-09-23T14:56:16.499 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.289613679Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T14:56:16.499 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.306559119Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T14:56:16.499 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.331636802Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T14:56:16.499 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.356946411Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T14:56:16.499 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.382095676Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T14:56:16.500 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.424015321Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T14:56:16.500 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.449037024Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T14:56:16.500 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.474316609Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T14:56:16.500 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.499650273Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T14:56:16.767 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:16 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2462146870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:56:16.767 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:16 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3601507019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.524787561Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.566858591Z level=info msg="Executing migration" id="create session table" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.591988619Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.633931572Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.675830837Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.700824565Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T14:56:16.768 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.767905513Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T14:56:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:16 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2462146870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:56:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:16 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3601507019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:56:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:16 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2462146870' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:56:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:16 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3601507019' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.80993455Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.851905853Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.877008037Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.902323285Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.927461139Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.952620844Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:16 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:16.994459875Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T14:56:17.036 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.036316555Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.061593552Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.0867811Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.111985935Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.153905348Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.195748491Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.220847158Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.24616555Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T14:56:17.296 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.271229471Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T14:56:17.360 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411364 got 77309411367 for osd.2 2023-09-23T14:56:17.360 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:17.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738414 got 34359738417 for osd.0 2023-09-23T14:56:17.401 DEBUG:teuthology.parallel:result is None 2023-09-23T14:56:17.401 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T14:56:17.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T14:56:17.547 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:17 smithi027 ceph-mon[105164]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.296384608Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.346980443Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.38867415Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.430765037Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.455955076Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.481207993Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.506359181Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T14:56:17.548 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.548196644Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T14:56:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:17 smithi017 ceph-mon[100189]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:17.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:17 smithi017 ceph-mon[118128]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:17.799 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.598809754Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T14:56:17.799 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.62399279Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T14:56:17.799 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.665700014Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T14:56:17.799 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.707676616Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T14:56:17.800 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.749592658Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T14:56:17.800 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.774890387Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T14:56:17.800 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.799834382Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T14:56:17.818 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.84195489Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.883759224Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.909015184Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.950969927Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:17 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:17.975898321Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.00115147Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.026363365Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T14:56:18.076 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.07650991Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T14:56:18.327 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.126892634Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T14:56:18.327 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.177219596Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T14:56:18.327 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.219125144Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T14:56:18.327 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.244283946Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T14:56:18.587 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.327593769Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.353064304Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.378091417Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.403415727Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.428530211Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.453575052Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.470519452Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.487358038Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.512665508Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.537667811Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T14:56:18.588 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.562796911Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T14:56:18.589 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.588053849Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.630171796Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.655456305Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.697205175Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.722469511Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.772856973Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.798096155Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T14:56:18.840 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.839970048Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T14:56:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:18 smithi027 ceph-mon[105164]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.865233443Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.882249778Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.898889343Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.915858602Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.949374369Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.982993221Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:18 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:18.999863137Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.016786013Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.033495481Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.050436619Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.067309777Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T14:56:19.100 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.084091107Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T14:56:19.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:18 smithi017 ceph-mon[100189]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:18 smithi017 ceph-mon[118128]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.126267951Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.151489065Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.193447971Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.218611782Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.24367476Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.268822184Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.310729105Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T14:56:19.386 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.335857557Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T14:56:19.387 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.361275535Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T14:56:19.387 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.386368282Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T14:56:19.646 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.428428258Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T14:56:19.646 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.445258237Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T14:56:19.646 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.462159189Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.47898022Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.495825071Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.529232621Z level=info msg="Executing migration" id="create team table" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.546348697Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.571342162Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.613382441Z level=info msg="Executing migration" id="create team member table" 2023-09-23T14:56:19.647 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.646801592Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T14:56:19.713 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:19.714 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.689004268Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.731265459Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.756464379Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.818991041Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.848468212Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T14:56:19.915 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.890415594Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T14:56:19.916 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.915499873Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T14:56:20.192 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.940674706Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T14:56:20.192 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:19 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:19.982514364Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T14:56:20.192 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.007726215Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T14:56:20.192 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.049787964Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T14:56:20.193 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.07493718Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T14:56:20.193 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.100002073Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T14:56:20.193 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.125228599Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T14:56:20.193 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.15043539Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T14:56:20.193 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.192359248Z level=info msg="Executing migration" id="create tag table" 2023-09-23T14:56:20.443 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.24271772Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T14:56:20.443 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.267718579Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T14:56:20.443 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.293052981Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T14:56:20.443 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.318114461Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T14:56:20.444 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.368477534Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T14:56:20.444 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.393484175Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T14:56:20.444 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.418606411Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T14:56:20.444 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.443744857Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T14:56:20.471 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":32,"stamp":"2023-09-23T14:56:18.603091+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50424,"kb_used_data":6712,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910792,"statfs":{"total":767960285184,"available":767908651008,"internally_reserved":0,"allocated":6873088,"data_stored":3897883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002260"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.858823+0000","last_change":"2023-09-23T14:55:42.668313+0000","last_active":"2023-09-23T14:55:43.858823+0000","last_peered":"2023-09-23T14:55:43.858823+0000","last_clean":"2023-09-23T14:55:43.858823+0000","last_became_active":"2023-09-23T14:55:42.667931+0000","last_became_peered":"2023-09-23T14:55:42.667931+0000","last_unstale":"2023-09-23T14:55:43.858823+0000","last_undegraded":"2023-09-23T14:55:43.858823+0000","last_fullsized":"2023-09-23T14:55:43.858823+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:11:42.832634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666255+0000","last_change":"2023-09-23T14:55:42.887554+0000","last_active":"2023-09-23T14:55:43.666255+0000","last_peered":"2023-09-23T14:55:43.666255+0000","last_clean":"2023-09-23T14:55:43.666255+0000","last_became_active":"2023-09-23T14:55:42.887232+0000","last_became_peered":"2023-09-23T14:55:42.887232+0000","last_unstale":"2023-09-23T14:55:43.666255+0000","last_undegraded":"2023-09-23T14:55:43.666255+0000","last_fullsized":"2023-09-23T14:55:43.666255+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:19.963176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859063+0000","last_change":"2023-09-23T14:55:42.887686+0000","last_active":"2023-09-23T14:55:43.859063+0000","last_peered":"2023-09-23T14:55:43.859063+0000","last_clean":"2023-09-23T14:55:43.859063+0000","last_became_active":"2023-09-23T14:55:42.887225+0000","last_became_peered":"2023-09-23T14:55:42.887225+0000","last_unstale":"2023-09-23T14:55:43.859063+0000","last_undegraded":"2023-09-23T14:55:43.859063+0000","last_fullsized":"2023-09-23T14:55:43.859063+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:32.886192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859008+0000","last_change":"2023-09-23T14:55:42.667453+0000","last_active":"2023-09-23T14:55:43.859008+0000","last_peered":"2023-09-23T14:55:43.859008+0000","last_clean":"2023-09-23T14:55:43.859008+0000","last_became_active":"2023-09-23T14:55:42.667131+0000","last_became_peered":"2023-09-23T14:55:42.667131+0000","last_unstale":"2023-09-23T14:55:43.859008+0000","last_undegraded":"2023-09-23T14:55:43.859008+0000","last_fullsized":"2023-09-23T14:55:43.859008+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:20.524459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859020+0000","last_change":"2023-09-23T14:54:58.918703+0000","last_active":"2023-09-23T14:55:43.859020+0000","last_peered":"2023-09-23T14:55:43.859020+0000","last_clean":"2023-09-23T14:55:43.859020+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T14:55:43.859020+0000","last_undegraded":"2023-09-23T14:55:43.859020+0000","last_fullsized":"2023-09-23T14:55:43.859020+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:41.896866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666646+0000","last_change":"2023-09-23T14:55:42.668315+0000","last_active":"2023-09-23T14:55:43.666646+0000","last_peered":"2023-09-23T14:55:43.666646+0000","last_clean":"2023-09-23T14:55:43.666646+0000","last_became_active":"2023-09-23T14:55:42.667972+0000","last_became_peered":"2023-09-23T14:55:42.667972+0000","last_unstale":"2023-09-23T14:55:43.666646+0000","last_undegraded":"2023-09-23T14:55:43.666646+0000","last_fullsized":"2023-09-23T14:55:43.666646+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.858345+0000","last_change":"2023-09-23T14:55:42.670066+0000","last_active":"2023-09-23T14:55:43.858345+0000","last_peered":"2023-09-23T14:55:43.858345+0000","last_clean":"2023-09-23T14:55:43.858345+0000","last_became_active":"2023-09-23T14:55:42.669743+0000","last_became_peered":"2023-09-23T14:55:42.669743+0000","last_unstale":"2023-09-23T14:55:43.858345+0000","last_undegraded":"2023-09-23T14:55:43.858345+0000","last_fullsized":"2023-09-23T14:55:43.858345+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:17.259363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666643+0000","last_change":"2023-09-23T14:55:42.666173+0000","last_active":"2023-09-23T14:55:43.666643+0000","last_peered":"2023-09-23T14:55:43.666643+0000","last_clean":"2023-09-23T14:55:43.666643+0000","last_became_active":"2023-09-23T14:55:42.665827+0000","last_became_peered":"2023-09-23T14:55:42.665827+0000","last_unstale":"2023-09-23T14:55:43.666643+0000","last_undegraded":"2023-09-23T14:55:43.666643+0000","last_fullsized":"2023-09-23T14:55:43.666643+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:29.577101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666318+0000","last_change":"2023-09-23T14:55:42.665607+0000","last_active":"2023-09-23T14:55:43.666318+0000","last_peered":"2023-09-23T14:55:43.666318+0000","last_clean":"2023-09-23T14:55:43.666318+0000","last_became_active":"2023-09-23T14:55:42.665279+0000","last_became_peered":"2023-09-23T14:55:42.665279+0000","last_unstale":"2023-09-23T14:55:43.666318+0000","last_undegraded":"2023-09-23T14:55:43.666318+0000","last_fullsized":"2023-09-23T14:55:43.666318+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:58.018948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201298,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.48999999999999999}]},{"osd":2,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.69999999999999996}]},{"osd":4,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.71199999999999997}]},{"osd":5,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.63}]}]},{"osd":6,"up_from":47,"seq":201863462934,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6392,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738760,"statfs":{"total":95995035648,"available":95988490240,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.45300000000000001}]},{"osd":4,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.51800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574892,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6516,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738636,"statfs":{"total":95995035648,"available":95988363264,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.41699999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36199999999999999,"15min":0.34999999999999998},"min":{"1min":0.25900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55200000000000005,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 14:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.41799999999999998,"15min":0.42799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.48299999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.443},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41899999999999998,"15min":0.42099999999999999},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.4470000000000001,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53400000000000003,"15min":0.54500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68600000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.48099999999999998,"15min":0.49099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.621,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 14:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.60699999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 14:55:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.65800000000000003,"5min":0.65800000000000003,"15min":0.65800000000000003},"last":0.42599999999999999}]},{"osd":6,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.51700000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.61199999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738417,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7164,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737988,"statfs":{"total":95995035648,"available":95987699712,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 14:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.48599999999999999,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58099999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.32700000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.54800000000000004,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61299999999999999,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.41499999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.50800000000000001,"15min":0.51200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.46999999999999997}]},{"osd":4,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60499999999999998,"15min":0.60799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61099999999999999,"15min":0.61599999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 14:55:15 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.625,"5min":0.625,"15min":0.625},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.63}]},{"osd":7,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.54400000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411367,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6456,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738696,"statfs":{"total":95995035648,"available":95988424704,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.47799999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.375,"5min":0.437,"15min":0.443},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.49399999999999999,"5min":0.748,"15min":0.748},"last":0.42699999999999999}]},{"osd":1,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41299999999999998,"15min":0.42699999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51900000000000002,"5min":0.627,"15min":0.627},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46500000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65300000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.51700000000000002,"15min":0.53300000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65200000000000002,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56299999999999994,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79300000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.42999999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73599999999999999,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.71099999999999997}]},{"osd":5,"last update":"Sat Sep 23 14:55:14 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 14:55:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.755}]},{"osd":7,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.79600000000000004}]}]},{"osd":3,"up_from":26,"seq":111669149731,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.376,"15min":0.38100000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.49399999999999999,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.436,"15min":0.44500000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.505},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.44700000000000001,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.48799999999999999,"15min":0.496},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69599999999999995,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.47799999999999998,"15min":0.48699999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64600000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 14:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.59899999999999998,"15min":0.61399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79500000000000004,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59399999999999997,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.59199999999999997}]},{"osd":5,"last update":"Sat Sep 23 14:56:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57699999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.67600000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.59399999999999997,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67500000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.46300000000000002}]},{"osd":6,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 14:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.51700000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986207,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5876,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739276,"statfs":{"total":95995035648,"available":95989018624,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.498,"15min":0.495},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.79500000000000004,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.371},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.60499999999999998,"15min":0.56299999999999994},"min":{"1min":0.317,"5min":0.224,"15min":0.224},"max":{"1min":6.5570000000000004,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63500000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.50600000000000001,"15min":0.498},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.33400000000000002}]},{"osd":2,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.439},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.38400000000000001}]},{"osd":3,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77300000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53000000000000003,"15min":0.52500000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.35799999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:55:13 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.5}]}]},{"osd":5,"up_from":39,"seq":167503724570,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5812,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739340,"statfs":{"total":95995035648,"available":95989084160,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53400000000000003}]},{"osd":1,"last update":"Sat Sep 23 14:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.58699999999999997}]},{"osd":3,"last update":"Sat Sep 23 14:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.66800000000000004}]},{"osd":4,"last update":"Sat Sep 23 14:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.46100000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T14:56:20.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T14:56:20.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.713 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[118128]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.714 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:20.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:20.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:20 smithi017 ceph-mon[100189]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.486023557Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.519590409Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.552885972Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.60332023Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.636655515Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.686844761Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T14:56:20.720 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.720517956Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:20.721 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:20 smithi027 ceph-mon[105164]: from='client.24527 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.762510175Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.787646468Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.804550916Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.829711529Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.871653417Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T14:56:20.972 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.888507987Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T14:56:20.973 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.905444095Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T14:56:20.973 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.922201611Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T14:56:20.973 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.955782867Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T14:56:20.973 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:20 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:20.972583615Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.015026674Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.040294724Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.082055768Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.107186949Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.132383238Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.157455359Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T14:56:21.224 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.199368945Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T14:56:21.225 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.224535951Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.258360944Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.283442769Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.30846641Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.38374381Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.408951088Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.434299646Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.459280754Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T14:56:21.484 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.484493636Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T14:56:21.503 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:21.752 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:21 smithi027 ceph-mon[105164]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:56:21.752 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:21 smithi027 ceph-mon[105164]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:56:21.752 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:21 smithi027 ceph-mon[105164]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:21.752 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.534865297Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T14:56:21.753 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.576834243Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T14:56:21.753 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.618700218Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T14:56:21.753 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.668814844Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T14:56:21.753 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.710657908Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T14:56:21.753 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.752638004Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T14:56:21.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[100189]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:56:21.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[100189]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:56:21.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[100189]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:21.815 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[118128]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:56:21.815 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[118128]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:56:21.815 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:21 smithi017 ceph-mon[118128]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.803076201Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.828289159Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.853616911Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.895512188Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.920468066Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.945708346Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.970974905Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:21 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:21.996021513Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T14:56:22.021 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.021293959Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T14:56:22.272 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.063311156Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T14:56:22.272 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.088567182Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T14:56:22.273 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.130392625Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T14:56:22.273 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.155481884Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T14:56:22.273 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.197506951Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T14:56:22.273 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.239484006Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T14:56:22.273 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.273012656Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T14:56:22.532 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.323321554Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T14:56:22.533 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.373600295Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T14:56:22.533 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.407096863Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T14:56:22.533 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.432253761Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T14:56:22.533 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.457321425Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T14:56:22.533 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.482527513Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T14:56:22.534 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.507674053Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T14:56:22.534 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.532961985Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T14:56:22.784 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.58344976Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T14:56:22.784 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.608515992Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T14:56:22.784 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.650301323Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T14:56:22.785 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.692222672Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T14:56:22.785 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.734276465Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T14:56:22.785 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.759520591Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T14:56:22.785 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.784595396Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T14:56:23.052 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.826661762Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T14:56:23.052 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.886777953Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T14:56:23.052 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.927160541Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T14:56:23.053 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.952327848Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T14:56:23.053 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:22 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:22.977471066Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T14:56:23.053 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.002523838Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T14:56:23.053 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.027761286Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T14:56:23.053 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.052931649Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T14:56:23.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:22 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:56:23.313 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.103292201Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.145261999Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.187186462Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.204161575Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.220818532Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.237680825Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.254659833Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T14:56:23.313 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.279656504Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T14:56:23.314 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.29651914Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T14:56:23.314 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.313378243Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T14:56:23.582 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.363939887Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T14:56:23.582 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.380711911Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T14:56:23.582 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.39768844Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T14:56:23.582 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.414578648Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T14:56:23.582 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.431575035Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.43173944Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.456681656Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.482026572Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.507146758Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.50736479Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.507540569Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.532357924Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.557419424Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T14:56:23.583 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.582604451Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T14:56:23.618 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:23 smithi017 conmon[164514]: ts=2023-09-23T14:56:23.387Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T14:56:23.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:23 smithi027 ceph-mon[105164]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.633314949Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.675080598Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.700338211Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.725520313Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.767480485Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.792699204Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T14:56:23.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.817651389Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T14:56:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:23 smithi017 ceph-mon[100189]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:23.943 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:23 smithi017 bash[175219]: a2571ef49f8f1800c4ba604d5135cfa387f0ad2fec1b9cf999f3c65fd806dbea 2023-09-23T14:56:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:23 smithi017 ceph-mon[118128]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:24.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.87642406Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T14:56:24.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.918287198Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T14:56:24.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.960228852Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T14:56:24.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:23 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:23.99371774Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T14:56:24.144 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.027190498Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T14:56:24.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.052536309Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T14:56:24.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.094356067Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T14:56:24.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.119523839Z level=info msg="Executing migration" id="create permission table" 2023-09-23T14:56:24.145 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.144535932Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T14:56:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:24.382 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.195111642Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.220310671Z level=info msg="Executing migration" id="create role table" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.245348029Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.270469038Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.295544379Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.320795478Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.362762718Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T14:56:24.446 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.387869801Z level=info msg="Executing migration" id="create team role table" 2023-09-23T14:56:24.692 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:24 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T14:56:24.692 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:24 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:56:24.693 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:24 smithi017 systemd[1]: Starting Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:56:24.697 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.446478239Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T14:56:24.697 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.488483937Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T14:56:24.697 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.513774567Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T14:56:24.697 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.547248425Z level=info msg="Executing migration" id="create user role table" 2023-09-23T14:56:24.698 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.597430105Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T14:56:24.698 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.647584123Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T14:56:24.698 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.697935521Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T14:56:24.913 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":34,"stamp":"2023-09-23T14:56:22.603822+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50424,"kb_used_data":6712,"kb_used_omap":0,"kb_used_meta":43648,"kb_avail":749910792,"statfs":{"total":767960285184,"available":767908651008,"internally_reserved":0,"allocated":6873088,"data_stored":3897883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44695552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002116"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.858823+0000","last_change":"2023-09-23T14:55:42.668313+0000","last_active":"2023-09-23T14:55:43.858823+0000","last_peered":"2023-09-23T14:55:43.858823+0000","last_clean":"2023-09-23T14:55:43.858823+0000","last_became_active":"2023-09-23T14:55:42.667931+0000","last_became_peered":"2023-09-23T14:55:42.667931+0000","last_unstale":"2023-09-23T14:55:43.858823+0000","last_undegraded":"2023-09-23T14:55:43.858823+0000","last_fullsized":"2023-09-23T14:55:43.858823+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:11:42.832634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666255+0000","last_change":"2023-09-23T14:55:42.887554+0000","last_active":"2023-09-23T14:55:43.666255+0000","last_peered":"2023-09-23T14:55:43.666255+0000","last_clean":"2023-09-23T14:55:43.666255+0000","last_became_active":"2023-09-23T14:55:42.887232+0000","last_became_peered":"2023-09-23T14:55:42.887232+0000","last_unstale":"2023-09-23T14:55:43.666255+0000","last_undegraded":"2023-09-23T14:55:43.666255+0000","last_fullsized":"2023-09-23T14:55:43.666255+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:19.963176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859063+0000","last_change":"2023-09-23T14:55:42.887686+0000","last_active":"2023-09-23T14:55:43.859063+0000","last_peered":"2023-09-23T14:55:43.859063+0000","last_clean":"2023-09-23T14:55:43.859063+0000","last_became_active":"2023-09-23T14:55:42.887225+0000","last_became_peered":"2023-09-23T14:55:42.887225+0000","last_unstale":"2023-09-23T14:55:43.859063+0000","last_undegraded":"2023-09-23T14:55:43.859063+0000","last_fullsized":"2023-09-23T14:55:43.859063+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:32.886192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859008+0000","last_change":"2023-09-23T14:55:42.667453+0000","last_active":"2023-09-23T14:55:43.859008+0000","last_peered":"2023-09-23T14:55:43.859008+0000","last_clean":"2023-09-23T14:55:43.859008+0000","last_became_active":"2023-09-23T14:55:42.667131+0000","last_became_peered":"2023-09-23T14:55:42.667131+0000","last_unstale":"2023-09-23T14:55:43.859008+0000","last_undegraded":"2023-09-23T14:55:43.859008+0000","last_fullsized":"2023-09-23T14:55:43.859008+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:20.524459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.859020+0000","last_change":"2023-09-23T14:54:58.918703+0000","last_active":"2023-09-23T14:55:43.859020+0000","last_peered":"2023-09-23T14:55:43.859020+0000","last_clean":"2023-09-23T14:55:43.859020+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T14:55:43.859020+0000","last_undegraded":"2023-09-23T14:55:43.859020+0000","last_fullsized":"2023-09-23T14:55:43.859020+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:41.896866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666646+0000","last_change":"2023-09-23T14:55:42.668315+0000","last_active":"2023-09-23T14:55:43.666646+0000","last_peered":"2023-09-23T14:55:43.666646+0000","last_clean":"2023-09-23T14:55:43.666646+0000","last_became_active":"2023-09-23T14:55:42.667972+0000","last_became_peered":"2023-09-23T14:55:42.667972+0000","last_unstale":"2023-09-23T14:55:43.666646+0000","last_undegraded":"2023-09-23T14:55:43.666646+0000","last_fullsized":"2023-09-23T14:55:43.666646+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.858345+0000","last_change":"2023-09-23T14:55:42.670066+0000","last_active":"2023-09-23T14:55:43.858345+0000","last_peered":"2023-09-23T14:55:43.858345+0000","last_clean":"2023-09-23T14:55:43.858345+0000","last_became_active":"2023-09-23T14:55:42.669743+0000","last_became_peered":"2023-09-23T14:55:42.669743+0000","last_unstale":"2023-09-23T14:55:43.858345+0000","last_undegraded":"2023-09-23T14:55:43.858345+0000","last_fullsized":"2023-09-23T14:55:43.858345+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:17.259363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666643+0000","last_change":"2023-09-23T14:55:42.666173+0000","last_active":"2023-09-23T14:55:43.666643+0000","last_peered":"2023-09-23T14:55:43.666643+0000","last_clean":"2023-09-23T14:55:43.666643+0000","last_became_active":"2023-09-23T14:55:42.665827+0000","last_became_peered":"2023-09-23T14:55:42.665827+0000","last_unstale":"2023-09-23T14:55:43.666643+0000","last_undegraded":"2023-09-23T14:55:43.666643+0000","last_fullsized":"2023-09-23T14:55:43.666643+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:29.577101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2023-09-23T14:55:43.666318+0000","last_change":"2023-09-23T14:55:42.665607+0000","last_active":"2023-09-23T14:55:43.666318+0000","last_peered":"2023-09-23T14:55:43.666318+0000","last_clean":"2023-09-23T14:55:43.666318+0000","last_became_active":"2023-09-23T14:55:42.665279+0000","last_became_peered":"2023-09-23T14:55:42.665279+0000","last_unstale":"2023-09-23T14:55:43.666318+0000","last_undegraded":"2023-09-23T14:55:43.666318+0000","last_fullsized":"2023-09-23T14:55:43.666318+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:58.018948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201299,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6328,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738824,"statfs":{"total":95995035648,"available":95988555776,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.47999999999999998,"15min":0.47999999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.91800000000000004}]},{"osd":1,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.51300000000000001,"15min":0.51300000000000001},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.876}]},{"osd":2,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.82099999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":1.1259999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.7529999999999999,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":1},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.90000000000000002}]},{"osd":4,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.628},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.57599999999999996}]},{"osd":6,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.66200000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462935,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6392,"kb_used_data":1200,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738760,"statfs":{"total":95995035648,"available":95988490240,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.52700000000000002,"15min":0.52700000000000002},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.47799999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.45300000000000001}]},{"osd":4,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.442,"5min":0.442,"15min":0.442},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.51800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574893,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6516,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738636,"statfs":{"total":95995035648,"available":95988363264,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32800000000000001,"15min":0.31900000000000001},"min":{"1min":0.23300000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.41699999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.36199999999999999,"15min":0.34999999999999998},"min":{"1min":0.25900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.55200000000000005,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 14:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.41799999999999998,"15min":0.42799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.48299999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41899999999999998,"15min":0.42099999999999999},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.4470000000000001,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.53400000000000003,"15min":0.54500000000000004},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.68600000000000005,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.48099999999999998,"15min":0.49099999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.621,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 14:55:52 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.56299999999999994,"15min":0.57299999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.60699999999999998,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.80900000000000005}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70399999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":1.206}]},{"osd":6,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":1.224},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.61199999999999999}]},{"osd":7,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":1.175},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.64800000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738418,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7164,"kb_used_data":1204,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93737988,"statfs":{"total":95995035648,"available":95987699712,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 14:55:46 2023","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.34000000000000002,"15min":0.32600000000000001},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.48599999999999999,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.436},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.58099999999999996,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.47699999999999998}]},{"osd":2,"last update":"Sat Sep 23 14:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.54800000000000004,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.501},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61299999999999999,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.53000000000000003}]},{"osd":3,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44600000000000001,"15min":0.45000000000000001},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.50800000000000001,"15min":0.51200000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.58899999999999997}]},{"osd":4,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.60499999999999998,"15min":0.60799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.61099999999999999,"15min":0.61599999999999999},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.81799999999999995}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84299999999999997,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.45100000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89800000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.84799999999999998}]},{"osd":6,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.83199999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":1.0109999999999999,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.92800000000000005}]},{"osd":7,"last update":"Sat Sep 23 14:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.877},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.86799999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411368,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6456,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738696,"statfs":{"total":95995035648,"available":95988424704,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.47799999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.375,"5min":0.437,"15min":0.443},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.49399999999999999,"5min":0.748,"15min":0.748},"last":0.56000000000000005}]},{"osd":1,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41299999999999998,"15min":0.42699999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51900000000000002,"5min":0.627,"15min":0.627},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46500000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.47599999999999998,"15min":0.48199999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.65300000000000002,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.51700000000000002,"15min":0.53300000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.65200000000000002,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56299999999999994,"15min":0.56999999999999995},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79300000000000004,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.625,"15min":0.63500000000000001},"min":{"1min":0.42999999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.73599999999999999,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.80600000000000005,"5min":1.419,"15min":1.419},"last":0.62},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.74399999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 14:55:38 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"max":{"1min":0.86199999999999999,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.70399999999999996}]}]},{"osd":3,"up_from":26,"seq":111669149732,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5880,"kb_used_data":624,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739272,"statfs":{"total":95995035648,"available":95989014528,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34699999999999998,"5min":0.376,"15min":0.38100000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.49399999999999999,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.38100000000000001,"5min":0.436,"15min":0.44500000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"last":0.41099999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.45400000000000001,"15min":0.46200000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":0.72899999999999998,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.44700000000000001,"15min":0.45500000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.48799999999999999,"15min":0.496},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.69599999999999995,"5min":0.72399999999999998,"15min":0.72399999999999998},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.42399999999999999,"5min":0.47799999999999998,"15min":0.48699999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.64600000000000002,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.379}]},{"osd":4,"last update":"Sat Sep 23 14:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.59899999999999998,"15min":0.61399999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.79500000000000004,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59399999999999997,"15min":0.60299999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:56:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57699999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.67600000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.504,"5min":0.59399999999999997,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67500000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 14:55:33 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"min":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"max":{"1min":0.84299999999999997,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.72099999999999997,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.52800000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986208,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5876,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739276,"statfs":{"total":95995035648,"available":95989018624,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.498,"15min":0.495},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.79500000000000004,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.60499999999999998,"15min":0.56299999999999994},"min":{"1min":0.317,"5min":0.224,"15min":0.224},"max":{"1min":6.5570000000000004,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.51200000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.33200000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63500000000000001,"5min":0.89400000000000002,"15min":0.89400000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.50600000000000001,"15min":0.498},"min":{"1min":0.38700000000000001,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.498}]},{"osd":2,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.54200000000000004,"15min":0.54000000000000004},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.53600000000000003,"15min":0.53500000000000003},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.56999999999999995}]},{"osd":3,"last update":"Sat Sep 23 14:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.35899999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.77300000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.53000000000000003,"15min":0.52500000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.46800000000000003,"15min":0.46500000000000002},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.495,"5min":0.46700000000000003,"15min":0.46200000000000002},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.39200000000000002}]},{"osd":6,"last update":"Sat Sep 23 14:55:34 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":1.4610000000000001,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.437}]},{"osd":7,"last update":"Sat Sep 23 14:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.45600000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724571,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5812,"kb_used_data":620,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739340,"statfs":{"total":95995035648,"available":95989084160,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48599999999999999,"15min":0.48699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.64800000000000002,"5min":0.747,"15min":0.747},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.67900000000000005}]},{"osd":1,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.505,"15min":0.505},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.498,"5min":0.5,"15min":0.5},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67900000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.81499999999999995}]},{"osd":2,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.498,"15min":0.497},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.61399999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.82699999999999996}]},{"osd":3,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.80800000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.78700000000000003}]},{"osd":4,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.32600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72299999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45500000000000002,"15min":0.45100000000000001},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73899999999999999,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.626}]},{"osd":6,"last update":"Sat Sep 23 14:55:35 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.69699999999999995,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.755},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.76600000000000001}]},{"osd":7,"last update":"Sat Sep 23 14:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.71899999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T14:56:24.915 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T14:56:24.915 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T14:56:24.916 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T14:56:24.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T14:56:24.949 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.740105195Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T14:56:24.949 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.765293985Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T14:56:24.949 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.790400535Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.832330824Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.857385849Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.891104878Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.907994186Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.924944402Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T14:56:24.950 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:24 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:24.949958202Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T14:56:25.201 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.017179116Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.042395384Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.067566185Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.092574172Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.134350573Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.159674444Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T14:56:25.202 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.201504685Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T14:56:25.461 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.235355726Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.260304742Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.285409445Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.327412419Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.352557217Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T14:56:25.486 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.377638797Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T14:56:25.487 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.40334267Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T14:56:25.487 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.444736277Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.379Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.379Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.381Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.382Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 bash[175722]: ca695ff81f5c197ba9f57d58d9ba75ab69e147cf365e9365502c1d1c8eeb2d5a 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 systemd[1]: Started Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.416Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:56:25.659 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.416Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:56:25.660 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.421Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T14:56:25.660 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:25 smithi017 conmon[175786]: ts=2023-09-23T14:56:25.421Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T14:56:25.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[118128]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:25.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[118128]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:25.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:25.661 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:25.738 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:25 smithi027 ceph-mon[105164]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:25.738 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:25 smithi027 ceph-mon[105164]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:25.738 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:25 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:25.738 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:25 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:25.738 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.486730892Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.528811816Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.570729478Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.595969719Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.621068141Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.646241903Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.646405597Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.688140982Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.688314462Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.721820182Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T14:56:25.739 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.738646488Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T14:56:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[100189]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:56:25.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[100189]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:25.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:25.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:25 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.772430545Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.789379131Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.822911593Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.873877571Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:25 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:25.958172724Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T14:56:26.011 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.011545192Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T14:56:26.274 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.078161702Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T14:56:26.274 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.136474092Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T14:56:26.274 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.235796129Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T14:56:26.528 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.28519167Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T14:56:26.528 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.327450784Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T14:56:26.528 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.369224673Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T14:56:26.528 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.436168644Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T14:56:26.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:26 smithi017 ceph-mon[100189]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:56:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:26 smithi017 ceph-mon[100189]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:56:26.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:26 smithi017 ceph-mon[118128]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:56:26.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:26 smithi017 ceph-mon[118128]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:56:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:26 smithi027 ceph-mon[105164]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:56:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:26 smithi027 ceph-mon[105164]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:56:26.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.528241405Z level=info msg="Executing migration" id="add share column" 2023-09-23T14:56:26.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.603403296Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T14:56:26.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.694942238Z level=info msg="Executing migration" id="create file table" 2023-09-23T14:56:26.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:26 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:26.771088512Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T14:56:27.123 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:27.123 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T14:56:27.335 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.055421618Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T14:56:27.336 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.101212974Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T14:56:27.336 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.264283488Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T14:56:27.336 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.336193213Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T14:56:27.442 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:27 smithi017 conmon[175786]: ts=2023-09-23T14:56:27.383Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000324761s 2023-09-23T14:56:27.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.399478019Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T14:56:27.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.449882357Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T14:56:27.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.512190621Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T14:56:27.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.567868077Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T14:56:27.757 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T14:56:27.758 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T14:56:27.758 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T14:56:27.766 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2023-09-23T14:56:27.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T14:56:27.933 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:27 smithi027 ceph-mon[105164]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:27.933 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:27 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4279040441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:56:27.933 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.669040263Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T14:56:27.933 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.744465407Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T14:56:27.933 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.803131017Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T14:56:27.933 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.895208482Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T14:56:27.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:27 smithi017 ceph-mon[100189]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:27 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4279040441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:56:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:27 smithi017 ceph-mon[118128]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:27 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4279040441' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:56:28.127 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:28.197 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.949493389Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T14:56:28.197 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:27 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:27.996343898Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T14:56:28.197 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.038300551Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T14:56:28.197 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.105189834Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T14:56:28.197 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.19732227Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T14:56:28.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.26951804Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T14:56:28.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.342755952Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T14:56:28.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.406405384Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T14:56:28.886 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.62603115Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T14:56:28.886 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.761651603Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T14:56:28.886 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.82555546Z level=info msg="Executing migration" id="create folder table" 2023-09-23T14:56:28.886 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.886207268Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T14:56:28.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:28 smithi017 ceph-mon[118128]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:28.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:29.178 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.943308456Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T14:56:29.178 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:28 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:28.96852386Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T14:56:29.178 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:29.002022781Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T14:56:29.178 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=migrator t=2023-09-23T14:56:29.069070839Z level=info msg="migrations completed" performed=485 skipped=0 duration=17.932335781s 2023-09-23T14:56:29.179 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=sqlstore t=2023-09-23T14:56:29.069686762Z level=info msg="Created default organization" 2023-09-23T14:56:29.179 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=plugin.loader t=2023-09-23T14:56:29.178533991Z level=info msg="Plugin registered" pluginID=input 2023-09-23T14:56:29.179 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:28 smithi027 systemd[1]: Stopping Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:56:29.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:28 smithi027 ceph-mon[105164]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:29.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:28 smithi017 ceph-mon[100189]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:29.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=plugin.loader t=2023-09-23T14:56:29.209403265Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T14:56:29.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=plugin.loader t=2023-09-23T14:56:29.209439544Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T14:56:29.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=secrets t=2023-09-23T14:56:29.20948464Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T14:56:29.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=query_data t=2023-09-23T14:56:29.211588855Z level=info msg="Query Service initialization" 2023-09-23T14:56:29.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[152434]: logger=live.push_http t=2023-09-23T14:56:29.214712875Z level=info msg="Live Push Gateway initialization" 2023-09-23T14:56:29.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T14:56:29.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T14:56:29.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T14:56:29.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.184Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.185Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.185Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T14:56:29.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 conmon[148619]: ts=2023-09-23T14:56:29.185Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T14:56:30.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 bash[153984]: 3ec20d6247fb4072ff406348df60c065be1f0a4366d4c35fcd2ebb0e75935b85 2023-09-23T14:56:30.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T14:56:30.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 systemd[1]: Stopped Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:56:30.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:29 smithi027 systemd[1]: Starting Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:56:30.457 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:30 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.457 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:30 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.457 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=infra.usagestats.collector t=2023-09-23T14:56:30.139542535Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T14:56:30.457 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=provisioning.datasources t=2023-09-23T14:56:30.139971417Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T14:56:30.457 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=provisioning.datasources t=2023-09-23T14:56:30.173359773Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T14:56:30.457 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=provisioning.datasources t=2023-09-23T14:56:30.258166693Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T14:56:30.458 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=provisioning.alerting t=2023-09-23T14:56:30.345124394Z level=info msg="starting to provision alerting" 2023-09-23T14:56:30.458 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=provisioning.alerting t=2023-09-23T14:56:30.345179615Z level=info msg="finished to provision alerting" 2023-09-23T14:56:30.458 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=grafanaStorageLogger t=2023-09-23T14:56:30.345978056Z level=info msg="storage starting" 2023-09-23T14:56:30.458 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=ngalert.state.manager t=2023-09-23T14:56:30.350605764Z level=info msg="Warming state cache for startup" 2023-09-23T14:56:30.458 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=http.server t=2023-09-23T14:56:30.3531904Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T14:56:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:30 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:30 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:30 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:30 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T14:56:30.849 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=ngalert.state.manager t=2023-09-23T14:56:30.486414178Z level=info msg="State cache has been initialized" states=0 duration=135.807037ms 2023-09-23T14:56:30.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=ticker t=2023-09-23T14:56:30.486488824Z level=info msg=starting first_tick=2023-09-23T14:56:40Z 2023-09-23T14:56:30.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=ngalert.multiorg.alertmanager t=2023-09-23T14:56:30.486482023Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[152434]: logger=sqlstore.transactions t=2023-09-23T14:56:30.525719014Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.667Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.667Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.667Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi027 (none))" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.667Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T14:56:30.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.667Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 bash[154304]: 5252779692deffd7b738741a5e766d2a4e33db427d6458df93351e364e9640f7 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.669Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.670Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.670Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.671Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.672Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.672Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.5µs 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.672Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.673Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 systemd[1]: Started Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.674Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T14:56:30.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.674Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.327µs wal_replay_duration=2.039435ms wbl_replay_duration=147ns total_replay_duration=2.079313ms 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.675Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.675Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.675Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.690Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.219256ms db_storage=834ns remote_storage=1.212µs web_handler=369ns query_engine=567ns scrape=242.029µs scrape_sd=25.248µs notify=19.969µs notify_sd=6.616µs rules=13.696492ms tracing=6.213µs 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.690Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T14:56:30.852 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:56:30 smithi027 conmon[154350]: ts=2023-09-23T14:56:30.690Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T14:56:31.158 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STOPPING 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STOPPED 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STARTING 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Serving on http://:::9283 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STARTED 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STOPPING 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STOPPED 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:30 smithi017 conmon[100620]: [23/Sep/2023:14:56:30] ENGINE Bus STARTING 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Serving on http://:::9283 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Bus STARTED 2023-09-23T14:56:31.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Bus STOPPING 2023-09-23T14:56:31.160 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:56:31.160 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Bus STOPPED 2023-09-23T14:56:31.160 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Bus STARTING 2023-09-23T14:56:31.160 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Serving on http://:::9283 2023-09-23T14:56:31.160 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100620]: [23/Sep/2023:14:56:31] ENGINE Bus STARTED 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T14:56:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[118128]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:31.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 ceph-mon[100189]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T14:56:31.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:31 smithi017 conmon[100165]: 2023-09-23T14:56:31.342+0000 7f64e5709700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='client.24545 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: pgmap v39: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T14:56:31.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:31 smithi027 ceph-mon[105164]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T14:56:32.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T14:56:32.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T14:56:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T14:56:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T14:56:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: fsmap foo:0 2023-09-23T14:56:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: Saving service mds.foo spec with placement count:2 2023-09-23T14:56:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:32 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:32.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T14:56:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: fsmap foo:0 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: Saving service mds.foo spec with placement count:2 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017:9093"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T14:56:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: fsmap foo:0 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: Saving service mds.foo spec with placement count:2 2023-09-23T14:56:32.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:32 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:32.913 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:33.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:33 smithi017 ceph-mon[100189]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T14:56:33.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:33 smithi017 ceph-mon[100189]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:33.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:33 smithi017 ceph-mon[118128]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T14:56:33.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:33 smithi017 ceph-mon[118128]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:33 smithi027 ceph-mon[105164]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T14:56:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:33 smithi027 ceph-mon[105164]: pgmap v43: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:34 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:34 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:34 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:34 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:34 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T14:56:35.655 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:56:35 smithi017 conmon[175786]: ts=2023-09-23T14:56:35.386Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003690554s 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:35.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:35.922 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:35 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: pgmap v44: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:35 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:36.151 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:36 smithi027 ceph-mon[105164]: Deploying daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T14:56:37.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:36 smithi017 ceph-mon[100189]: Deploying daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T14:56:37.122 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:36 smithi017 ceph-mon[118128]: Deploying daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T14:56:37.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:37 smithi017 ceph-mon[100189]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:37.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:37 smithi017 ceph-mon[118128]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:37 smithi027 ceph-mon[105164]: pgmap v45: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:38.581 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:38 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:38] "GET /metrics HTTP/1.1" 200 203112 "" "Prometheus/2.43.0" 2023-09-23T14:56:39.037 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T14:56:39.070 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T14:56:39.070 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T14:56:39.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T14:56:39.445 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:boot 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: daemon mds.foo.smithi017.ygjhom assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: Cluster is now healthy 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: fsmap foo:0 1 up:standby 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:creating} 2023-09-23T14:56:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[100189]: daemon mds.foo.smithi017.ygjhom is now active in filesystem foo as rank 0 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:boot 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: daemon mds.foo.smithi017.ygjhom assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T14:56:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T14:56:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: Cluster is now healthy 2023-09-23T14:56:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: fsmap foo:0 1 up:standby 2023-09-23T14:56:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:56:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:creating} 2023-09-23T14:56:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:39 smithi017 ceph-mon[118128]: daemon mds.foo.smithi017.ygjhom is now active in filesystem foo as rank 0 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: pgmap v46: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:boot 2023-09-23T14:56:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: daemon mds.foo.smithi017.ygjhom assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: Cluster is now healthy 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: fsmap foo:0 1 up:standby 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:creating} 2023-09-23T14:56:40.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:39 smithi027 ceph-mon[105164]: daemon mds.foo.smithi017.ygjhom is now active in filesystem foo as rank 0 2023-09-23T14:56:40.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:40 smithi027 ceph-mon[105164]: Deploying daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T14:56:40.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:40 smithi027 ceph-mon[105164]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:40.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:40 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 2023-09-23T14:56:40.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[100189]: Deploying daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T14:56:40.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[100189]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 2023-09-23T14:56:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[118128]: Deploying daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T14:56:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[118128]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:40 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 2023-09-23T14:56:41.078 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:41.661 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":19,"active_gid":14463,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":625940860},{"type":"v1","addr":"172.21.15.17:6801","nonce":625940860}]},"active_addr":"172.21.15.17:6801/625940860","active_change":"2023-09-23T14:55:28.581764+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24343,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":774095134}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":1851970177}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2135416630}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2670521468}]}]} 2023-09-23T14:56:41.667 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T14:56:41.667 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T14:56:41.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T14:56:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T14:56:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2612506151' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:56:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2612506151' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:41.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:41 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.4 KiB/s wr, 4 op/s 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2612506151' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:41 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:42.192 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[100189]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:boot 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[100189]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[118128]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:boot 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:42 smithi017 ceph-mon[118128]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T14:56:43.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:56:43 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:56:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:42 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:56:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:42 smithi027 ceph-mon[105164]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:boot 2023-09-23T14:56:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:42 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:42 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:42 smithi027 ceph-mon[105164]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.3 KiB/s wr, 4 op/s 2023-09-23T14:56:43.842 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:43.843 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":68,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:56:32.350561+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:56:44.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[100189]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/540001078' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[118128]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:43 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/540001078' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:43 smithi027 ceph-mon[105164]: mds.? [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] up:active 2023-09-23T14:56:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:43 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:43 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:43 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:43 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/540001078' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:44.531 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T14:56:44.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T14:56:44.962 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:45.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:45.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:45.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:45 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:45.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:45.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:45 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:56:46.685 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:56:46.685 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":68,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T14:56:32.350561+0000","last_up_change":"2023-09-23T14:54:55.124817+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6803","nonce":2828543106}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6805","nonce":2828543106}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6809","nonce":2828543106}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":2828543106},{"type":"v1","addr":"172.21.15.17:6807","nonce":2828543106}]},"public_addr":"172.21.15.17:6803/2828543106","cluster_addr":"172.21.15.17:6805/2828543106","heartbeat_back_addr":"172.21.15.17:6809/2828543106","heartbeat_front_addr":"172.21.15.17:6807/2828543106","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6811","nonce":1437136339}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6813","nonce":1437136339}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6817","nonce":1437136339}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":1437136339},{"type":"v1","addr":"172.21.15.17:6815","nonce":1437136339}]},"public_addr":"172.21.15.17:6811/1437136339","cluster_addr":"172.21.15.17:6813/1437136339","heartbeat_back_addr":"172.21.15.17:6817/1437136339","heartbeat_front_addr":"172.21.15.17:6815/1437136339","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6819","nonce":542107690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6821","nonce":542107690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6825","nonce":542107690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":542107690},{"type":"v1","addr":"172.21.15.17:6823","nonce":542107690}]},"public_addr":"172.21.15.17:6819/542107690","cluster_addr":"172.21.15.17:6821/542107690","heartbeat_back_addr":"172.21.15.17:6825/542107690","heartbeat_front_addr":"172.21.15.17:6823/542107690","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6827","nonce":4002411025}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6829","nonce":4002411025}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6833","nonce":4002411025}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":4002411025},{"type":"v1","addr":"172.21.15.17:6831","nonce":4002411025}]},"public_addr":"172.21.15.17:6827/4002411025","cluster_addr":"172.21.15.17:6829/4002411025","heartbeat_back_addr":"172.21.15.17:6833/4002411025","heartbeat_front_addr":"172.21.15.17:6831/4002411025","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T14:56:47.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[100189]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:standby 2023-09-23T14:56:47.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[100189]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:47.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[100189]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:47.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/387671788' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:47 smithi027 ceph-mon[105164]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:standby 2023-09-23T14:56:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:47 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:47 smithi027 ceph-mon[105164]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:47.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:47 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/387671788' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:47.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[118128]: mds.? [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] up:standby 2023-09-23T14:56:47.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T14:56:47.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[118128]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:47.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:47 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/387671788' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T14:56:47.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T14:56:47.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T14:56:47.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T14:56:47.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T14:56:47.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T14:56:47.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T14:56:47.401 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T14:56:47.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T14:56:48.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:48 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:48] "GET /metrics HTTP/1.1" 200 238591 "" "Prometheus/2.43.0" 2023-09-23T14:56:48.866 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:48.916 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:48.983 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:49.225 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:49.254 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:49.256 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:49.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[100189]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:49.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:48 smithi017 ceph-mon[118128]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:49.294 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:48 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:48 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:48 smithi027 ceph-mon[105164]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:49.841 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:51.771 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:51 smithi017 ceph-mon[100189]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:51.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:51 smithi017 ceph-mon[118128]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:51 smithi027 ceph-mon[105164]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T14:56:53.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:56:53 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:56:53.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:53 smithi017 ceph-mon[100189]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:53 smithi017 ceph-mon[118128]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:53 smithi027 ceph-mon[105164]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:55.179 INFO:teuthology.orchestra.run.smithi017.stdout:167503724578 2023-09-23T14:56:55.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T14:56:55.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:55 smithi017 ceph-mon[100189]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:55.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:55 smithi017 ceph-mon[118128]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:56.026 INFO:teuthology.orchestra.run.smithi017.stdout:77309411375 2023-09-23T14:56:56.026 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T14:56:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:55 smithi027 ceph-mon[105164]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T14:56:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:57 smithi027 ceph-mon[105164]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:57.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[118128]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:56:57.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:56:57.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:57 smithi017 ceph-mon[100189]: pgmap v55: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:57.711 INFO:teuthology.orchestra.run.smithi017.stdout:201863462943 2023-09-23T14:56:57.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T14:56:58.008 INFO:teuthology.orchestra.run.smithi017.stdout:34359738426 2023-09-23T14:56:58.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T14:56:58.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:56:58 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:56:58] "GET /metrics HTTP/1.1" 200 238591 "" "Prometheus/2.43.0" 2023-09-23T14:56:59.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:58 smithi017 ceph-mon[100189]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:59.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:56:58 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:59.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:58 smithi017 ceph-mon[118128]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:59.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:56:58 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:59.191 INFO:teuthology.orchestra.run.smithi017.stdout:55834574901 2023-09-23T14:56:59.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T14:56:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:58 smithi027 ceph-mon[105164]: pgmap v56: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:56:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:56:58 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:56:59.392 INFO:teuthology.orchestra.run.smithi017.stdout:133143986216 2023-09-23T14:56:59.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T14:56:59.432 INFO:teuthology.orchestra.run.smithi017.stdout:236223201307 2023-09-23T14:56:59.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T14:56:59.874 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:59.889 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:56:59.913 INFO:teuthology.orchestra.run.smithi017.stdout:111669149740 2023-09-23T14:56:59.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T14:57:00.225 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:00.400 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:01.066 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:01.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:01 smithi017 ceph-mon[118128]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:01.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:01.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:01 smithi017 ceph-mon[100189]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:01 smithi027 ceph-mon[105164]: pgmap v57: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:02.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:02.761 INFO:teuthology.orchestra.run.smithi017.stdout:167503724580 2023-09-23T14:57:03.307 INFO:teuthology.orchestra.run.smithi017.stdout:77309411377 2023-09-23T14:57:03.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:03 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[100189]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2316012340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3056015860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[118128]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2316012340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:57:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:03 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3056015860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:57:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:03 smithi027 ceph-mon[105164]: pgmap v58: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:03 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2316012340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T14:57:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:03 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3056015860' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T14:57:04.778 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411375 got 77309411377 for osd.2 2023-09-23T14:57:04.778 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:05.047 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:05.077 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724578 got 167503724580 for osd.5 2023-09-23T14:57:05.077 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:05.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:05 smithi017 ceph-mon[100189]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:05 smithi017 ceph-mon[118128]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:06.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:05 smithi027 ceph-mon[105164]: pgmap v59: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:06.976 INFO:teuthology.orchestra.run.smithi017.stdout:201863462945 2023-09-23T14:57:07.066 INFO:teuthology.orchestra.run.smithi017.stdout:55834574903 2023-09-23T14:57:07.081 INFO:teuthology.orchestra.run.smithi017.stdout:34359738428 2023-09-23T14:57:07.272 INFO:teuthology.orchestra.run.smithi017.stdout:236223201309 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[118128]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1705337744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2553820549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1589672256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3176149819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:57:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[100189]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:07.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1705337744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:57:07.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2553820549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:57:07.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1589672256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:57:07.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:07 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3176149819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:57:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:07 smithi027 ceph-mon[105164]: pgmap v60: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:07 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1705337744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T14:57:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:07 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2553820549' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T14:57:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:07 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1589672256' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T14:57:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:07 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3176149819' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T14:57:08.235 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462943 got 201863462945 for osd.6 2023-09-23T14:57:08.236 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:08.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:08 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:08] "GET /metrics HTTP/1.1" 200 238584 "" "Prometheus/2.43.0" 2023-09-23T14:57:09.072 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738426 got 34359738428 for osd.0 2023-09-23T14:57:09.072 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:09.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:08 smithi017 ceph-mon[100189]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:08 smithi017 ceph-mon[118128]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:09.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:08 smithi027 ceph-mon[105164]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:09.504 INFO:teuthology.orchestra.run.smithi017.stdout:133143986218 2023-09-23T14:57:09.558 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574901 got 55834574903 for osd.1 2023-09-23T14:57:09.558 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:09.636 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201307 got 236223201309 for osd.7 2023-09-23T14:57:09.636 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:09.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:09 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/398894799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:57:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:09 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/398894799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:57:10.349 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986216 got 133143986218 for osd.4 2023-09-23T14:57:10.350 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:09 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/398894799' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T14:57:10.607 INFO:teuthology.orchestra.run.smithi017.stdout:111669149742 2023-09-23T14:57:11.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:10 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2704687761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:57:11.295 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:10 smithi017 ceph-mon[118128]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:11.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:10 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2704687761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:57:11.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:10 smithi017 ceph-mon[100189]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:11.308 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149740 got 111669149742 for osd.3 2023-09-23T14:57:11.308 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:11.308 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T14:57:11.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T14:57:11.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:10 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2704687761' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T14:57:11.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:10 smithi027 ceph-mon[105164]: pgmap v62: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:11.677 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:13.404 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:57:13.405 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T14:57:13.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:13 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:13.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:13 smithi017 ceph-mon[100189]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:13 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:13 smithi017 ceph-mon[118128]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:13 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:14.032 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":63,"stamp":"2023-09-23T14:57:12.615108+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52248,"kb_used_data":7192,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749908968,"statfs":{"total":767960285184,"available":767906783232,"internally_reserved":0,"allocated":7364608,"data_stored":4140241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002898"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.366192+0000","last_change":"2023-09-23T14:56:31.365798+0000","last_active":"2023-09-23T14:56:31.366192+0000","last_peered":"2023-09-23T14:56:31.366192+0000","last_clean":"2023-09-23T14:56:31.366192+0000","last_became_active":"2023-09-23T14:56:31.365571+0000","last_became_peered":"2023-09-23T14:56:31.365571+0000","last_unstale":"2023-09-23T14:56:31.366192+0000","last_undegraded":"2023-09-23T14:56:31.366192+0000","last_fullsized":"2023-09-23T14:56:31.366192+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:26.620629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943557+0000","last_change":"2023-09-23T14:56:31.340532+0000","last_active":"2023-09-23T14:56:38.943557+0000","last_peered":"2023-09-23T14:56:38.943557+0000","last_clean":"2023-09-23T14:56:38.943557+0000","last_became_active":"2023-09-23T14:56:31.340248+0000","last_became_peered":"2023-09-23T14:56:31.340248+0000","last_unstale":"2023-09-23T14:56:38.943557+0000","last_undegraded":"2023-09-23T14:56:38.943557+0000","last_fullsized":"2023-09-23T14:56:38.943557+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:59.101892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369065+0000","last_change":"2023-09-23T14:56:31.365130+0000","last_active":"2023-09-23T14:56:31.369065+0000","last_peered":"2023-09-23T14:56:31.369065+0000","last_clean":"2023-09-23T14:56:31.369065+0000","last_became_active":"2023-09-23T14:56:31.364858+0000","last_became_peered":"2023-09-23T14:56:31.364858+0000","last_unstale":"2023-09-23T14:56:31.369065+0000","last_undegraded":"2023-09-23T14:56:31.369065+0000","last_fullsized":"2023-09-23T14:56:31.369065+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:18.579740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.354305+0000","last_change":"2023-09-23T14:56:31.338860+0000","last_active":"2023-09-23T14:56:32.354305+0000","last_peered":"2023-09-23T14:56:32.354305+0000","last_clean":"2023-09-23T14:56:32.354305+0000","last_became_active":"2023-09-23T14:56:31.338576+0000","last_became_peered":"2023-09-23T14:56:31.338576+0000","last_unstale":"2023-09-23T14:56:32.354305+0000","last_undegraded":"2023-09-23T14:56:32.354305+0000","last_fullsized":"2023-09-23T14:56:32.354305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:57:57.956735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364443+0000","last_change":"2023-09-23T14:56:31.364162+0000","last_active":"2023-09-23T14:56:31.364443+0000","last_peered":"2023-09-23T14:56:31.364443+0000","last_clean":"2023-09-23T14:56:31.364443+0000","last_became_active":"2023-09-23T14:56:31.363873+0000","last_became_peered":"2023-09-23T14:56:31.363873+0000","last_unstale":"2023-09-23T14:56:31.364443+0000","last_undegraded":"2023-09-23T14:56:31.364443+0000","last_fullsized":"2023-09-23T14:56:31.364443+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:45.045755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943627+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T14:56:38.943627+0000","last_peered":"2023-09-23T14:56:38.943627+0000","last_clean":"2023-09-23T14:56:38.943627+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T14:56:38.943627+0000","last_undegraded":"2023-09-23T14:56:38.943627+0000","last_fullsized":"2023-09-23T14:56:38.943627+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365684+0000","last_change":"2023-09-23T14:56:31.365313+0000","last_active":"2023-09-23T14:56:31.365684+0000","last_peered":"2023-09-23T14:56:31.365684+0000","last_clean":"2023-09-23T14:56:31.365684+0000","last_became_active":"2023-09-23T14:56:31.365029+0000","last_became_peered":"2023-09-23T14:56:31.365029+0000","last_unstale":"2023-09-23T14:56:31.365684+0000","last_undegraded":"2023-09-23T14:56:31.365684+0000","last_fullsized":"2023-09-23T14:56:31.365684+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:55.083515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.945418+0000","last_change":"2023-09-23T14:56:31.343452+0000","last_active":"2023-09-23T14:56:38.945418+0000","last_peered":"2023-09-23T14:56:38.945418+0000","last_clean":"2023-09-23T14:56:38.945418+0000","last_became_active":"2023-09-23T14:56:31.343218+0000","last_became_peered":"2023-09-23T14:56:31.343218+0000","last_unstale":"2023-09-23T14:56:38.945418+0000","last_undegraded":"2023-09-23T14:56:38.945418+0000","last_fullsized":"2023-09-23T14:56:38.945418+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.216966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365736+0000","last_change":"2023-09-23T14:56:31.362635+0000","last_active":"2023-09-23T14:56:31.365736+0000","last_peered":"2023-09-23T14:56:31.365736+0000","last_clean":"2023-09-23T14:56:31.365736+0000","last_became_active":"2023-09-23T14:56:31.362363+0000","last_became_peered":"2023-09-23T14:56:31.362363+0000","last_unstale":"2023-09-23T14:56:31.365736+0000","last_undegraded":"2023-09-23T14:56:31.365736+0000","last_fullsized":"2023-09-23T14:56:31.365736+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:11.181610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.355422+0000","last_change":"2023-09-23T14:56:31.333730+0000","last_active":"2023-09-23T14:56:32.355422+0000","last_peered":"2023-09-23T14:56:32.355422+0000","last_clean":"2023-09-23T14:56:32.355422+0000","last_became_active":"2023-09-23T14:56:31.333141+0000","last_became_peered":"2023-09-23T14:56:31.333141+0000","last_unstale":"2023-09-23T14:56:32.355422+0000","last_undegraded":"2023-09-23T14:56:32.355422+0000","last_fullsized":"2023-09-23T14:56:32.355422+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:36.418231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362420+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T14:56:31.362420+0000","last_peered":"2023-09-23T14:56:31.362420+0000","last_clean":"2023-09-23T14:56:31.362420+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T14:56:31.362420+0000","last_undegraded":"2023-09-23T14:56:31.362420+0000","last_fullsized":"2023-09-23T14:56:31.362420+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.355707+0000","last_change":"2023-09-23T14:56:31.337422+0000","last_active":"2023-09-23T14:56:32.355707+0000","last_peered":"2023-09-23T14:56:32.355707+0000","last_clean":"2023-09-23T14:56:32.355707+0000","last_became_active":"2023-09-23T14:56:31.337132+0000","last_became_peered":"2023-09-23T14:56:31.337132+0000","last_unstale":"2023-09-23T14:56:32.355707+0000","last_undegraded":"2023-09-23T14:56:32.355707+0000","last_fullsized":"2023-09-23T14:56:32.355707+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:45:28.461444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356314+0000","last_change":"2023-09-23T14:55:42.668313+0000","last_active":"2023-09-23T14:56:31.356314+0000","last_peered":"2023-09-23T14:56:31.356314+0000","last_clean":"2023-09-23T14:56:31.356314+0000","last_became_active":"2023-09-23T14:55:42.667931+0000","last_became_peered":"2023-09-23T14:55:42.667931+0000","last_unstale":"2023-09-23T14:56:31.356314+0000","last_undegraded":"2023-09-23T14:56:31.356314+0000","last_fullsized":"2023-09-23T14:56:31.356314+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:11:42.832634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642283+0000","last_change":"2023-09-23T14:56:31.640804+0000","last_active":"2023-09-23T14:56:31.642283+0000","last_peered":"2023-09-23T14:56:31.642283+0000","last_clean":"2023-09-23T14:56:31.642283+0000","last_became_active":"2023-09-23T14:56:31.640477+0000","last_became_peered":"2023-09-23T14:56:31.640477+0000","last_unstale":"2023-09-23T14:56:31.642283+0000","last_undegraded":"2023-09-23T14:56:31.642283+0000","last_fullsized":"2023-09-23T14:56:31.642283+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:43:38.124239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365845+0000","last_change":"2023-09-23T14:55:42.887554+0000","last_active":"2023-09-23T14:56:31.365845+0000","last_peered":"2023-09-23T14:56:31.365845+0000","last_clean":"2023-09-23T14:56:31.365845+0000","last_became_active":"2023-09-23T14:55:42.887232+0000","last_became_peered":"2023-09-23T14:55:42.887232+0000","last_unstale":"2023-09-23T14:56:31.365845+0000","last_undegraded":"2023-09-23T14:56:31.365845+0000","last_fullsized":"2023-09-23T14:56:31.365845+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:19.963176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.944474+0000","last_change":"2023-09-23T14:56:31.344484+0000","last_active":"2023-09-23T14:56:38.944474+0000","last_peered":"2023-09-23T14:56:38.944474+0000","last_clean":"2023-09-23T14:56:38.944474+0000","last_became_active":"2023-09-23T14:56:31.344177+0000","last_became_peered":"2023-09-23T14:56:31.344177+0000","last_unstale":"2023-09-23T14:56:38.944474+0000","last_undegraded":"2023-09-23T14:56:38.944474+0000","last_fullsized":"2023-09-23T14:56:38.944474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:44.592205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642686+0000","last_change":"2023-09-23T14:56:31.640792+0000","last_active":"2023-09-23T14:56:31.642686+0000","last_peered":"2023-09-23T14:56:31.642686+0000","last_clean":"2023-09-23T14:56:31.642686+0000","last_became_active":"2023-09-23T14:56:31.640462+0000","last_became_peered":"2023-09-23T14:56:31.640462+0000","last_unstale":"2023-09-23T14:56:31.642686+0000","last_undegraded":"2023-09-23T14:56:31.642686+0000","last_fullsized":"2023-09-23T14:56:31.642686+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:09.002025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.358747+0000","last_change":"2023-09-23T14:56:31.340458+0000","last_active":"2023-09-23T14:56:31.358747+0000","last_peered":"2023-09-23T14:56:31.358747+0000","last_clean":"2023-09-23T14:56:31.358747+0000","last_became_active":"2023-09-23T14:56:31.340180+0000","last_became_peered":"2023-09-23T14:56:31.340180+0000","last_unstale":"2023-09-23T14:56:31.358747+0000","last_undegraded":"2023-09-23T14:56:31.358747+0000","last_fullsized":"2023-09-23T14:56:31.358747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:21.662326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639978+0000","last_change":"2023-09-23T14:55:42.887686+0000","last_active":"2023-09-23T14:56:31.639978+0000","last_peered":"2023-09-23T14:56:31.639978+0000","last_clean":"2023-09-23T14:56:31.639978+0000","last_became_active":"2023-09-23T14:55:42.887225+0000","last_became_peered":"2023-09-23T14:55:42.887225+0000","last_unstale":"2023-09-23T14:56:31.639978+0000","last_undegraded":"2023-09-23T14:56:31.639978+0000","last_fullsized":"2023-09-23T14:56:31.639978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:32.886192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.363895+0000","last_change":"2023-09-23T14:56:31.362815+0000","last_active":"2023-09-23T14:56:31.363895+0000","last_peered":"2023-09-23T14:56:31.363895+0000","last_clean":"2023-09-23T14:56:31.363895+0000","last_became_active":"2023-09-23T14:56:31.362456+0000","last_became_peered":"2023-09-23T14:56:31.362456+0000","last_unstale":"2023-09-23T14:56:31.363895+0000","last_undegraded":"2023-09-23T14:56:31.363895+0000","last_fullsized":"2023-09-23T14:56:31.363895+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:42.421930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355527+0000","last_change":"2023-09-23T14:56:31.336392+0000","last_active":"2023-09-23T14:56:31.355527+0000","last_peered":"2023-09-23T14:56:31.355527+0000","last_clean":"2023-09-23T14:56:31.355527+0000","last_became_active":"2023-09-23T14:56:31.336144+0000","last_became_peered":"2023-09-23T14:56:31.336144+0000","last_unstale":"2023-09-23T14:56:31.355527+0000","last_undegraded":"2023-09-23T14:56:31.355527+0000","last_fullsized":"2023-09-23T14:56:31.355527+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:52.389183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639600+0000","last_change":"2023-09-23T14:55:42.667453+0000","last_active":"2023-09-23T14:56:31.639600+0000","last_peered":"2023-09-23T14:56:31.639600+0000","last_clean":"2023-09-23T14:56:31.639600+0000","last_became_active":"2023-09-23T14:55:42.667131+0000","last_became_peered":"2023-09-23T14:55:42.667131+0000","last_unstale":"2023-09-23T14:56:31.639600+0000","last_undegraded":"2023-09-23T14:56:31.639600+0000","last_fullsized":"2023-09-23T14:56:31.639600+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:20.524459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.644325+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T14:56:31.644325+0000","last_peered":"2023-09-23T14:56:31.644325+0000","last_clean":"2023-09-23T14:56:31.644325+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T14:56:31.644325+0000","last_undegraded":"2023-09-23T14:56:31.644325+0000","last_fullsized":"2023-09-23T14:56:31.644325+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.353732+0000","last_change":"2023-09-23T14:56:31.337515+0000","last_active":"2023-09-23T14:56:31.353732+0000","last_peered":"2023-09-23T14:56:31.353732+0000","last_clean":"2023-09-23T14:56:31.353732+0000","last_became_active":"2023-09-23T14:56:31.337223+0000","last_became_peered":"2023-09-23T14:56:31.337223+0000","last_unstale":"2023-09-23T14:56:31.353732+0000","last_undegraded":"2023-09-23T14:56:31.353732+0000","last_fullsized":"2023-09-23T14:56:31.353732+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:06:44.128414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639554+0000","last_change":"2023-09-23T14:54:58.918703+0000","last_active":"2023-09-23T14:56:31.639554+0000","last_peered":"2023-09-23T14:56:31.639554+0000","last_clean":"2023-09-23T14:56:31.639554+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T14:56:31.639554+0000","last_undegraded":"2023-09-23T14:56:31.639554+0000","last_fullsized":"2023-09-23T14:56:31.639554+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:41.896866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355510+0000","last_change":"2023-09-23T14:55:42.668315+0000","last_active":"2023-09-23T14:56:31.355510+0000","last_peered":"2023-09-23T14:56:31.355510+0000","last_clean":"2023-09-23T14:56:31.355510+0000","last_became_active":"2023-09-23T14:55:42.667972+0000","last_became_peered":"2023-09-23T14:55:42.667972+0000","last_unstale":"2023-09-23T14:56:31.355510+0000","last_undegraded":"2023-09-23T14:56:31.355510+0000","last_fullsized":"2023-09-23T14:56:31.355510+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373687+0000","last_change":"2023-09-23T14:56:31.366479+0000","last_active":"2023-09-23T14:56:31.373687+0000","last_peered":"2023-09-23T14:56:31.373687+0000","last_clean":"2023-09-23T14:56:31.373687+0000","last_became_active":"2023-09-23T14:56:31.365485+0000","last_became_peered":"2023-09-23T14:56:31.365485+0000","last_unstale":"2023-09-23T14:56:31.373687+0000","last_undegraded":"2023-09-23T14:56:31.373687+0000","last_fullsized":"2023-09-23T14:56:31.373687+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:21.531015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364144+0000","last_change":"2023-09-23T14:56:31.334902+0000","last_active":"2023-09-23T14:56:31.364144+0000","last_peered":"2023-09-23T14:56:31.364144+0000","last_clean":"2023-09-23T14:56:31.364144+0000","last_became_active":"2023-09-23T14:56:31.334525+0000","last_became_peered":"2023-09-23T14:56:31.334525+0000","last_unstale":"2023-09-23T14:56:31.364144+0000","last_undegraded":"2023-09-23T14:56:31.364144+0000","last_fullsized":"2023-09-23T14:56:31.364144+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:24.922324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.352282+0000","last_change":"2023-09-23T14:55:42.670066+0000","last_active":"2023-09-23T14:56:31.352282+0000","last_peered":"2023-09-23T14:56:31.352282+0000","last_clean":"2023-09-23T14:56:31.352282+0000","last_became_active":"2023-09-23T14:55:42.669743+0000","last_became_peered":"2023-09-23T14:55:42.669743+0000","last_unstale":"2023-09-23T14:56:31.352282+0000","last_undegraded":"2023-09-23T14:56:31.352282+0000","last_fullsized":"2023-09-23T14:56:31.352282+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:17.259363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373938+0000","last_change":"2023-09-23T14:56:31.369934+0000","last_active":"2023-09-23T14:56:31.373938+0000","last_peered":"2023-09-23T14:56:31.373938+0000","last_clean":"2023-09-23T14:56:31.373938+0000","last_became_active":"2023-09-23T14:56:31.369753+0000","last_became_peered":"2023-09-23T14:56:31.369753+0000","last_unstale":"2023-09-23T14:56:31.373938+0000","last_undegraded":"2023-09-23T14:56:31.373938+0000","last_fullsized":"2023-09-23T14:56:31.373938+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:03.998336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356704+0000","last_change":"2023-09-23T14:56:31.337354+0000","last_active":"2023-09-23T14:56:31.356704+0000","last_peered":"2023-09-23T14:56:31.356704+0000","last_clean":"2023-09-23T14:56:31.356704+0000","last_became_active":"2023-09-23T14:56:31.336634+0000","last_became_peered":"2023-09-23T14:56:31.336634+0000","last_unstale":"2023-09-23T14:56:31.356704+0000","last_undegraded":"2023-09-23T14:56:31.356704+0000","last_fullsized":"2023-09-23T14:56:31.356704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:30.083470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355064+0000","last_change":"2023-09-23T14:55:42.666173+0000","last_active":"2023-09-23T14:56:31.355064+0000","last_peered":"2023-09-23T14:56:31.355064+0000","last_clean":"2023-09-23T14:56:31.355064+0000","last_became_active":"2023-09-23T14:55:42.665827+0000","last_became_peered":"2023-09-23T14:55:42.665827+0000","last_unstale":"2023-09-23T14:56:31.355064+0000","last_undegraded":"2023-09-23T14:56:31.355064+0000","last_fullsized":"2023-09-23T14:56:31.355064+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:29.577101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373715+0000","last_change":"2023-09-23T14:56:31.367210+0000","last_active":"2023-09-23T14:56:31.373715+0000","last_peered":"2023-09-23T14:56:31.373715+0000","last_clean":"2023-09-23T14:56:31.373715+0000","last_became_active":"2023-09-23T14:56:31.366915+0000","last_became_peered":"2023-09-23T14:56:31.366915+0000","last_unstale":"2023-09-23T14:56:31.373715+0000","last_undegraded":"2023-09-23T14:56:31.373715+0000","last_fullsized":"2023-09-23T14:56:31.373715+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:30.521148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355346+0000","last_change":"2023-09-23T14:56:31.344343+0000","last_active":"2023-09-23T14:56:31.355346+0000","last_peered":"2023-09-23T14:56:31.355346+0000","last_clean":"2023-09-23T14:56:31.355346+0000","last_became_active":"2023-09-23T14:56:31.344105+0000","last_became_peered":"2023-09-23T14:56:31.344105+0000","last_unstale":"2023-09-23T14:56:31.355346+0000","last_undegraded":"2023-09-23T14:56:31.355346+0000","last_fullsized":"2023-09-23T14:56:31.355346+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:15.458054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365118+0000","last_change":"2023-09-23T14:55:42.665607+0000","last_active":"2023-09-23T14:56:31.365118+0000","last_peered":"2023-09-23T14:56:31.365118+0000","last_clean":"2023-09-23T14:56:31.365118+0000","last_became_active":"2023-09-23T14:55:42.665279+0000","last_became_peered":"2023-09-23T14:55:42.665279+0000","last_unstale":"2023-09-23T14:56:31.365118+0000","last_undegraded":"2023-09-23T14:56:31.365118+0000","last_fullsized":"2023-09-23T14:56:31.365118+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:58.018948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373782+0000","last_change":"2023-09-23T14:56:31.369890+0000","last_active":"2023-09-23T14:56:31.373782+0000","last_peered":"2023-09-23T14:56:31.373782+0000","last_clean":"2023-09-23T14:56:31.373782+0000","last_became_active":"2023-09-23T14:56:31.369618+0000","last_became_peered":"2023-09-23T14:56:31.369618+0000","last_unstale":"2023-09-23T14:56:31.373782+0000","last_undegraded":"2023-09-23T14:56:31.373782+0000","last_fullsized":"2023-09-23T14:56:31.373782+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:56.163148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.354214+0000","last_change":"2023-09-23T14:56:31.343731+0000","last_active":"2023-09-23T14:56:31.354214+0000","last_peered":"2023-09-23T14:56:31.354214+0000","last_clean":"2023-09-23T14:56:31.354214+0000","last_became_active":"2023-09-23T14:56:31.343418+0000","last_became_peered":"2023-09-23T14:56:31.343418+0000","last_unstale":"2023-09-23T14:56:31.354214+0000","last_undegraded":"2023-09-23T14:56:31.354214+0000","last_fullsized":"2023-09-23T14:56:31.354214+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:25.986640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362943+0000","last_change":"2023-09-23T14:56:31.362550+0000","last_active":"2023-09-23T14:56:31.362943+0000","last_peered":"2023-09-23T14:56:31.362943+0000","last_clean":"2023-09-23T14:56:31.362943+0000","last_became_active":"2023-09-23T14:56:31.362261+0000","last_became_peered":"2023-09-23T14:56:31.362261+0000","last_unstale":"2023-09-23T14:56:31.362943+0000","last_undegraded":"2023-09-23T14:56:31.362943+0000","last_fullsized":"2023-09-23T14:56:31.362943+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:26.597585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.357503+0000","last_change":"2023-09-23T14:56:31.338927+0000","last_active":"2023-09-23T14:56:31.357503+0000","last_peered":"2023-09-23T14:56:31.357503+0000","last_clean":"2023-09-23T14:56:31.357503+0000","last_became_active":"2023-09-23T14:56:31.338287+0000","last_became_peered":"2023-09-23T14:56:31.338287+0000","last_unstale":"2023-09-23T14:56:31.357503+0000","last_undegraded":"2023-09-23T14:56:31.357503+0000","last_fullsized":"2023-09-23T14:56:31.357503+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:43.291216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.361924+0000","last_change":"2023-09-23T14:56:31.361613+0000","last_active":"2023-09-23T14:56:31.361924+0000","last_peered":"2023-09-23T14:56:31.361924+0000","last_clean":"2023-09-23T14:56:31.361924+0000","last_became_active":"2023-09-23T14:56:31.361427+0000","last_became_peered":"2023-09-23T14:56:31.361427+0000","last_unstale":"2023-09-23T14:56:31.361924+0000","last_undegraded":"2023-09-23T14:56:31.361924+0000","last_fullsized":"2023-09-23T14:56:31.361924+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:50.247243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356315+0000","last_change":"2023-09-23T14:56:31.333503+0000","last_active":"2023-09-23T14:56:31.356315+0000","last_peered":"2023-09-23T14:56:31.356315+0000","last_clean":"2023-09-23T14:56:31.356315+0000","last_became_active":"2023-09-23T14:56:31.333045+0000","last_became_peered":"2023-09-23T14:56:31.333045+0000","last_unstale":"2023-09-23T14:56:31.356315+0000","last_undegraded":"2023-09-23T14:56:31.356315+0000","last_fullsized":"2023-09-23T14:56:31.356315+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:20.987619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.361877+0000","last_change":"2023-09-23T14:56:31.360820+0000","last_active":"2023-09-23T14:56:31.361877+0000","last_peered":"2023-09-23T14:56:31.361877+0000","last_clean":"2023-09-23T14:56:31.361877+0000","last_became_active":"2023-09-23T14:56:31.360436+0000","last_became_peered":"2023-09-23T14:56:31.360436+0000","last_unstale":"2023-09-23T14:56:31.361877+0000","last_undegraded":"2023-09-23T14:56:31.361877+0000","last_fullsized":"2023-09-23T14:56:31.361877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:13.769760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943417+0000","last_change":"2023-09-23T14:56:31.343639+0000","last_active":"2023-09-23T14:56:38.943417+0000","last_peered":"2023-09-23T14:56:38.943417+0000","last_clean":"2023-09-23T14:56:38.943417+0000","last_became_active":"2023-09-23T14:56:31.343307+0000","last_became_peered":"2023-09-23T14:56:31.343307+0000","last_unstale":"2023-09-23T14:56:38.943417+0000","last_undegraded":"2023-09-23T14:56:38.943417+0000","last_fullsized":"2023-09-23T14:56:38.943417+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:47.909520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.368155+0000","last_change":"2023-09-23T14:56:31.366975+0000","last_active":"2023-09-23T14:56:31.368155+0000","last_peered":"2023-09-23T14:56:31.368155+0000","last_clean":"2023-09-23T14:56:31.368155+0000","last_became_active":"2023-09-23T14:56:31.366669+0000","last_became_peered":"2023-09-23T14:56:31.366669+0000","last_unstale":"2023-09-23T14:56:31.368155+0000","last_undegraded":"2023-09-23T14:56:31.368155+0000","last_fullsized":"2023-09-23T14:56:31.368155+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:32.225320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355638+0000","last_change":"2023-09-23T14:56:31.342259+0000","last_active":"2023-09-23T14:56:31.355638+0000","last_peered":"2023-09-23T14:56:31.355638+0000","last_clean":"2023-09-23T14:56:31.355638+0000","last_became_active":"2023-09-23T14:56:31.341657+0000","last_became_peered":"2023-09-23T14:56:31.341657+0000","last_unstale":"2023-09-23T14:56:31.355638+0000","last_undegraded":"2023-09-23T14:56:31.355638+0000","last_fullsized":"2023-09-23T14:56:31.355638+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:52.375439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.641216+0000","last_change":"2023-09-23T14:56:31.640765+0000","last_active":"2023-09-23T14:56:31.641216+0000","last_peered":"2023-09-23T14:56:31.641216+0000","last_clean":"2023-09-23T14:56:31.641216+0000","last_became_active":"2023-09-23T14:56:31.640435+0000","last_became_peered":"2023-09-23T14:56:31.640435+0000","last_unstale":"2023-09-23T14:56:31.641216+0000","last_undegraded":"2023-09-23T14:56:31.641216+0000","last_fullsized":"2023-09-23T14:56:31.641216+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:51.676196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.496280+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T14:56:32.496280+0000","last_peered":"2023-09-23T14:56:32.496280+0000","last_clean":"2023-09-23T14:56:32.496280+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T14:56:32.496280+0000","last_undegraded":"2023-09-23T14:56:32.496280+0000","last_fullsized":"2023-09-23T14:56:32.496280+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362942+0000","last_change":"2023-09-23T14:56:31.362551+0000","last_active":"2023-09-23T14:56:31.362942+0000","last_peered":"2023-09-23T14:56:31.362942+0000","last_clean":"2023-09-23T14:56:31.362942+0000","last_became_active":"2023-09-23T14:56:31.362240+0000","last_became_peered":"2023-09-23T14:56:31.362240+0000","last_unstale":"2023-09-23T14:56:31.362942+0000","last_undegraded":"2023-09-23T14:56:31.362942+0000","last_fullsized":"2023-09-23T14:56:31.362942+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:54.851534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.946505+0000","last_change":"2023-09-23T14:56:31.343468+0000","last_active":"2023-09-23T14:56:38.946505+0000","last_peered":"2023-09-23T14:56:38.946505+0000","last_clean":"2023-09-23T14:56:38.946505+0000","last_became_active":"2023-09-23T14:56:31.342156+0000","last_became_peered":"2023-09-23T14:56:31.342156+0000","last_unstale":"2023-09-23T14:56:38.946505+0000","last_undegraded":"2023-09-23T14:56:38.946505+0000","last_fullsized":"2023-09-23T14:56:38.946505+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:44.209095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.641103+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T14:56:31.641103+0000","last_peered":"2023-09-23T14:56:31.641103+0000","last_clean":"2023-09-23T14:56:31.641103+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T14:56:31.641103+0000","last_undegraded":"2023-09-23T14:56:31.641103+0000","last_fullsized":"2023-09-23T14:56:31.641103+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.496355+0000","last_change":"2023-09-23T14:56:31.341739+0000","last_active":"2023-09-23T14:56:32.496355+0000","last_peered":"2023-09-23T14:56:32.496355+0000","last_clean":"2023-09-23T14:56:32.496355+0000","last_became_active":"2023-09-23T14:56:31.340906+0000","last_became_peered":"2023-09-23T14:56:31.340906+0000","last_unstale":"2023-09-23T14:56:32.496355+0000","last_undegraded":"2023-09-23T14:56:32.496355+0000","last_fullsized":"2023-09-23T14:56:32.496355+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:20.857073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642208+0000","last_change":"2023-09-23T14:56:31.640894+0000","last_active":"2023-09-23T14:56:31.642208+0000","last_peered":"2023-09-23T14:56:31.642208+0000","last_clean":"2023-09-23T14:56:31.642208+0000","last_became_active":"2023-09-23T14:56:31.640576+0000","last_became_peered":"2023-09-23T14:56:31.640576+0000","last_unstale":"2023-09-23T14:56:31.642208+0000","last_undegraded":"2023-09-23T14:56:31.642208+0000","last_fullsized":"2023-09-23T14:56:31.642208+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:43.572217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.947195+0000","last_change":"2023-09-23T14:56:31.347827+0000","last_active":"2023-09-23T14:56:38.947195+0000","last_peered":"2023-09-23T14:56:38.947195+0000","last_clean":"2023-09-23T14:56:38.947195+0000","last_became_active":"2023-09-23T14:56:31.347502+0000","last_became_peered":"2023-09-23T14:56:31.347502+0000","last_unstale":"2023-09-23T14:56:38.947195+0000","last_undegraded":"2023-09-23T14:56:38.947195+0000","last_fullsized":"2023-09-23T14:56:38.947195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:32.331497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642757+0000","last_change":"2023-09-23T14:56:31.641209+0000","last_active":"2023-09-23T14:56:31.642757+0000","last_peered":"2023-09-23T14:56:31.642757+0000","last_clean":"2023-09-23T14:56:31.642757+0000","last_became_active":"2023-09-23T14:56:31.640477+0000","last_became_peered":"2023-09-23T14:56:31.640477+0000","last_unstale":"2023-09-23T14:56:31.642757+0000","last_undegraded":"2023-09-23T14:56:31.642757+0000","last_fullsized":"2023-09-23T14:56:31.642757+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:24.082906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639471+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T14:56:31.639471+0000","last_peered":"2023-09-23T14:56:31.639471+0000","last_clean":"2023-09-23T14:56:31.639471+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T14:56:31.639471+0000","last_undegraded":"2023-09-23T14:56:31.639471+0000","last_fullsized":"2023-09-23T14:56:31.639471+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.643030+0000","last_change":"2023-09-23T14:56:31.642178+0000","last_active":"2023-09-23T14:56:31.643030+0000","last_peered":"2023-09-23T14:56:31.643030+0000","last_clean":"2023-09-23T14:56:31.643030+0000","last_became_active":"2023-09-23T14:56:31.641822+0000","last_became_peered":"2023-09-23T14:56:31.641822+0000","last_unstale":"2023-09-23T14:56:31.643030+0000","last_undegraded":"2023-09-23T14:56:31.643030+0000","last_fullsized":"2023-09-23T14:56:31.643030+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:36.187232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356429+0000","last_change":"2023-09-23T14:56:31.332428+0000","last_active":"2023-09-23T14:56:31.356429+0000","last_peered":"2023-09-23T14:56:31.356429+0000","last_clean":"2023-09-23T14:56:31.356429+0000","last_became_active":"2023-09-23T14:56:31.332128+0000","last_became_peered":"2023-09-23T14:56:31.332128+0000","last_unstale":"2023-09-23T14:56:31.356429+0000","last_undegraded":"2023-09-23T14:56:31.356429+0000","last_fullsized":"2023-09-23T14:56:31.356429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:13.596995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642257+0000","last_change":"2023-09-23T14:56:31.641239+0000","last_active":"2023-09-23T14:56:31.642257+0000","last_peered":"2023-09-23T14:56:31.642257+0000","last_clean":"2023-09-23T14:56:31.642257+0000","last_became_active":"2023-09-23T14:56:31.640888+0000","last_became_peered":"2023-09-23T14:56:31.640888+0000","last_unstale":"2023-09-23T14:56:31.642257+0000","last_undegraded":"2023-09-23T14:56:31.642257+0000","last_fullsized":"2023-09-23T14:56:31.642257+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:11.816464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.947454+0000","last_change":"2023-09-23T14:56:31.341608+0000","last_active":"2023-09-23T14:56:38.947454+0000","last_peered":"2023-09-23T14:56:38.947454+0000","last_clean":"2023-09-23T14:56:38.947454+0000","last_became_active":"2023-09-23T14:56:31.340802+0000","last_became_peered":"2023-09-23T14:56:31.340802+0000","last_unstale":"2023-09-23T14:56:38.947454+0000","last_undegraded":"2023-09-23T14:56:38.947454+0000","last_fullsized":"2023-09-23T14:56:38.947454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:21.513338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365701+0000","last_change":"2023-09-23T14:56:31.365386+0000","last_active":"2023-09-23T14:56:31.365701+0000","last_peered":"2023-09-23T14:56:31.365701+0000","last_clean":"2023-09-23T14:56:31.365701+0000","last_became_active":"2023-09-23T14:56:31.365100+0000","last_became_peered":"2023-09-23T14:56:31.365100+0000","last_unstale":"2023-09-23T14:56:31.365701+0000","last_undegraded":"2023-09-23T14:56:31.365701+0000","last_fullsized":"2023-09-23T14:56:31.365701+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:32.427060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.948362+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T14:56:38.948362+0000","last_peered":"2023-09-23T14:56:38.948362+0000","last_clean":"2023-09-23T14:56:38.948362+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T14:56:38.948362+0000","last_undegraded":"2023-09-23T14:56:38.948362+0000","last_fullsized":"2023-09-23T14:56:38.948362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373693+0000","last_change":"2023-09-23T14:56:31.366542+0000","last_active":"2023-09-23T14:56:31.373693+0000","last_peered":"2023-09-23T14:56:31.373693+0000","last_clean":"2023-09-23T14:56:31.373693+0000","last_became_active":"2023-09-23T14:56:31.366224+0000","last_became_peered":"2023-09-23T14:56:31.366224+0000","last_unstale":"2023-09-23T14:56:31.373693+0000","last_undegraded":"2023-09-23T14:56:31.373693+0000","last_fullsized":"2023-09-23T14:56:31.373693+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:35.698729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.946571+0000","last_change":"2023-09-23T14:56:31.342751+0000","last_active":"2023-09-23T14:56:38.946571+0000","last_peered":"2023-09-23T14:56:38.946571+0000","last_clean":"2023-09-23T14:56:38.946571+0000","last_became_active":"2023-09-23T14:56:31.342324+0000","last_became_peered":"2023-09-23T14:56:31.342324+0000","last_unstale":"2023-09-23T14:56:38.946571+0000","last_undegraded":"2023-09-23T14:56:38.946571+0000","last_fullsized":"2023-09-23T14:56:38.946571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:26.082641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642635+0000","last_change":"2023-09-23T14:56:31.641933+0000","last_active":"2023-09-23T14:56:31.642635+0000","last_peered":"2023-09-23T14:56:31.642635+0000","last_clean":"2023-09-23T14:56:31.642635+0000","last_became_active":"2023-09-23T14:56:31.641472+0000","last_became_peered":"2023-09-23T14:56:31.641472+0000","last_unstale":"2023-09-23T14:56:31.642635+0000","last_undegraded":"2023-09-23T14:56:31.642635+0000","last_fullsized":"2023-09-23T14:56:31.642635+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:48.245517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.354841+0000","last_change":"2023-09-23T14:56:31.345727+0000","last_active":"2023-09-23T14:56:31.354841+0000","last_peered":"2023-09-23T14:56:31.354841+0000","last_clean":"2023-09-23T14:56:31.354841+0000","last_became_active":"2023-09-23T14:56:31.345420+0000","last_became_peered":"2023-09-23T14:56:31.345420+0000","last_unstale":"2023-09-23T14:56:31.354841+0000","last_undegraded":"2023-09-23T14:56:31.354841+0000","last_fullsized":"2023-09-23T14:56:31.354841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:01:01.054173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369526+0000","last_change":"2023-09-23T14:56:31.369043+0000","last_active":"2023-09-23T14:56:31.369526+0000","last_peered":"2023-09-23T14:56:31.369526+0000","last_clean":"2023-09-23T14:56:31.369526+0000","last_became_active":"2023-09-23T14:56:31.368739+0000","last_became_peered":"2023-09-23T14:56:31.368739+0000","last_unstale":"2023-09-23T14:56:31.369526+0000","last_undegraded":"2023-09-23T14:56:31.369526+0000","last_fullsized":"2023-09-23T14:56:31.369526+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:51.072664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356685+0000","last_change":"2023-09-23T14:56:31.341346+0000","last_active":"2023-09-23T14:56:31.356685+0000","last_peered":"2023-09-23T14:56:31.356685+0000","last_clean":"2023-09-23T14:56:31.356685+0000","last_became_active":"2023-09-23T14:56:31.341034+0000","last_became_peered":"2023-09-23T14:56:31.341034+0000","last_unstale":"2023-09-23T14:56:31.356685+0000","last_undegraded":"2023-09-23T14:56:31.356685+0000","last_fullsized":"2023-09-23T14:56:31.356685+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:34.563484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364150+0000","last_change":"2023-09-23T14:56:31.363888+0000","last_active":"2023-09-23T14:56:31.364150+0000","last_peered":"2023-09-23T14:56:31.364150+0000","last_clean":"2023-09-23T14:56:31.364150+0000","last_became_active":"2023-09-23T14:56:31.363583+0000","last_became_peered":"2023-09-23T14:56:31.363583+0000","last_unstale":"2023-09-23T14:56:31.364150+0000","last_undegraded":"2023-09-23T14:56:31.364150+0000","last_fullsized":"2023-09-23T14:56:31.364150+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:41.598918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.353942+0000","last_change":"2023-09-23T14:56:31.339544+0000","last_active":"2023-09-23T14:56:31.353942+0000","last_peered":"2023-09-23T14:56:31.353942+0000","last_clean":"2023-09-23T14:56:31.353942+0000","last_became_active":"2023-09-23T14:56:31.339267+0000","last_became_peered":"2023-09-23T14:56:31.339267+0000","last_unstale":"2023-09-23T14:56:31.353942+0000","last_undegraded":"2023-09-23T14:56:31.353942+0000","last_fullsized":"2023-09-23T14:56:31.353942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:50.763217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642769+0000","last_change":"2023-09-23T14:56:31.641417+0000","last_active":"2023-09-23T14:56:31.642769+0000","last_peered":"2023-09-23T14:56:31.642769+0000","last_clean":"2023-09-23T14:56:31.642769+0000","last_became_active":"2023-09-23T14:56:31.640911+0000","last_became_peered":"2023-09-23T14:56:31.640911+0000","last_unstale":"2023-09-23T14:56:31.642769+0000","last_undegraded":"2023-09-23T14:56:31.642769+0000","last_fullsized":"2023-09-23T14:56:31.642769+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:46.446020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364546+0000","last_change":"2023-09-23T14:56:31.336238+0000","last_active":"2023-09-23T14:56:31.364546+0000","last_peered":"2023-09-23T14:56:31.364546+0000","last_clean":"2023-09-23T14:56:31.364546+0000","last_became_active":"2023-09-23T14:56:31.335825+0000","last_became_peered":"2023-09-23T14:56:31.335825+0000","last_unstale":"2023-09-23T14:56:31.364546+0000","last_undegraded":"2023-09-23T14:56:31.364546+0000","last_fullsized":"2023-09-23T14:56:31.364546+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:29.982344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369631+0000","last_change":"2023-09-23T14:56:31.369298+0000","last_active":"2023-09-23T14:56:31.369631+0000","last_peered":"2023-09-23T14:56:31.369631+0000","last_clean":"2023-09-23T14:56:31.369631+0000","last_became_active":"2023-09-23T14:56:31.368974+0000","last_became_peered":"2023-09-23T14:56:31.368974+0000","last_unstale":"2023-09-23T14:56:31.369631+0000","last_undegraded":"2023-09-23T14:56:31.369631+0000","last_fullsized":"2023-09-23T14:56:31.369631+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:24.500525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.494439+0000","last_change":"2023-09-23T14:56:31.335057+0000","last_active":"2023-09-23T14:56:32.494439+0000","last_peered":"2023-09-23T14:56:32.494439+0000","last_clean":"2023-09-23T14:56:32.494439+0000","last_became_active":"2023-09-23T14:56:31.334371+0000","last_became_peered":"2023-09-23T14:56:31.334371+0000","last_unstale":"2023-09-23T14:56:32.494439+0000","last_undegraded":"2023-09-23T14:56:32.494439+0000","last_fullsized":"2023-09-23T14:56:32.494439+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:18.966697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201310,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6532,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738620,"statfs":{"total":95995035648,"available":95988346880,"internally_reserved":0,"allocated":1306624,"data_stored":887269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.47899999999999998,"15min":0.46999999999999997},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.49399999999999999,"15min":0.48399999999999999},"min":{"1min":0.33500000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.50600000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.51700000000000002},"min":{"1min":0.39500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.54800000000000004,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":0.63700000000000001}]},{"osd":3,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.39900000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.58499999999999996,"15min":0.57499999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48299999999999998,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.56299999999999994,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56299999999999994,"15min":0.55700000000000005},"min":{"1min":0.42199999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.81399999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57699999999999996,"15min":0.57199999999999995},"min":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462946,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":887123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70299999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.47599999999999998,"15min":0.47399999999999998},"min":{"1min":0.33400000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.69599999999999995,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.44400000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46300000000000002,"15min":0.46200000000000002},"min":{"1min":0.34300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":0.67300000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.37,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.54000000000000004}]},{"osd":2,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73799999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.40300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45400000000000001},"min":{"1min":0.371,"5min":0.312,"15min":0.312},"max":{"1min":0.67000000000000004,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45500000000000002,"15min":0.44600000000000001},"min":{"1min":0.374,"5min":0.309,"15min":0.309},"max":{"1min":0.71499999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.48799999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.749,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.497,"15min":0.49199999999999999},"min":{"1min":0.38600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76700000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 14:57:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.501,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79200000000000004,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.67600000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574904,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6756,"kb_used_data":668,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738396,"statfs":{"total":95995035648,"available":95988117504,"internally_reserved":0,"allocated":684032,"data_stored":295207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:47 2023","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"min":{"1min":0.23200000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.50600000000000001,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.373,"15min":0.35399999999999998},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.61199999999999999,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.58499999999999996}]},{"osd":2,"last update":"Sat Sep 23 14:56:10 2023","interfaces":[{"interface":"back","average":{"1min":0.35999999999999999,"5min":0.41799999999999998,"15min":0.42799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.48299999999999998,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.78900000000000003},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.41899999999999998,"15min":0.42099999999999999},"min":{"1min":0.27600000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.4470000000000001,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.29399999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.50700000000000001,"15min":0.53600000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.46200000000000002,"15min":0.48399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59899999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 14:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56100000000000005,"15min":0.57199999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.621},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.60699999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70399999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.436}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55200000000000005,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71099999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 14:57:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.65500000000000003}]}]},{"osd":0,"up_from":8,"seq":34359738429,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7412,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737740,"statfs":{"total":95995035648,"available":95987445760,"internally_reserved":0,"allocated":1290240,"data_stored":886416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 14:56:46 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.34399999999999997,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.215,"15min":0.215},"max":{"1min":0.46300000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.223},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56399999999999995,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.33500000000000002}]},{"osd":2,"last update":"Sat Sep 23 14:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.377,"15min":0.36399999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.54800000000000004,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.26100000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40500000000000003,"15min":0.39500000000000002},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.61299999999999999,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.30599999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.435,"15min":0.44700000000000001},"min":{"1min":0.26500000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52100000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.247},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.496,"15min":0.50800000000000001},"min":{"1min":0.34200000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.58899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.60499999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85699999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.84499999999999997,"5min":0.879,"15min":0.879},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84299999999999997,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.45100000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89800000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.40999999999999998}]},{"osd":6,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61099999999999999,"15min":0.61499999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91400000000000003,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.502},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.48799999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.92800000000000005,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.435,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.872,"5min":1.038,"15min":1.038},"last":0.55000000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411379,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6576,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738576,"statfs":{"total":95995035648,"available":95988301824,"internally_reserved":0,"allocated":696320,"data_stored":296060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.47799999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.375,"5min":0.437,"15min":0.443},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.49399999999999999,"5min":0.748,"15min":0.748},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41299999999999998,"15min":0.42699999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51900000000000002,"5min":0.627,"15min":0.627},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46500000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.30599999999999999}]},{"osd":3,"last update":"Sat Sep 23 14:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.45700000000000002,"15min":0.47499999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.48499999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.48299999999999998,"15min":0.52100000000000002},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.53800000000000003,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.55500000000000005,"15min":0.56699999999999995},"min":{"1min":0.378,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.84199999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.61399999999999999,"15min":0.63100000000000001},"min":{"1min":0.42399999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.71699999999999997}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.80600000000000005,"5min":1.419,"15min":1.419},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.74399999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.80400000000000005}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.628,"15min":0.63500000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.46800000000000003,"5min":0.443,"15min":0.443},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.65400000000000003}]}]},{"osd":3,"up_from":26,"seq":111669149743,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6124,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739028,"statfs":{"total":95995035648,"available":95988764672,"internally_reserved":0,"allocated":692224,"data_stored":295562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.40899999999999997,"15min":0.39200000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.4660000000000002,"5min":3.4660000000000002,"15min":3.4660000000000002},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.433,"5min":0.432,"15min":0.44400000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.6020000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":0.29599999999999999}]},{"osd":1,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44900000000000001,"15min":0.46000000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.438,"15min":0.45200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48099999999999998,"15min":0.49299999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.497,"15min":0.49299999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.8239999999999998,"5min":3.8239999999999998,"15min":3.8239999999999998},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 14:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.60499999999999998,"15min":0.61599999999999999},"min":{"1min":0.44500000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.625,"15min":0.61299999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":0.47799999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:56:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57699999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.67600000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.504,"5min":0.59399999999999997,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67500000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.57799999999999996,"15min":0.57199999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 14:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.58199999999999996,"15min":0.57099999999999995},"min":{"1min":0.44900000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.60999999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.722,"5min":3.722,"15min":3.722},"last":0.58199999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986219,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6204,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738948,"statfs":{"total":95995035648,"available":95988682752,"internally_reserved":0,"allocated":708608,"data_stored":296831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.502,"15min":0.496},"min":{"1min":0.35699999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67500000000000004,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.60099999999999998,"15min":0.56100000000000005},"min":{"1min":0.32800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.92700000000000005,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.63500000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.968,"5min":1.968,"15min":1.968},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51400000000000001,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82299999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81100000000000005,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.45900000000000002}]},{"osd":3,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54900000000000004,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78400000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.53400000000000003,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.70799999999999996}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.46800000000000003,"15min":0.46500000000000002},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.495,"5min":0.46700000000000003,"15min":0.46200000000000002},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 23 14:56:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.89700000000000002,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55600000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.755}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.625,"15min":0.626},"min":{"1min":0.45600000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.74099999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724582,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":692224,"data_stored":295773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48599999999999999,"15min":0.48699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.64800000000000002,"5min":0.747,"15min":0.747},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.35699999999999998}]},{"osd":1,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.505,"15min":0.505},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.5,"15min":0.5},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67900000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.498,"15min":0.497},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.61399999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.56799999999999995}]},{"osd":3,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.80800000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.38},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.32600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72299999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45500000000000002,"15min":0.45100000000000001},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73899999999999999,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48999999999999999,"15min":0.47999999999999998},"min":{"1min":0.42199999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.56200000000000006,"15min":0.55200000000000005},"min":{"1min":0.42999999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.61899999999999999}]},{"osd":7,"last update":"Sat Sep 23 14:57:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59099999999999997,"15min":0.58999999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.754,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64200000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T14:57:14.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T14:57:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:13 smithi027 ceph-mon[105164]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:13 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:14.399 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:14.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:14 smithi017 ceph-mon[100189]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:14.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:14 smithi017 ceph-mon[118128]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:14 smithi027 ceph-mon[105164]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:15.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:15 smithi017 ceph-mon[100189]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:15 smithi017 ceph-mon[118128]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:16.069 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:57:16.070 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T14:57:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:15 smithi027 ceph-mon[105164]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:16.768 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":64,"stamp":"2023-09-23T14:57:14.615445+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52248,"kb_used_data":7192,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749908968,"statfs":{"total":767960285184,"available":767906783232,"internally_reserved":0,"allocated":7364608,"data_stored":4140241,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002743"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.366192+0000","last_change":"2023-09-23T14:56:31.365798+0000","last_active":"2023-09-23T14:56:31.366192+0000","last_peered":"2023-09-23T14:56:31.366192+0000","last_clean":"2023-09-23T14:56:31.366192+0000","last_became_active":"2023-09-23T14:56:31.365571+0000","last_became_peered":"2023-09-23T14:56:31.365571+0000","last_unstale":"2023-09-23T14:56:31.366192+0000","last_undegraded":"2023-09-23T14:56:31.366192+0000","last_fullsized":"2023-09-23T14:56:31.366192+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:26.620629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943557+0000","last_change":"2023-09-23T14:56:31.340532+0000","last_active":"2023-09-23T14:56:38.943557+0000","last_peered":"2023-09-23T14:56:38.943557+0000","last_clean":"2023-09-23T14:56:38.943557+0000","last_became_active":"2023-09-23T14:56:31.340248+0000","last_became_peered":"2023-09-23T14:56:31.340248+0000","last_unstale":"2023-09-23T14:56:38.943557+0000","last_undegraded":"2023-09-23T14:56:38.943557+0000","last_fullsized":"2023-09-23T14:56:38.943557+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:59.101892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369065+0000","last_change":"2023-09-23T14:56:31.365130+0000","last_active":"2023-09-23T14:56:31.369065+0000","last_peered":"2023-09-23T14:56:31.369065+0000","last_clean":"2023-09-23T14:56:31.369065+0000","last_became_active":"2023-09-23T14:56:31.364858+0000","last_became_peered":"2023-09-23T14:56:31.364858+0000","last_unstale":"2023-09-23T14:56:31.369065+0000","last_undegraded":"2023-09-23T14:56:31.369065+0000","last_fullsized":"2023-09-23T14:56:31.369065+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:18.579740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.354305+0000","last_change":"2023-09-23T14:56:31.338860+0000","last_active":"2023-09-23T14:56:32.354305+0000","last_peered":"2023-09-23T14:56:32.354305+0000","last_clean":"2023-09-23T14:56:32.354305+0000","last_became_active":"2023-09-23T14:56:31.338576+0000","last_became_peered":"2023-09-23T14:56:31.338576+0000","last_unstale":"2023-09-23T14:56:32.354305+0000","last_undegraded":"2023-09-23T14:56:32.354305+0000","last_fullsized":"2023-09-23T14:56:32.354305+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:57:57.956735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364443+0000","last_change":"2023-09-23T14:56:31.364162+0000","last_active":"2023-09-23T14:56:31.364443+0000","last_peered":"2023-09-23T14:56:31.364443+0000","last_clean":"2023-09-23T14:56:31.364443+0000","last_became_active":"2023-09-23T14:56:31.363873+0000","last_became_peered":"2023-09-23T14:56:31.363873+0000","last_unstale":"2023-09-23T14:56:31.364443+0000","last_undegraded":"2023-09-23T14:56:31.364443+0000","last_fullsized":"2023-09-23T14:56:31.364443+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:45.045755+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943627+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T14:56:38.943627+0000","last_peered":"2023-09-23T14:56:38.943627+0000","last_clean":"2023-09-23T14:56:38.943627+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T14:56:38.943627+0000","last_undegraded":"2023-09-23T14:56:38.943627+0000","last_fullsized":"2023-09-23T14:56:38.943627+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365684+0000","last_change":"2023-09-23T14:56:31.365313+0000","last_active":"2023-09-23T14:56:31.365684+0000","last_peered":"2023-09-23T14:56:31.365684+0000","last_clean":"2023-09-23T14:56:31.365684+0000","last_became_active":"2023-09-23T14:56:31.365029+0000","last_became_peered":"2023-09-23T14:56:31.365029+0000","last_unstale":"2023-09-23T14:56:31.365684+0000","last_undegraded":"2023-09-23T14:56:31.365684+0000","last_fullsized":"2023-09-23T14:56:31.365684+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:55.083515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.945418+0000","last_change":"2023-09-23T14:56:31.343452+0000","last_active":"2023-09-23T14:56:38.945418+0000","last_peered":"2023-09-23T14:56:38.945418+0000","last_clean":"2023-09-23T14:56:38.945418+0000","last_became_active":"2023-09-23T14:56:31.343218+0000","last_became_peered":"2023-09-23T14:56:31.343218+0000","last_unstale":"2023-09-23T14:56:38.945418+0000","last_undegraded":"2023-09-23T14:56:38.945418+0000","last_fullsized":"2023-09-23T14:56:38.945418+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.216966+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365736+0000","last_change":"2023-09-23T14:56:31.362635+0000","last_active":"2023-09-23T14:56:31.365736+0000","last_peered":"2023-09-23T14:56:31.365736+0000","last_clean":"2023-09-23T14:56:31.365736+0000","last_became_active":"2023-09-23T14:56:31.362363+0000","last_became_peered":"2023-09-23T14:56:31.362363+0000","last_unstale":"2023-09-23T14:56:31.365736+0000","last_undegraded":"2023-09-23T14:56:31.365736+0000","last_fullsized":"2023-09-23T14:56:31.365736+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:11.181610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.355422+0000","last_change":"2023-09-23T14:56:31.333730+0000","last_active":"2023-09-23T14:56:32.355422+0000","last_peered":"2023-09-23T14:56:32.355422+0000","last_clean":"2023-09-23T14:56:32.355422+0000","last_became_active":"2023-09-23T14:56:31.333141+0000","last_became_peered":"2023-09-23T14:56:31.333141+0000","last_unstale":"2023-09-23T14:56:32.355422+0000","last_undegraded":"2023-09-23T14:56:32.355422+0000","last_fullsized":"2023-09-23T14:56:32.355422+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:36.418231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362420+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T14:56:31.362420+0000","last_peered":"2023-09-23T14:56:31.362420+0000","last_clean":"2023-09-23T14:56:31.362420+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T14:56:31.362420+0000","last_undegraded":"2023-09-23T14:56:31.362420+0000","last_fullsized":"2023-09-23T14:56:31.362420+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.355707+0000","last_change":"2023-09-23T14:56:31.337422+0000","last_active":"2023-09-23T14:56:32.355707+0000","last_peered":"2023-09-23T14:56:32.355707+0000","last_clean":"2023-09-23T14:56:32.355707+0000","last_became_active":"2023-09-23T14:56:31.337132+0000","last_became_peered":"2023-09-23T14:56:31.337132+0000","last_unstale":"2023-09-23T14:56:32.355707+0000","last_undegraded":"2023-09-23T14:56:32.355707+0000","last_fullsized":"2023-09-23T14:56:32.355707+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:45:28.461444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356314+0000","last_change":"2023-09-23T14:55:42.668313+0000","last_active":"2023-09-23T14:56:31.356314+0000","last_peered":"2023-09-23T14:56:31.356314+0000","last_clean":"2023-09-23T14:56:31.356314+0000","last_became_active":"2023-09-23T14:55:42.667931+0000","last_became_peered":"2023-09-23T14:55:42.667931+0000","last_unstale":"2023-09-23T14:56:31.356314+0000","last_undegraded":"2023-09-23T14:56:31.356314+0000","last_fullsized":"2023-09-23T14:56:31.356314+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:11:42.832634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642283+0000","last_change":"2023-09-23T14:56:31.640804+0000","last_active":"2023-09-23T14:56:31.642283+0000","last_peered":"2023-09-23T14:56:31.642283+0000","last_clean":"2023-09-23T14:56:31.642283+0000","last_became_active":"2023-09-23T14:56:31.640477+0000","last_became_peered":"2023-09-23T14:56:31.640477+0000","last_unstale":"2023-09-23T14:56:31.642283+0000","last_undegraded":"2023-09-23T14:56:31.642283+0000","last_fullsized":"2023-09-23T14:56:31.642283+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:43:38.124239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365845+0000","last_change":"2023-09-23T14:55:42.887554+0000","last_active":"2023-09-23T14:56:31.365845+0000","last_peered":"2023-09-23T14:56:31.365845+0000","last_clean":"2023-09-23T14:56:31.365845+0000","last_became_active":"2023-09-23T14:55:42.887232+0000","last_became_peered":"2023-09-23T14:55:42.887232+0000","last_unstale":"2023-09-23T14:56:31.365845+0000","last_undegraded":"2023-09-23T14:56:31.365845+0000","last_fullsized":"2023-09-23T14:56:31.365845+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:19.963176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.944474+0000","last_change":"2023-09-23T14:56:31.344484+0000","last_active":"2023-09-23T14:56:38.944474+0000","last_peered":"2023-09-23T14:56:38.944474+0000","last_clean":"2023-09-23T14:56:38.944474+0000","last_became_active":"2023-09-23T14:56:31.344177+0000","last_became_peered":"2023-09-23T14:56:31.344177+0000","last_unstale":"2023-09-23T14:56:38.944474+0000","last_undegraded":"2023-09-23T14:56:38.944474+0000","last_fullsized":"2023-09-23T14:56:38.944474+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:44.592205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642686+0000","last_change":"2023-09-23T14:56:31.640792+0000","last_active":"2023-09-23T14:56:31.642686+0000","last_peered":"2023-09-23T14:56:31.642686+0000","last_clean":"2023-09-23T14:56:31.642686+0000","last_became_active":"2023-09-23T14:56:31.640462+0000","last_became_peered":"2023-09-23T14:56:31.640462+0000","last_unstale":"2023-09-23T14:56:31.642686+0000","last_undegraded":"2023-09-23T14:56:31.642686+0000","last_fullsized":"2023-09-23T14:56:31.642686+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:09.002025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.358747+0000","last_change":"2023-09-23T14:56:31.340458+0000","last_active":"2023-09-23T14:56:31.358747+0000","last_peered":"2023-09-23T14:56:31.358747+0000","last_clean":"2023-09-23T14:56:31.358747+0000","last_became_active":"2023-09-23T14:56:31.340180+0000","last_became_peered":"2023-09-23T14:56:31.340180+0000","last_unstale":"2023-09-23T14:56:31.358747+0000","last_undegraded":"2023-09-23T14:56:31.358747+0000","last_fullsized":"2023-09-23T14:56:31.358747+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:21.662326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639978+0000","last_change":"2023-09-23T14:55:42.887686+0000","last_active":"2023-09-23T14:56:31.639978+0000","last_peered":"2023-09-23T14:56:31.639978+0000","last_clean":"2023-09-23T14:56:31.639978+0000","last_became_active":"2023-09-23T14:55:42.887225+0000","last_became_peered":"2023-09-23T14:55:42.887225+0000","last_unstale":"2023-09-23T14:56:31.639978+0000","last_undegraded":"2023-09-23T14:56:31.639978+0000","last_fullsized":"2023-09-23T14:56:31.639978+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:32.886192+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.363895+0000","last_change":"2023-09-23T14:56:31.362815+0000","last_active":"2023-09-23T14:56:31.363895+0000","last_peered":"2023-09-23T14:56:31.363895+0000","last_clean":"2023-09-23T14:56:31.363895+0000","last_became_active":"2023-09-23T14:56:31.362456+0000","last_became_peered":"2023-09-23T14:56:31.362456+0000","last_unstale":"2023-09-23T14:56:31.363895+0000","last_undegraded":"2023-09-23T14:56:31.363895+0000","last_fullsized":"2023-09-23T14:56:31.363895+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:42.421930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355527+0000","last_change":"2023-09-23T14:56:31.336392+0000","last_active":"2023-09-23T14:56:31.355527+0000","last_peered":"2023-09-23T14:56:31.355527+0000","last_clean":"2023-09-23T14:56:31.355527+0000","last_became_active":"2023-09-23T14:56:31.336144+0000","last_became_peered":"2023-09-23T14:56:31.336144+0000","last_unstale":"2023-09-23T14:56:31.355527+0000","last_undegraded":"2023-09-23T14:56:31.355527+0000","last_fullsized":"2023-09-23T14:56:31.355527+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:52.389183+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639600+0000","last_change":"2023-09-23T14:55:42.667453+0000","last_active":"2023-09-23T14:56:31.639600+0000","last_peered":"2023-09-23T14:56:31.639600+0000","last_clean":"2023-09-23T14:56:31.639600+0000","last_became_active":"2023-09-23T14:55:42.667131+0000","last_became_peered":"2023-09-23T14:55:42.667131+0000","last_unstale":"2023-09-23T14:56:31.639600+0000","last_undegraded":"2023-09-23T14:56:31.639600+0000","last_fullsized":"2023-09-23T14:56:31.639600+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:20.524459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.644325+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T14:56:31.644325+0000","last_peered":"2023-09-23T14:56:31.644325+0000","last_clean":"2023-09-23T14:56:31.644325+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T14:56:31.644325+0000","last_undegraded":"2023-09-23T14:56:31.644325+0000","last_fullsized":"2023-09-23T14:56:31.644325+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.353732+0000","last_change":"2023-09-23T14:56:31.337515+0000","last_active":"2023-09-23T14:56:31.353732+0000","last_peered":"2023-09-23T14:56:31.353732+0000","last_clean":"2023-09-23T14:56:31.353732+0000","last_became_active":"2023-09-23T14:56:31.337223+0000","last_became_peered":"2023-09-23T14:56:31.337223+0000","last_unstale":"2023-09-23T14:56:31.353732+0000","last_undegraded":"2023-09-23T14:56:31.353732+0000","last_fullsized":"2023-09-23T14:56:31.353732+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:06:44.128414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":275,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639554+0000","last_change":"2023-09-23T14:54:58.918703+0000","last_active":"2023-09-23T14:56:31.639554+0000","last_peered":"2023-09-23T14:56:31.639554+0000","last_clean":"2023-09-23T14:56:31.639554+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T14:56:31.639554+0000","last_undegraded":"2023-09-23T14:56:31.639554+0000","last_fullsized":"2023-09-23T14:56:31.639554+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:41.896866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355510+0000","last_change":"2023-09-23T14:55:42.668315+0000","last_active":"2023-09-23T14:56:31.355510+0000","last_peered":"2023-09-23T14:56:31.355510+0000","last_clean":"2023-09-23T14:56:31.355510+0000","last_became_active":"2023-09-23T14:55:42.667972+0000","last_became_peered":"2023-09-23T14:55:42.667972+0000","last_unstale":"2023-09-23T14:56:31.355510+0000","last_undegraded":"2023-09-23T14:56:31.355510+0000","last_fullsized":"2023-09-23T14:56:31.355510+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.240751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373687+0000","last_change":"2023-09-23T14:56:31.366479+0000","last_active":"2023-09-23T14:56:31.373687+0000","last_peered":"2023-09-23T14:56:31.373687+0000","last_clean":"2023-09-23T14:56:31.373687+0000","last_became_active":"2023-09-23T14:56:31.365485+0000","last_became_peered":"2023-09-23T14:56:31.365485+0000","last_unstale":"2023-09-23T14:56:31.373687+0000","last_undegraded":"2023-09-23T14:56:31.373687+0000","last_fullsized":"2023-09-23T14:56:31.373687+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:22:21.531015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364144+0000","last_change":"2023-09-23T14:56:31.334902+0000","last_active":"2023-09-23T14:56:31.364144+0000","last_peered":"2023-09-23T14:56:31.364144+0000","last_clean":"2023-09-23T14:56:31.364144+0000","last_became_active":"2023-09-23T14:56:31.334525+0000","last_became_peered":"2023-09-23T14:56:31.334525+0000","last_unstale":"2023-09-23T14:56:31.364144+0000","last_undegraded":"2023-09-23T14:56:31.364144+0000","last_fullsized":"2023-09-23T14:56:31.364144+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:24.922324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.352282+0000","last_change":"2023-09-23T14:55:42.670066+0000","last_active":"2023-09-23T14:56:31.352282+0000","last_peered":"2023-09-23T14:56:31.352282+0000","last_clean":"2023-09-23T14:56:31.352282+0000","last_became_active":"2023-09-23T14:55:42.669743+0000","last_became_peered":"2023-09-23T14:55:42.669743+0000","last_unstale":"2023-09-23T14:56:31.352282+0000","last_undegraded":"2023-09-23T14:56:31.352282+0000","last_fullsized":"2023-09-23T14:56:31.352282+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:17.259363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373938+0000","last_change":"2023-09-23T14:56:31.369934+0000","last_active":"2023-09-23T14:56:31.373938+0000","last_peered":"2023-09-23T14:56:31.373938+0000","last_clean":"2023-09-23T14:56:31.373938+0000","last_became_active":"2023-09-23T14:56:31.369753+0000","last_became_peered":"2023-09-23T14:56:31.369753+0000","last_unstale":"2023-09-23T14:56:31.373938+0000","last_undegraded":"2023-09-23T14:56:31.373938+0000","last_fullsized":"2023-09-23T14:56:31.373938+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:03.998336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356704+0000","last_change":"2023-09-23T14:56:31.337354+0000","last_active":"2023-09-23T14:56:31.356704+0000","last_peered":"2023-09-23T14:56:31.356704+0000","last_clean":"2023-09-23T14:56:31.356704+0000","last_became_active":"2023-09-23T14:56:31.336634+0000","last_became_peered":"2023-09-23T14:56:31.336634+0000","last_unstale":"2023-09-23T14:56:31.356704+0000","last_undegraded":"2023-09-23T14:56:31.356704+0000","last_fullsized":"2023-09-23T14:56:31.356704+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:30.083470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355064+0000","last_change":"2023-09-23T14:55:42.666173+0000","last_active":"2023-09-23T14:56:31.355064+0000","last_peered":"2023-09-23T14:56:31.355064+0000","last_clean":"2023-09-23T14:56:31.355064+0000","last_became_active":"2023-09-23T14:55:42.665827+0000","last_became_peered":"2023-09-23T14:55:42.665827+0000","last_unstale":"2023-09-23T14:56:31.355064+0000","last_undegraded":"2023-09-23T14:56:31.355064+0000","last_fullsized":"2023-09-23T14:56:31.355064+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:29.577101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373715+0000","last_change":"2023-09-23T14:56:31.367210+0000","last_active":"2023-09-23T14:56:31.373715+0000","last_peered":"2023-09-23T14:56:31.373715+0000","last_clean":"2023-09-23T14:56:31.373715+0000","last_became_active":"2023-09-23T14:56:31.366915+0000","last_became_peered":"2023-09-23T14:56:31.366915+0000","last_unstale":"2023-09-23T14:56:31.373715+0000","last_undegraded":"2023-09-23T14:56:31.373715+0000","last_fullsized":"2023-09-23T14:56:31.373715+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:30.521148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355346+0000","last_change":"2023-09-23T14:56:31.344343+0000","last_active":"2023-09-23T14:56:31.355346+0000","last_peered":"2023-09-23T14:56:31.355346+0000","last_clean":"2023-09-23T14:56:31.355346+0000","last_became_active":"2023-09-23T14:56:31.344105+0000","last_became_peered":"2023-09-23T14:56:31.344105+0000","last_unstale":"2023-09-23T14:56:31.355346+0000","last_undegraded":"2023-09-23T14:56:31.355346+0000","last_fullsized":"2023-09-23T14:56:31.355346+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:15.458054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":15,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365118+0000","last_change":"2023-09-23T14:55:42.665607+0000","last_active":"2023-09-23T14:56:31.365118+0000","last_peered":"2023-09-23T14:56:31.365118+0000","last_clean":"2023-09-23T14:56:31.365118+0000","last_became_active":"2023-09-23T14:55:42.665279+0000","last_became_peered":"2023-09-23T14:55:42.665279+0000","last_unstale":"2023-09-23T14:56:31.365118+0000","last_undegraded":"2023-09-23T14:56:31.365118+0000","last_fullsized":"2023-09-23T14:56:31.365118+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:58.018948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373782+0000","last_change":"2023-09-23T14:56:31.369890+0000","last_active":"2023-09-23T14:56:31.373782+0000","last_peered":"2023-09-23T14:56:31.373782+0000","last_clean":"2023-09-23T14:56:31.373782+0000","last_became_active":"2023-09-23T14:56:31.369618+0000","last_became_peered":"2023-09-23T14:56:31.369618+0000","last_unstale":"2023-09-23T14:56:31.373782+0000","last_undegraded":"2023-09-23T14:56:31.373782+0000","last_fullsized":"2023-09-23T14:56:31.373782+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:56.163148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.354214+0000","last_change":"2023-09-23T14:56:31.343731+0000","last_active":"2023-09-23T14:56:31.354214+0000","last_peered":"2023-09-23T14:56:31.354214+0000","last_clean":"2023-09-23T14:56:31.354214+0000","last_became_active":"2023-09-23T14:56:31.343418+0000","last_became_peered":"2023-09-23T14:56:31.343418+0000","last_unstale":"2023-09-23T14:56:31.354214+0000","last_undegraded":"2023-09-23T14:56:31.354214+0000","last_fullsized":"2023-09-23T14:56:31.354214+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:25.986640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362943+0000","last_change":"2023-09-23T14:56:31.362550+0000","last_active":"2023-09-23T14:56:31.362943+0000","last_peered":"2023-09-23T14:56:31.362943+0000","last_clean":"2023-09-23T14:56:31.362943+0000","last_became_active":"2023-09-23T14:56:31.362261+0000","last_became_peered":"2023-09-23T14:56:31.362261+0000","last_unstale":"2023-09-23T14:56:31.362943+0000","last_undegraded":"2023-09-23T14:56:31.362943+0000","last_fullsized":"2023-09-23T14:56:31.362943+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:26.597585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.357503+0000","last_change":"2023-09-23T14:56:31.338927+0000","last_active":"2023-09-23T14:56:31.357503+0000","last_peered":"2023-09-23T14:56:31.357503+0000","last_clean":"2023-09-23T14:56:31.357503+0000","last_became_active":"2023-09-23T14:56:31.338287+0000","last_became_peered":"2023-09-23T14:56:31.338287+0000","last_unstale":"2023-09-23T14:56:31.357503+0000","last_undegraded":"2023-09-23T14:56:31.357503+0000","last_fullsized":"2023-09-23T14:56:31.357503+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:43.291216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.361924+0000","last_change":"2023-09-23T14:56:31.361613+0000","last_active":"2023-09-23T14:56:31.361924+0000","last_peered":"2023-09-23T14:56:31.361924+0000","last_clean":"2023-09-23T14:56:31.361924+0000","last_became_active":"2023-09-23T14:56:31.361427+0000","last_became_peered":"2023-09-23T14:56:31.361427+0000","last_unstale":"2023-09-23T14:56:31.361924+0000","last_undegraded":"2023-09-23T14:56:31.361924+0000","last_fullsized":"2023-09-23T14:56:31.361924+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:50.247243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356315+0000","last_change":"2023-09-23T14:56:31.333503+0000","last_active":"2023-09-23T14:56:31.356315+0000","last_peered":"2023-09-23T14:56:31.356315+0000","last_clean":"2023-09-23T14:56:31.356315+0000","last_became_active":"2023-09-23T14:56:31.333045+0000","last_became_peered":"2023-09-23T14:56:31.333045+0000","last_unstale":"2023-09-23T14:56:31.356315+0000","last_undegraded":"2023-09-23T14:56:31.356315+0000","last_fullsized":"2023-09-23T14:56:31.356315+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:37:20.987619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.361877+0000","last_change":"2023-09-23T14:56:31.360820+0000","last_active":"2023-09-23T14:56:31.361877+0000","last_peered":"2023-09-23T14:56:31.361877+0000","last_clean":"2023-09-23T14:56:31.361877+0000","last_became_active":"2023-09-23T14:56:31.360436+0000","last_became_peered":"2023-09-23T14:56:31.360436+0000","last_unstale":"2023-09-23T14:56:31.361877+0000","last_undegraded":"2023-09-23T14:56:31.361877+0000","last_fullsized":"2023-09-23T14:56:31.361877+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:13.769760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.943417+0000","last_change":"2023-09-23T14:56:31.343639+0000","last_active":"2023-09-23T14:56:38.943417+0000","last_peered":"2023-09-23T14:56:38.943417+0000","last_clean":"2023-09-23T14:56:38.943417+0000","last_became_active":"2023-09-23T14:56:31.343307+0000","last_became_peered":"2023-09-23T14:56:31.343307+0000","last_unstale":"2023-09-23T14:56:38.943417+0000","last_undegraded":"2023-09-23T14:56:38.943417+0000","last_fullsized":"2023-09-23T14:56:38.943417+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:47.909520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.368155+0000","last_change":"2023-09-23T14:56:31.366975+0000","last_active":"2023-09-23T14:56:31.368155+0000","last_peered":"2023-09-23T14:56:31.368155+0000","last_clean":"2023-09-23T14:56:31.368155+0000","last_became_active":"2023-09-23T14:56:31.366669+0000","last_became_peered":"2023-09-23T14:56:31.366669+0000","last_unstale":"2023-09-23T14:56:31.368155+0000","last_undegraded":"2023-09-23T14:56:31.368155+0000","last_fullsized":"2023-09-23T14:56:31.368155+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:32.225320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.355638+0000","last_change":"2023-09-23T14:56:31.342259+0000","last_active":"2023-09-23T14:56:31.355638+0000","last_peered":"2023-09-23T14:56:31.355638+0000","last_clean":"2023-09-23T14:56:31.355638+0000","last_became_active":"2023-09-23T14:56:31.341657+0000","last_became_peered":"2023-09-23T14:56:31.341657+0000","last_unstale":"2023-09-23T14:56:31.355638+0000","last_undegraded":"2023-09-23T14:56:31.355638+0000","last_fullsized":"2023-09-23T14:56:31.355638+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:52.375439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.641216+0000","last_change":"2023-09-23T14:56:31.640765+0000","last_active":"2023-09-23T14:56:31.641216+0000","last_peered":"2023-09-23T14:56:31.641216+0000","last_clean":"2023-09-23T14:56:31.641216+0000","last_became_active":"2023-09-23T14:56:31.640435+0000","last_became_peered":"2023-09-23T14:56:31.640435+0000","last_unstale":"2023-09-23T14:56:31.641216+0000","last_undegraded":"2023-09-23T14:56:31.641216+0000","last_fullsized":"2023-09-23T14:56:31.641216+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:51.676196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.496280+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T14:56:32.496280+0000","last_peered":"2023-09-23T14:56:32.496280+0000","last_clean":"2023-09-23T14:56:32.496280+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T14:56:32.496280+0000","last_undegraded":"2023-09-23T14:56:32.496280+0000","last_fullsized":"2023-09-23T14:56:32.496280+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.362942+0000","last_change":"2023-09-23T14:56:31.362551+0000","last_active":"2023-09-23T14:56:31.362942+0000","last_peered":"2023-09-23T14:56:31.362942+0000","last_clean":"2023-09-23T14:56:31.362942+0000","last_became_active":"2023-09-23T14:56:31.362240+0000","last_became_peered":"2023-09-23T14:56:31.362240+0000","last_unstale":"2023-09-23T14:56:31.362942+0000","last_undegraded":"2023-09-23T14:56:31.362942+0000","last_fullsized":"2023-09-23T14:56:31.362942+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:54.851534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.946505+0000","last_change":"2023-09-23T14:56:31.343468+0000","last_active":"2023-09-23T14:56:38.946505+0000","last_peered":"2023-09-23T14:56:38.946505+0000","last_clean":"2023-09-23T14:56:38.946505+0000","last_became_active":"2023-09-23T14:56:31.342156+0000","last_became_peered":"2023-09-23T14:56:31.342156+0000","last_unstale":"2023-09-23T14:56:38.946505+0000","last_undegraded":"2023-09-23T14:56:38.946505+0000","last_fullsized":"2023-09-23T14:56:38.946505+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:44.209095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.641103+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T14:56:31.641103+0000","last_peered":"2023-09-23T14:56:31.641103+0000","last_clean":"2023-09-23T14:56:31.641103+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T14:56:31.641103+0000","last_undegraded":"2023-09-23T14:56:31.641103+0000","last_fullsized":"2023-09-23T14:56:31.641103+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.496355+0000","last_change":"2023-09-23T14:56:31.341739+0000","last_active":"2023-09-23T14:56:32.496355+0000","last_peered":"2023-09-23T14:56:32.496355+0000","last_clean":"2023-09-23T14:56:32.496355+0000","last_became_active":"2023-09-23T14:56:31.340906+0000","last_became_peered":"2023-09-23T14:56:31.340906+0000","last_unstale":"2023-09-23T14:56:32.496355+0000","last_undegraded":"2023-09-23T14:56:32.496355+0000","last_fullsized":"2023-09-23T14:56:32.496355+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:20.857073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642208+0000","last_change":"2023-09-23T14:56:31.640894+0000","last_active":"2023-09-23T14:56:31.642208+0000","last_peered":"2023-09-23T14:56:31.642208+0000","last_clean":"2023-09-23T14:56:31.642208+0000","last_became_active":"2023-09-23T14:56:31.640576+0000","last_became_peered":"2023-09-23T14:56:31.640576+0000","last_unstale":"2023-09-23T14:56:31.642208+0000","last_undegraded":"2023-09-23T14:56:31.642208+0000","last_fullsized":"2023-09-23T14:56:31.642208+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:43.572217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.947195+0000","last_change":"2023-09-23T14:56:31.347827+0000","last_active":"2023-09-23T14:56:38.947195+0000","last_peered":"2023-09-23T14:56:38.947195+0000","last_clean":"2023-09-23T14:56:38.947195+0000","last_became_active":"2023-09-23T14:56:31.347502+0000","last_became_peered":"2023-09-23T14:56:31.347502+0000","last_unstale":"2023-09-23T14:56:38.947195+0000","last_undegraded":"2023-09-23T14:56:38.947195+0000","last_fullsized":"2023-09-23T14:56:38.947195+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:32.331497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642757+0000","last_change":"2023-09-23T14:56:31.641209+0000","last_active":"2023-09-23T14:56:31.642757+0000","last_peered":"2023-09-23T14:56:31.642757+0000","last_clean":"2023-09-23T14:56:31.642757+0000","last_became_active":"2023-09-23T14:56:31.640477+0000","last_became_peered":"2023-09-23T14:56:31.640477+0000","last_unstale":"2023-09-23T14:56:31.642757+0000","last_undegraded":"2023-09-23T14:56:31.642757+0000","last_fullsized":"2023-09-23T14:56:31.642757+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:24.082906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.639471+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T14:56:31.639471+0000","last_peered":"2023-09-23T14:56:31.639471+0000","last_clean":"2023-09-23T14:56:31.639471+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T14:56:31.639471+0000","last_undegraded":"2023-09-23T14:56:31.639471+0000","last_fullsized":"2023-09-23T14:56:31.639471+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.643030+0000","last_change":"2023-09-23T14:56:31.642178+0000","last_active":"2023-09-23T14:56:31.643030+0000","last_peered":"2023-09-23T14:56:31.643030+0000","last_clean":"2023-09-23T14:56:31.643030+0000","last_became_active":"2023-09-23T14:56:31.641822+0000","last_became_peered":"2023-09-23T14:56:31.641822+0000","last_unstale":"2023-09-23T14:56:31.643030+0000","last_undegraded":"2023-09-23T14:56:31.643030+0000","last_fullsized":"2023-09-23T14:56:31.643030+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:23:36.187232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356429+0000","last_change":"2023-09-23T14:56:31.332428+0000","last_active":"2023-09-23T14:56:31.356429+0000","last_peered":"2023-09-23T14:56:31.356429+0000","last_clean":"2023-09-23T14:56:31.356429+0000","last_became_active":"2023-09-23T14:56:31.332128+0000","last_became_peered":"2023-09-23T14:56:31.332128+0000","last_unstale":"2023-09-23T14:56:31.356429+0000","last_undegraded":"2023-09-23T14:56:31.356429+0000","last_fullsized":"2023-09-23T14:56:31.356429+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:13.596995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642257+0000","last_change":"2023-09-23T14:56:31.641239+0000","last_active":"2023-09-23T14:56:31.642257+0000","last_peered":"2023-09-23T14:56:31.642257+0000","last_clean":"2023-09-23T14:56:31.642257+0000","last_became_active":"2023-09-23T14:56:31.640888+0000","last_became_peered":"2023-09-23T14:56:31.640888+0000","last_unstale":"2023-09-23T14:56:31.642257+0000","last_undegraded":"2023-09-23T14:56:31.642257+0000","last_fullsized":"2023-09-23T14:56:31.642257+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:11.816464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.947454+0000","last_change":"2023-09-23T14:56:31.341608+0000","last_active":"2023-09-23T14:56:38.947454+0000","last_peered":"2023-09-23T14:56:38.947454+0000","last_clean":"2023-09-23T14:56:38.947454+0000","last_became_active":"2023-09-23T14:56:31.340802+0000","last_became_peered":"2023-09-23T14:56:31.340802+0000","last_unstale":"2023-09-23T14:56:38.947454+0000","last_undegraded":"2023-09-23T14:56:38.947454+0000","last_fullsized":"2023-09-23T14:56:38.947454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:21.513338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.365701+0000","last_change":"2023-09-23T14:56:31.365386+0000","last_active":"2023-09-23T14:56:31.365701+0000","last_peered":"2023-09-23T14:56:31.365701+0000","last_clean":"2023-09-23T14:56:31.365701+0000","last_became_active":"2023-09-23T14:56:31.365100+0000","last_became_peered":"2023-09-23T14:56:31.365100+0000","last_unstale":"2023-09-23T14:56:31.365701+0000","last_undegraded":"2023-09-23T14:56:31.365701+0000","last_fullsized":"2023-09-23T14:56:31.365701+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:32.427060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"68'6","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.948362+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T14:56:38.948362+0000","last_peered":"2023-09-23T14:56:38.948362+0000","last_clean":"2023-09-23T14:56:38.948362+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T14:56:38.948362+0000","last_undegraded":"2023-09-23T14:56:38.948362+0000","last_fullsized":"2023-09-23T14:56:38.948362+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.373693+0000","last_change":"2023-09-23T14:56:31.366542+0000","last_active":"2023-09-23T14:56:31.373693+0000","last_peered":"2023-09-23T14:56:31.373693+0000","last_clean":"2023-09-23T14:56:31.373693+0000","last_became_active":"2023-09-23T14:56:31.366224+0000","last_became_peered":"2023-09-23T14:56:31.366224+0000","last_unstale":"2023-09-23T14:56:31.373693+0000","last_undegraded":"2023-09-23T14:56:31.373693+0000","last_fullsized":"2023-09-23T14:56:31.373693+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:35.698729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:38.946571+0000","last_change":"2023-09-23T14:56:31.342751+0000","last_active":"2023-09-23T14:56:38.946571+0000","last_peered":"2023-09-23T14:56:38.946571+0000","last_clean":"2023-09-23T14:56:38.946571+0000","last_became_active":"2023-09-23T14:56:31.342324+0000","last_became_peered":"2023-09-23T14:56:31.342324+0000","last_unstale":"2023-09-23T14:56:38.946571+0000","last_undegraded":"2023-09-23T14:56:38.946571+0000","last_fullsized":"2023-09-23T14:56:38.946571+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:26.082641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642635+0000","last_change":"2023-09-23T14:56:31.641933+0000","last_active":"2023-09-23T14:56:31.642635+0000","last_peered":"2023-09-23T14:56:31.642635+0000","last_clean":"2023-09-23T14:56:31.642635+0000","last_became_active":"2023-09-23T14:56:31.641472+0000","last_became_peered":"2023-09-23T14:56:31.641472+0000","last_unstale":"2023-09-23T14:56:31.642635+0000","last_undegraded":"2023-09-23T14:56:31.642635+0000","last_fullsized":"2023-09-23T14:56:31.642635+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:48.245517+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.354841+0000","last_change":"2023-09-23T14:56:31.345727+0000","last_active":"2023-09-23T14:56:31.354841+0000","last_peered":"2023-09-23T14:56:31.354841+0000","last_clean":"2023-09-23T14:56:31.354841+0000","last_became_active":"2023-09-23T14:56:31.345420+0000","last_became_peered":"2023-09-23T14:56:31.345420+0000","last_unstale":"2023-09-23T14:56:31.354841+0000","last_undegraded":"2023-09-23T14:56:31.354841+0000","last_fullsized":"2023-09-23T14:56:31.354841+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:01:01.054173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369526+0000","last_change":"2023-09-23T14:56:31.369043+0000","last_active":"2023-09-23T14:56:31.369526+0000","last_peered":"2023-09-23T14:56:31.369526+0000","last_clean":"2023-09-23T14:56:31.369526+0000","last_became_active":"2023-09-23T14:56:31.368739+0000","last_became_peered":"2023-09-23T14:56:31.368739+0000","last_unstale":"2023-09-23T14:56:31.369526+0000","last_undegraded":"2023-09-23T14:56:31.369526+0000","last_fullsized":"2023-09-23T14:56:31.369526+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:51.072664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.356685+0000","last_change":"2023-09-23T14:56:31.341346+0000","last_active":"2023-09-23T14:56:31.356685+0000","last_peered":"2023-09-23T14:56:31.356685+0000","last_clean":"2023-09-23T14:56:31.356685+0000","last_became_active":"2023-09-23T14:56:31.341034+0000","last_became_peered":"2023-09-23T14:56:31.341034+0000","last_unstale":"2023-09-23T14:56:31.356685+0000","last_undegraded":"2023-09-23T14:56:31.356685+0000","last_fullsized":"2023-09-23T14:56:31.356685+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:34.563484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364150+0000","last_change":"2023-09-23T14:56:31.363888+0000","last_active":"2023-09-23T14:56:31.364150+0000","last_peered":"2023-09-23T14:56:31.364150+0000","last_clean":"2023-09-23T14:56:31.364150+0000","last_became_active":"2023-09-23T14:56:31.363583+0000","last_became_peered":"2023-09-23T14:56:31.363583+0000","last_unstale":"2023-09-23T14:56:31.364150+0000","last_undegraded":"2023-09-23T14:56:31.364150+0000","last_fullsized":"2023-09-23T14:56:31.364150+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:41.598918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.353942+0000","last_change":"2023-09-23T14:56:31.339544+0000","last_active":"2023-09-23T14:56:31.353942+0000","last_peered":"2023-09-23T14:56:31.353942+0000","last_clean":"2023-09-23T14:56:31.353942+0000","last_became_active":"2023-09-23T14:56:31.339267+0000","last_became_peered":"2023-09-23T14:56:31.339267+0000","last_unstale":"2023-09-23T14:56:31.353942+0000","last_undegraded":"2023-09-23T14:56:31.353942+0000","last_fullsized":"2023-09-23T14:56:31.353942+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:50.763217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.642769+0000","last_change":"2023-09-23T14:56:31.641417+0000","last_active":"2023-09-23T14:56:31.642769+0000","last_peered":"2023-09-23T14:56:31.642769+0000","last_clean":"2023-09-23T14:56:31.642769+0000","last_became_active":"2023-09-23T14:56:31.640911+0000","last_became_peered":"2023-09-23T14:56:31.640911+0000","last_unstale":"2023-09-23T14:56:31.642769+0000","last_undegraded":"2023-09-23T14:56:31.642769+0000","last_fullsized":"2023-09-23T14:56:31.642769+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:46.446020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":11,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.364546+0000","last_change":"2023-09-23T14:56:31.336238+0000","last_active":"2023-09-23T14:56:31.364546+0000","last_peered":"2023-09-23T14:56:31.364546+0000","last_clean":"2023-09-23T14:56:31.364546+0000","last_became_active":"2023-09-23T14:56:31.335825+0000","last_became_peered":"2023-09-23T14:56:31.335825+0000","last_unstale":"2023-09-23T14:56:31.364546+0000","last_undegraded":"2023-09-23T14:56:31.364546+0000","last_fullsized":"2023-09-23T14:56:31.364546+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:29.982344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":10,"reported_epoch":67,"state":"active+clean","last_fresh":"2023-09-23T14:56:31.369631+0000","last_change":"2023-09-23T14:56:31.369298+0000","last_active":"2023-09-23T14:56:31.369631+0000","last_peered":"2023-09-23T14:56:31.369631+0000","last_clean":"2023-09-23T14:56:31.369631+0000","last_became_active":"2023-09-23T14:56:31.368974+0000","last_became_peered":"2023-09-23T14:56:31.368974+0000","last_unstale":"2023-09-23T14:56:31.369631+0000","last_undegraded":"2023-09-23T14:56:31.369631+0000","last_fullsized":"2023-09-23T14:56:31.369631+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:24.500525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":12,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T14:56:32.494439+0000","last_change":"2023-09-23T14:56:31.335057+0000","last_active":"2023-09-23T14:56:32.494439+0000","last_peered":"2023-09-23T14:56:32.494439+0000","last_clean":"2023-09-23T14:56:32.494439+0000","last_became_active":"2023-09-23T14:56:31.334371+0000","last_became_peered":"2023-09-23T14:56:31.334371+0000","last_unstale":"2023-09-23T14:56:32.494439+0000","last_undegraded":"2023-09-23T14:56:32.494439+0000","last_fullsized":"2023-09-23T14:56:32.494439+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:18.966697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201311,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6532,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93738620,"statfs":{"total":95995035648,"available":95988346880,"internally_reserved":0,"allocated":1306624,"data_stored":887269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.47899999999999998,"15min":0.46999999999999997},"min":{"1min":0.36899999999999999,"5min":0.311,"15min":0.311},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.49399999999999999,"15min":0.48399999999999999},"min":{"1min":0.33500000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.50600000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.52500000000000002,"15min":0.51700000000000002},"min":{"1min":0.39500000000000002,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.57899999999999996,"5min":0.54100000000000004,"15min":0.53500000000000003},"min":{"1min":0.36799999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.51500000000000001,"15min":0.51100000000000001},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.54800000000000004,"15min":0.53200000000000003},"min":{"1min":0.41699999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.337,"5min":1.337,"15min":1.337},"last":0.63700000000000001}]},{"osd":3,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.60399999999999998,"15min":0.60799999999999998},"min":{"1min":0.39900000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1,"5min":1.7529999999999999,"15min":1.7529999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.64400000000000002,"5min":0.58499999999999996,"15min":0.57499999999999996},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.55100000000000005}]},{"osd":4,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.48299999999999998,"15min":0.47299999999999998},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.87,"5min":0.87,"15min":0.87},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.51600000000000001,"15min":0.51100000000000001},"min":{"1min":0.37,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.56299999999999994,"15min":0.55700000000000005},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.56200000000000006,"15min":0.55600000000000005},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 23 14:56:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.56299999999999994,"15min":0.55700000000000005},"min":{"1min":0.42199999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.81399999999999995,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.57699999999999996,"15min":0.57199999999999995},"min":{"1min":0.40899999999999997,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.84799999999999998,"5min":0.84799999999999998,"15min":0.84799999999999998},"last":0.58399999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462947,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6584,"kb_used_data":1264,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738568,"statfs":{"total":95995035648,"available":95988293632,"internally_reserved":0,"allocated":1294336,"data_stored":887123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.48499999999999999,"15min":0.47899999999999998},"min":{"1min":0.34100000000000003,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70299999999999996,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.47599999999999998,"15min":0.47399999999999998},"min":{"1min":0.33400000000000002,"5min":0.316,"15min":0.316},"max":{"1min":0.69599999999999995,"5min":0.72899999999999998,"15min":0.72899999999999998},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.46300000000000002,"15min":0.46200000000000002},"min":{"1min":0.34300000000000003,"5min":0.312,"15min":0.312},"max":{"1min":0.67300000000000004,"5min":0.68999999999999995,"15min":0.68999999999999995},"last":0.504},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.495,"15min":0.49099999999999999},"min":{"1min":0.37,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.63500000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.73799999999999999,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.495,"15min":0.48899999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.53700000000000003}]},{"osd":3,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.49399999999999999,"15min":0.49199999999999999},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.55100000000000005,"15min":0.54700000000000004},"min":{"1min":0.40300000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.85399999999999998,"5min":0.85399999999999998,"15min":0.85399999999999998},"last":0.58599999999999997}]},{"osd":4,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.46200000000000002,"15min":0.45400000000000001},"min":{"1min":0.371,"5min":0.312,"15min":0.312},"max":{"1min":0.67000000000000004,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.50800000000000001,"5min":0.45500000000000002,"15min":0.44600000000000001},"min":{"1min":0.374,"5min":0.309,"15min":0.309},"max":{"1min":0.71499999999999997,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.42699999999999999}]},{"osd":5,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.48799999999999999,"15min":0.48399999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.749,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.497,"15min":0.49199999999999999},"min":{"1min":0.38600000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76700000000000002,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.65000000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:57:02 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.501,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":0.79200000000000004,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.61599999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574905,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6756,"kb_used_data":668,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738396,"statfs":{"total":95995035648,"available":95988117504,"internally_reserved":0,"allocated":684032,"data_stored":295207,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:47 2023","interfaces":[{"interface":"back","average":{"1min":0.32600000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"min":{"1min":0.23200000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.50600000000000001,"5min":0.52000000000000002,"15min":0.52000000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.373,"15min":0.35399999999999998},"min":{"1min":0.28499999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.61199999999999999,"5min":0.69699999999999995,"15min":0.69699999999999995},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 14:57:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":0.41299999999999998,"15min":0.42599999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.34300000000000003,"5min":0.40300000000000002,"15min":0.41499999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.55900000000000005,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.28699999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.50700000000000001,"15min":0.53600000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.46200000000000002,"15min":0.48399999999999999},"min":{"1min":0.30199999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.59899999999999998,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.32000000000000001}]},{"osd":4,"last update":"Sat Sep 23 14:56:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56100000000000005,"15min":0.57199999999999995},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54400000000000004,"15min":0.54600000000000004},"min":{"1min":0.39100000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.54300000000000004,"15min":0.54800000000000004},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70399999999999996,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.41199999999999998,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":1.206,"5min":1.206,"15min":1.206},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55200000000000005,"15min":0.54800000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":1.224,"5min":1.224,"15min":1.224},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.55000000000000004,"5min":0.55400000000000005,"15min":0.55500000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.71099999999999997,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 14:57:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59699999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.175,"5min":1.175,"15min":1.175},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.64000000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738430,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7412,"kb_used_data":1260,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737740,"statfs":{"total":95995035648,"available":95987445760,"internally_reserved":0,"allocated":1290240,"data_stored":886416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 14:56:46 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.34399999999999997,"15min":0.32700000000000001},"min":{"1min":0.216,"5min":0.215,"15min":0.215},"max":{"1min":0.46300000000000002,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.34899999999999998,"5min":0.36799999999999999,"15min":0.35399999999999998},"min":{"1min":0.25700000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56399999999999995,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Sep 23 14:57:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.379,"15min":0.36499999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.40100000000000002,"5min":0.40699999999999997,"15min":0.39500000000000002},"min":{"1min":0.28799999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.63200000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.52600000000000002}]},{"osd":3,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":0.435,"15min":0.44700000000000001},"min":{"1min":0.26500000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.52100000000000002,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.45200000000000001,"5min":0.496,"15min":0.50800000000000001},"min":{"1min":0.34200000000000003,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.58899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.59499999999999997,"15min":0.60499999999999998},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.85699999999999998,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.42999999999999999,"5min":0.37,"15min":0.37},"max":{"1min":0.84499999999999997,"5min":0.879,"15min":0.879},"last":0.76100000000000001}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.59199999999999997,"15min":0.59499999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.84299999999999997,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.621,"15min":0.623},"min":{"1min":0.45100000000000001,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.89800000000000002,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.85199999999999998}]},{"osd":6,"last update":"Sat Sep 23 14:56:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.61099999999999999,"15min":0.61499999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.91400000000000003,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.89200000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.64800000000000002,"15min":0.65000000000000002},"min":{"1min":0.48799999999999999,"5min":0.434,"15min":0.434},"max":{"1min":0.92800000000000005,"5min":1.0109999999999999,"15min":1.0109999999999999},"last":0.90100000000000002}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.877,"5min":0.877,"15min":0.877},"last":0.86399999999999999},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.59099999999999997,"15min":0.59399999999999997},"min":{"1min":0.435,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":0.872,"5min":1.038,"15min":1.038},"last":0.73899999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411380,"num_pgs":17,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6576,"kb_used_data":680,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738576,"statfs":{"total":95995035648,"available":95988301824,"internally_reserved":0,"allocated":696320,"data_stored":296060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.36299999999999999,"5min":0.39800000000000002,"15min":0.40799999999999997},"min":{"1min":0.31,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.47799999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.375,"5min":0.437,"15min":0.443},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.49399999999999999,"5min":0.748,"15min":0.748},"last":0.38600000000000001}]},{"osd":1,"last update":"Sat Sep 23 14:56:12 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.41299999999999998,"15min":0.42699999999999999},"min":{"1min":0.249,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.51900000000000002,"5min":0.627,"15min":0.627},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.34100000000000003,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.46500000000000002,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 14:56:33 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.45700000000000002,"15min":0.47499999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.48499999999999999,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.48299999999999998,"15min":0.52100000000000002},"min":{"1min":0.29099999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.53800000000000003,"5min":0.95099999999999996,"15min":0.95099999999999996},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.55500000000000005,"15min":0.56699999999999995},"min":{"1min":0.378,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78400000000000003,"5min":0.84699999999999998,"15min":0.84699999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.61399999999999999,"15min":0.63100000000000001},"min":{"1min":0.42399999999999999,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.0189999999999999,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.60299999999999998}]},{"osd":5,"last update":"Sat Sep 23 14:56:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.61599999999999999,"15min":0.626},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.80600000000000005,"5min":1.419,"15min":1.419},"last":0.435},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.60099999999999998,"15min":0.61099999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.74399999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.55200000000000005}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.60599999999999998,"15min":0.61399999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.628,"15min":0.63500000000000001},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":1.117,"5min":1.117,"15min":1.117},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.60599999999999998,"15min":0.60399999999999998},"min":{"1min":0.46800000000000003,"5min":0.443,"15min":0.443},"max":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.60499999999999998,"15min":0.60599999999999998},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":0.52200000000000002}]}]},{"osd":3,"up_from":26,"seq":111669149744,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6124,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739028,"statfs":{"total":95995035648,"available":95988764672,"internally_reserved":0,"allocated":692224,"data_stored":295562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.40899999999999997,"15min":0.39200000000000002},"min":{"1min":0.20399999999999999,"5min":0.20399999999999999,"15min":0.20399999999999999},"max":{"1min":3.4660000000000002,"5min":3.4660000000000002,"15min":3.4660000000000002},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.433,"5min":0.432,"15min":0.44400000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":1.6020000000000001,"5min":1.6020000000000001,"15min":1.6020000000000001},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.44900000000000001,"15min":0.46000000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":1.3799999999999999,"5min":1.3799999999999999,"15min":1.3799999999999999},"last":0.308},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.438,"15min":0.45200000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48099999999999998,"15min":0.49299999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":1.1830000000000001,"5min":1.1830000000000001,"15min":1.1830000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.497,"15min":0.49299999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.8239999999999998,"5min":3.8239999999999998,"15min":3.8239999999999998},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 14:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.60499999999999998,"15min":0.61599999999999999},"min":{"1min":0.44500000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2190000000000001,"5min":1.2190000000000001,"15min":1.2190000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.625,"15min":0.61299999999999999},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":3.7919999999999998,"5min":3.7919999999999998,"15min":3.7919999999999998},"last":0.64900000000000002}]},{"osd":5,"last update":"Sat Sep 23 14:56:14 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.57699999999999996,"15min":0.58699999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.67600000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.504,"5min":0.59399999999999997,"15min":0.60899999999999999},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.67500000000000004,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 23 14:56:37 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.57799999999999996,"15min":0.57199999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.425,"5min":1.425,"15min":1.425},"last":0.76800000000000002},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.63800000000000001,"15min":0.63800000000000001},"min":{"1min":0.46800000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":0.78900000000000003}]},{"osd":7,"last update":"Sat Sep 23 14:57:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":0.58199999999999996,"15min":0.57099999999999995},"min":{"1min":0.44900000000000001,"5min":0.437,"15min":0.437},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.79500000000000004,"5min":0.60999999999999999,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.722,"5min":3.722,"15min":3.722},"last":0.80600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986220,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6204,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738948,"statfs":{"total":95995035648,"available":95988682752,"internally_reserved":0,"allocated":708608,"data_stored":296831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.502,"15min":0.496},"min":{"1min":0.35699999999999998,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.67500000000000004,"5min":0.85499999999999998,"15min":0.85499999999999998},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.60099999999999998,"15min":0.56100000000000005},"min":{"1min":0.32800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.92700000000000005,"5min":6.5570000000000004,"15min":6.5570000000000004},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.53900000000000003,"15min":0.53600000000000003},"min":{"1min":0.38700000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.968,"5min":1.968,"15min":1.968},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51400000000000001,"15min":0.501},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82299999999999995,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.60899999999999999}]},{"osd":2,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.54500000000000004,"5min":0.53800000000000003,"15min":0.53600000000000003},"min":{"1min":0.371,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.81100000000000005,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 14:56:53 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.54900000000000004,"15min":0.54300000000000004},"min":{"1min":0.41399999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.78400000000000003,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.505},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.53400000000000003,"15min":0.52600000000000002},"min":{"1min":0.35799999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.46800000000000003,"15min":0.46500000000000002},"min":{"1min":0.314,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.495,"5min":0.46700000000000003,"15min":0.46200000000000002},"min":{"1min":0.34499999999999997,"5min":0.308,"15min":0.308},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 23 14:56:35 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.58299999999999996,"15min":0.58499999999999996},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.89700000000000002,"5min":1.4610000000000001,"15min":1.4610000000000001},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55500000000000005,"15min":0.55600000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 14:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.55700000000000005,"15min":0.55600000000000005},"min":{"1min":0.39500000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.625,"15min":0.626},"min":{"1min":0.45600000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.72499999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724583,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6060,"kb_used_data":676,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739092,"statfs":{"total":95995035648,"available":95988830208,"internally_reserved":0,"allocated":692224,"data_stored":295773,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.48599999999999999,"15min":0.48699999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.64800000000000002,"5min":0.747,"15min":0.747},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.52500000000000002,"15min":0.53100000000000003},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.64100000000000001,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.503}]},{"osd":1,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.505,"15min":0.505},"min":{"1min":0.32800000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.373},{"interface":"front","average":{"1min":0.498,"5min":0.5,"15min":0.5},"min":{"1min":0.35799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.67900000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.54200000000000004}]},{"osd":2,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":0.498,"15min":0.497},"min":{"1min":0.35099999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.53900000000000003,"15min":0.54700000000000004},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.61399999999999999,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.60399999999999998}]},{"osd":3,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.39700000000000002,"5min":0.312,"15min":0.312},"max":{"1min":0.80800000000000005,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.88200000000000001,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 14:56:15 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":0.41699999999999998,"15min":0.41299999999999998},"min":{"1min":0.32600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.72299999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.45500000000000002,"15min":0.45100000000000001},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.73899999999999999,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.66500000000000004}]},{"osd":6,"last update":"Sat Sep 23 14:56:38 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.48999999999999999,"15min":0.47999999999999998},"min":{"1min":0.42199999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.755,"5min":0.755,"15min":0.755},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.56200000000000006,"15min":0.55200000000000005},"min":{"1min":0.42999999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 14:57:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59099999999999997,"15min":0.58999999999999997},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.754,"5min":0.80400000000000005,"15min":0.80400000000000005},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.56100000000000005,"15min":0.55500000000000005},"min":{"1min":0.39500000000000002,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.48599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T14:57:16.774 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T14:57:16.774 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T14:57:16.774 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T14:57:16.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T14:57:17.141 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:17.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:17 smithi017 ceph-mon[100189]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:17.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:17 smithi017 ceph-mon[100189]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:17 smithi017 ceph-mon[118128]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:17 smithi017 ceph-mon[118128]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:17 smithi027 ceph-mon[105164]: from='client.24686 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T14:57:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:17 smithi027 ceph-mon[105164]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:18.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:18 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:18] "GET /metrics HTTP/1.1" 200 238550 "" "Prometheus/2.43.0" 2023-09-23T14:57:18.861 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T14:57:18.862 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T14:57:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:18 smithi017 ceph-mon[100189]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:18 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3849734422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:57:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:18 smithi017 ceph-mon[118128]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:18 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3849734422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:57:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:18 smithi027 ceph-mon[105164]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:18 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3849734422' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T14:57:19.526 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T14:57:19.526 INFO:teuthology.run_tasks:Running task print... 2023-09-23T14:57:19.536 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T14:57:19.536 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T14:57:19.545 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T14:57:19.546 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T14:57:19.546 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T14:57:19.551 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T14:57:19.551 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T14:57:19.551 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T14:57:19.552 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T14:57:19.552 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T14:57:19.552 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi017.front.sepia.ceph.com 2023-09-23T14:57:19.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T14:57:19.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T14:57:19.573 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T14:57:19.573 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T14:57:19.630 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T14:57:19.630 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T14:57:19.686 INFO:tasks.workunit:timeout=3h 2023-09-23T14:57:19.687 INFO:tasks.workunit:cleanup=True 2023-09-23T14:57:19.687 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T14:57:19.744 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T14:57:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:21.658 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE Bus STOPPING 2023-09-23T14:57:21.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:21 smithi017 ceph-mon[100189]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:21 smithi017 ceph-mon[100189]: from='client.? ' entity='client.admin' 2023-09-23T14:57:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:21 smithi017 ceph-mon[118128]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:21 smithi017 ceph-mon[118128]: from='client.? ' entity='client.admin' 2023-09-23T14:57:21.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:57:21.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE Bus STOPPED 2023-09-23T14:57:21.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE Bus STARTING 2023-09-23T14:57:21.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE Serving on http://:::9283 2023-09-23T14:57:21.944 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:21 smithi017 conmon[100620]: [23/Sep/2023:14:57:21] ENGINE Bus STARTED 2023-09-23T14:57:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:21 smithi027 ceph-mon[105164]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:21 smithi027 ceph-mon[105164]: from='client.? ' entity='client.admin' 2023-09-23T14:57:22.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T14:57:22.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:22.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:22 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:22.852 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:22 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:22 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:22 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:22 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:23.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:23 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:23.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:23 smithi017 ceph-mon[100189]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:23 smithi017 ceph-mon[118128]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:23 smithi027 ceph-mon[105164]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:24.181 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T14:57:24.413 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T14:57:24.415 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T14:57:24.416 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T14:57:25.761 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T14:57:25.842 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T14:57:25.881 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T14:57:25.882 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T14:57:25.882 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T14:57:25.920 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T14:57:25.924 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T14:57:25.924 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T14:57:25.943 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T14:57:25.944 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T14:57:25.944 DEBUG:teuthology.orchestra.run.smithi017:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T14:57:25.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:25 smithi017 ceph-mon[100189]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:25.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:25 smithi017 ceph-mon[118128]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:26.004 INFO:tasks.workunit.client.0.smithi017.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T14:57:26.004 INFO:tasks.workunit.client.0.smithi017.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T14:57:26.004 INFO:tasks.workunit.client.0.smithi017.stderr:+ TMPDIR=/tmp/rbd_import_export_190949 2023-09-23T14:57:26.004 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_190949 2023-09-23T14:57:26.005 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir /tmp/rbd_import_export_190949 2023-09-23T14:57:26.006 INFO:tasks.workunit.client.0.smithi017.stderr:+ trap 'rm -rf /tmp/rbd_import_export_190949' INT TERM EXIT 2023-09-23T14:57:26.006 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir foo.190949 2023-09-23T14:57:26.007 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import foo.190949 foo.dir 2023-09-23T14:57:26.048 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: cannot import a directory 2023-09-23T14:57:26.048 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 0% complete...failed. 2023-09-23T14:57:26.048 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: import failed: (21) Is a directory 2023-09-23T14:57:26.050 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:26.050 INFO:tasks.workunit.client.0.smithi017.stderr:+ rmdir foo.190949 2023-09-23T14:57:26.054 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_190949/img bs=1k count=1 seek=10 2023-09-23T14:57:26.055 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T14:57:26.055 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T14:57:26.055 INFO:tasks.workunit.client.0.smithi017.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.6238e-05 s, 22.1 MB/s 2023-09-23T14:57:26.055 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_190949/img bs=1k count=10 seek=100 2023-09-23T14:57:26.055 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2023-09-23T14:57:26.056 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2023-09-23T14:57:26.056 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.8823e-05 s, 174 MB/s 2023-09-23T14:57:26.056 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_190949/img bs=1k count=100 seek=1000 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records in 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records out 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.00021581 s, 334 MB/s 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_190949/img bs=1k seek=10000 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:139+1 records in 2023-09-23T14:57:26.057 INFO:tasks.workunit.client.0.smithi017.stderr:139+1 records out 2023-09-23T14:57:26.058 INFO:tasks.workunit.client.0.smithi017.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000381342 s, 376 MB/s 2023-09-23T14:57:26.058 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_190949/img bs=1k seek=100000 2023-09-23T14:57:26.063 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records in 2023-09-23T14:57:26.063 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records out 2023-09-23T14:57:26.063 INFO:tasks.workunit.client.0.smithi017.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.00436446 s, 16.5 MB/s 2023-09-23T14:57:26.063 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_190949/img bs=1k seek=1000000 2023-09-23T14:57:26.064 INFO:tasks.workunit.client.0.smithi017.stderr:189+1 records in 2023-09-23T14:57:26.064 INFO:tasks.workunit.client.0.smithi017.stderr:189+1 records out 2023-09-23T14:57:26.064 INFO:tasks.workunit.client.0.smithi017.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000494846 s, 392 MB/s 2023-09-23T14:57:26.064 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T14:57:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:25 smithi027 ceph-mon[105164]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:26.102 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T14:57:26.102 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T14:57:26.105 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:26.105 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:27.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T14:57:27.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:27 smithi017 ceph-mon[100189]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:27.957 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:27 smithi017 ceph-mon[100189]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T14:57:27.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:27 smithi017 ceph-mon[118128]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:27.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:27 smithi017 ceph-mon[118128]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T14:57:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:27 smithi027 ceph-mon[105164]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:27 smithi027 ceph-mon[105164]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T14:57:28.243 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T14:57:28.243 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:28.246 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_190949/img2 2023-09-23T14:57:28.347 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:28.547 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:28.550 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2023-09-23T14:57:28.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:28 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:28] "GET /metrics HTTP/1.1" 200 238550 "" "Prometheus/2.43.0" 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.058 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]: dispatch 2023-09-23T14:57:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:29.278 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:29.282 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T14:57:29.371 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T14:57:29.374 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img2 2023-09-23T14:57:30.056 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img3 2023-09-23T14:57:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:29 smithi027 ceph-mon[105164]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:29 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T14:57:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:29 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T14:57:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:29 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2023-09-23T14:57:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:29 smithi027 ceph-mon[105164]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T14:57:30.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[100189]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[100189]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[118128]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 2]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 5]}]': finished 2023-09-23T14:57:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:29 smithi017 ceph-mon[118128]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T14:57:30.536 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img2 /tmp/rbd_import_export_190949/img3 2023-09-23T14:57:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:30 smithi027 ceph-mon[105164]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T14:57:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:30 smithi027 ceph-mon[105164]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 128 KiB/s wr, 76 op/s 2023-09-23T14:57:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:30 smithi017 ceph-mon[100189]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T14:57:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:30 smithi017 ceph-mon[100189]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 128 KiB/s wr, 76 op/s 2023-09-23T14:57:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:30 smithi017 ceph-mon[118128]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T14:57:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:30 smithi017 ceph-mon[118128]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 297 KiB/s rd, 128 KiB/s wr, 76 op/s 2023-09-23T14:57:33.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:33 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:33.920 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - testimg 2023-09-23T14:57:33.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:33 smithi017 ceph-mon[100189]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 86 KiB/s wr, 51 op/s 2023-09-23T14:57:33.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:33 smithi017 ceph-mon[100189]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T14:57:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:33 smithi017 ceph-mon[118128]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 86 KiB/s wr, 51 op/s 2023-09-23T14:57:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:33 smithi017 ceph-mon[118128]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T14:57:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:33 smithi027 ceph-mon[105164]: pgmap v77: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 86 KiB/s wr, 51 op/s 2023-09-23T14:57:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:33 smithi027 ceph-mon[105164]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T14:57:34.303 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:34.304 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:34.306 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_190949/img2 2023-09-23T14:57:34.559 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:34.563 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2023-09-23T14:57:35.279 INFO:teuthology.orchestra.run.smithi017.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:35.302 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:35.306 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T14:57:35.408 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T14:57:35.411 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img2 2023-09-23T14:57:35.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[100189]: pgmap v79: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 359 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T14:57:35.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:35.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:35.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[118128]: pgmap v79: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 359 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T14:57:35.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:35.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:35 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:36.074 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img3 2023-09-23T14:57:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:35 smithi027 ceph-mon[105164]: pgmap v79: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 359 KiB/s rd, 172 KiB/s wr, 108 op/s 2023-09-23T14:57:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:35 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:35 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:57:36.552 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img2 /tmp/rbd_import_export_190949/img3 2023-09-23T14:57:36.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:36 smithi017 ceph-mon[100189]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:36 smithi017 ceph-mon[100189]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:36 smithi017 ceph-mon[118128]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:36 smithi017 ceph-mon[118128]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:36 smithi027 ceph-mon[105164]: from='client.14874 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:36 smithi027 ceph-mon[105164]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:37 smithi017 ceph-mon[100189]: pgmap v80: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 KiB/s rd, 134 KiB/s wr, 84 op/s 2023-09-23T14:57:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:37 smithi017 ceph-mon[118128]: pgmap v80: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 KiB/s rd, 134 KiB/s wr, 84 op/s 2023-09-23T14:57:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:37 smithi027 ceph-mon[105164]: pgmap v80: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 281 KiB/s rd, 134 KiB/s wr, 84 op/s 2023-09-23T14:57:38.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:38 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:38] "GET /metrics HTTP/1.1" 200 238551 "" "Prometheus/2.43.0" 2023-09-23T14:57:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:38 smithi027 ceph-mon[105164]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 60 KiB/s wr, 39 op/s 2023-09-23T14:57:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:38 smithi017 ceph-mon[100189]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 60 KiB/s wr, 39 op/s 2023-09-23T14:57:39.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:38 smithi017 ceph-mon[118128]: pgmap v81: 73 pgs: 73 active+clean; 6.0 MiB data, 53 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 60 KiB/s wr, 39 op/s 2023-09-23T14:57:41.276 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd help export 2023-09-23T14:57:41.277 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -q export-format 2023-09-23T14:57:41.298 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_190949/img bs=1k count=10 seek=100 2023-09-23T14:57:41.298 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2023-09-23T14:57:41.299 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2023-09-23T14:57:41.299 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.4291e-05 s, 138 MB/s 2023-09-23T14:57:41.299 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:41.355 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:41.355 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:41.358 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap 2023-09-23T14:57:41.691 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T14:57:41.697 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T14:57:41.750 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg 2023-09-23T14:57:41.787 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T14:57:41.804 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T14:57:41.804 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T14:57:41.804 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T14:57:41.804 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2023-09-23T14:57:41.804 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:41.847 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:41.850 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190949/img_v2 testimg_import 2023-09-23T14:57:41.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:41 smithi017 ceph-mon[100189]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T14:57:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:41 smithi017 ceph-mon[118128]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T14:57:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:41 smithi027 ceph-mon[105164]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T14:57:42.191 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:42.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[100189]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T14:57:42.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:42.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:42.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:42.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[118128]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T14:57:42.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:42.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:42.686 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:42 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:42.777 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T14:57:42.780 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T14:57:42.817 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2023-09-23T14:57:42.817 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: id: 60e6bf31a9f2 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.60e6bf31a9f2 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.818 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.820 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: id: 60e6bf31a9f2 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.60e6bf31a9f2 2023-09-23T14:57:42.856 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Sat Sep 23 14:57:41 2023 2023-09-23T14:57:42.857 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2023-09-23T14:57:42.859 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg_import 2023-09-23T14:57:42.901 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T14:57:42.901 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T14:57:42.901 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T14:57:42.901 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' ']' 2023-09-23T14:57:42.902 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import /tmp/rbd_import_export_190949/img_import 2023-09-23T14:57:42.937 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T14:57:42.940 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img_import 2023-09-23T14:57:42.941 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/img /tmp/rbd_import_export_190949/img_import 2023-09-23T14:57:42.942 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/img --format %b 2023-09-23T14:57:42.943 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=24 2023-09-23T14:57:42.943 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/img_import --format %b 2023-09-23T14:57:42.944 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2023-09-23T14:57:42.944 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2023-09-23T14:57:42.945 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2023-09-23T14:57:42.945 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T14:57:42.945 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_190949/img_snap 2023-09-23T14:57:42.981 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T14:57:42.984 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_190949/img_snap_import 2023-09-23T14:57:43.022 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T14:57:43.025 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/img_snap /tmp/rbd_import_export_190949/img_snap_import 2023-09-23T14:57:43.025 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/img_snap /tmp/rbd_import_export_190949/img_snap_import 2023-09-23T14:57:43.026 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/img_snap --format %b 2023-09-23T14:57:43.027 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=224 2023-09-23T14:57:43.027 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/img_snap_import --format %b 2023-09-23T14:57:43.028 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2023-09-23T14:57:43.028 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:43.029 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:43.029 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T14:57:43.029 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:43.029 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_import 2023-09-23T14:57:43.029 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_snap 2023-09-23T14:57:43.030 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_snap_import 2023-09-23T14:57:43.031 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T14:57:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:42 smithi027 ceph-mon[105164]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T14:57:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:42 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:57:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:42 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:57:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:42 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:57:43.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:43 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[100189]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[100189]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 54 KiB/s wr, 51 op/s 2023-09-23T14:57:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[100189]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T14:57:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[118128]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[118128]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 54 KiB/s wr, 51 op/s 2023-09-23T14:57:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[118128]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T14:57:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:43 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:43.949 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2023-09-23T14:57:43.959 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T14:57:44.066 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:44.070 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg@snap 2023-09-23T14:57:44.098 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T14:57:44.098 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (79s) 47s ago 2m 17.3M - 0.25.0 c8568f914cd2 ca695ff81f5c 2023-09-23T14:57:44.098 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (93s) 56s ago 93s 84.6M - 9.4.7 2c41d148cca3 dec828213828 2023-09-23T14:57:44.098 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (66s) 47s ago 65s 14.9M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T14:57:44.098 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (63s) 56s ago 62s 15.4M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443 running (5m) 56s ago 5m 439M - 17.2.6-1123-g61431f6b 0db14aa01094 02c286027557 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:9283 running (7m) 47s ago 7m 494M - 17.2.6-1123-g61431f6b 0db14aa01094 2de58bf12b45 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (7m) 47s ago 7m 77.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 54ba4e8b4801 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 running (6m) 56s ago 6m 57.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 38ba724092d3 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (6m) 47s ago 6m 59.0M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 4d08e21f44ed 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (117s) 47s ago 117s 17.1M - 1.5.0 0da6a335fe13 af3d2ae00db6 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (113s) 56s ago 113s 13.7M - 1.5.0 0da6a335fe13 67db5f1f10b6 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (5m) 47s ago 5m 55.9M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 bb8e8a0efb36 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (5m) 47s ago 5m 56.8M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 1542a5a16945 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (4m) 47s ago 4m 49.5M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 d6cc1ec2d677 2023-09-23T14:57:44.099 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (4m) 47s ago 4m 49.7M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 1e310e53ac98 2023-09-23T14:57:44.100 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (4m) 56s ago 3m 49.3M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T14:57:44.100 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (3m) 56s ago 3m 50.8M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T14:57:44.100 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (3m) 56s ago 3m 49.1M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T14:57:44.100 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (2m) 56s ago 2m 48.9M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T14:57:44.100 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (73s) 56s ago 2m 34.1M - 2.43.0 a07b618ecd1d 5252779692de 2023-09-23T14:57:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:43 smithi027 ceph-mon[105164]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T14:57:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:43 smithi027 ceph-mon[105164]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 53 MiB used, 715 GiB / 715 GiB avail; 138 KiB/s rd, 54 KiB/s wr, 51 op/s 2023-09-23T14:57:44.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:43 smithi027 ceph-mon[105164]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T14:57:44.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:43 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:44.954 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2023-09-23T14:57:44.965 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T14:57:45.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T14:57:45.075 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:45.086 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --order 20 /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:45.143 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:45.144 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:45.146 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:45.187 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:45.190 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190949/img_v2 testimg_import 2023-09-23T14:57:45.261 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T14:57:45.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T14:57:45.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep order 2023-09-23T14:57:45.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $2}' 2023-09-23T14:57:45.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 20 2023-09-23T14:57:45.305 INFO:tasks.workunit.client.0.smithi017.stdout:20 2023-09-23T14:57:45.305 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:45.306 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T14:57:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:44 smithi027 ceph-mon[105164]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T14:57:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:44 smithi027 ceph-mon[105164]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:44 smithi027 ceph-mon[105164]: pgmap v87: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 11 KiB/s wr, 171 op/s 2023-09-23T14:57:45.408 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:45.412 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T14:57:45.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[100189]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T14:57:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[100189]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[100189]: pgmap v87: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 11 KiB/s wr, 171 op/s 2023-09-23T14:57:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[118128]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T14:57:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[118128]: from='client.24854 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T14:57:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:44 smithi017 ceph-mon[118128]: pgmap v87: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 177 KiB/s rd, 11 KiB/s wr, 171 op/s 2023-09-23T14:57:45.494 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:45.515 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:45.522 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:45.579 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:45.582 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg 2023-09-23T14:57:45.583 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2023-09-23T14:57:45.624 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T14:57:45.628 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2023-09-23T14:57:45.629 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:45.661 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:45.664 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190949/img_v2 testimg_import 2023-09-23T14:57:45.725 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T14:57:45.729 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg_import 2023-09-23T14:57:45.729 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2023-09-23T14:57:45.767 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T14:57:45.767 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2023-09-23T14:57:45.767 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' ' features: layering 2023-09-23T14:57:45.767 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' '!=' ' features: layering 2023-09-23T14:57:45.767 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' ']' 2023-09-23T14:57:45.768 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:45.768 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T14:57:45.842 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:45.845 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T14:57:45.920 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:45.923 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:45.990 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:45.994 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:46.036 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:46.039 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190949/img_v2 testimg_import 2023-09-23T14:57:46.101 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T14:57:46.104 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T14:57:46.105 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe unit' 2023-09-23T14:57:46.105 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T14:57:46.143 INFO:tasks.workunit.client.0.smithi017.stdout: stripe unit: 4 KiB 2023-09-23T14:57:46.144 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T14:57:46.144 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe count' 2023-09-23T14:57:46.144 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $3}' 2023-09-23T14:57:46.144 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 1000 2023-09-23T14:57:46.192 INFO:tasks.workunit.client.0.smithi017.stdout:1000 2023-09-23T14:57:46.192 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/img_v2 2023-09-23T14:57:46.192 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T14:57:46.279 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T14:57:46.282 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T14:57:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:45 smithi027 ceph-mon[105164]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T14:57:46.387 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T14:57:46.390 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_190949/img testimg 2023-09-23T14:57:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:45 smithi017 ceph-mon[100189]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T14:57:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:45 smithi017 ceph-mon[118128]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T14:57:46.448 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:46.451 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap1 2023-09-23T14:57:46.969 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T14:57:46.976 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap2 2023-09-23T14:57:47.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:46 smithi017 ceph-mon[100189]: pgmap v89: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-23T14:57:47.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:46 smithi017 ceph-mon[118128]: pgmap v89: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-23T14:57:47.136 INFO:teuthology.orchestra.run.smithi017.stdout:wait for minority of mons to upgrade 2023-09-23T14:57:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:46 smithi027 ceph-mon[105164]: pgmap v89: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 12 KiB/s wr, 176 op/s 2023-09-23T14:57:47.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T14:57:47.964 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T14:57:47.971 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap protect testimg@snap2 2023-09-23T14:57:48.022 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_190949/snap_protect 2023-09-23T14:57:48.065 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:48.068 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_190949/snap_protect testimg_import 2023-09-23T14:57:48.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:47 smithi017 ceph-mon[100189]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T14:57:48.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:47 smithi017 ceph-mon[100189]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T14:57:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:47 smithi017 ceph-mon[118128]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T14:57:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:47 smithi017 ceph-mon[118128]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T14:57:48.346 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T14:57:48.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:47 smithi027 ceph-mon[105164]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T14:57:48.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:47 smithi027 ceph-mon[105164]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T14:57:48.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:48 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:48] "GET /metrics HTTP/1.1" 200 239108 "" "Prometheus/2.43.0" 2023-09-23T14:57:49.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:48 smithi017 ceph-mon[100189]: pgmap v92: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 147 KiB/s rd, 13 KiB/s wr, 186 op/s 2023-09-23T14:57:49.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:48 smithi017 ceph-mon[118128]: pgmap v92: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 147 KiB/s rd, 13 KiB/s wr, 186 op/s 2023-09-23T14:57:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:48 smithi027 ceph-mon[105164]: pgmap v92: 73 pgs: 73 active+clean; 689 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 147 KiB/s rd, 13 KiB/s wr, 186 op/s 2023-09-23T14:57:49.988 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T14:57:49.993 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap1 2023-09-23T14:57:49.993 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: False' 2023-09-23T14:57:50.038 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2023-09-23T14:57:50.038 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap2 2023-09-23T14:57:50.039 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: True' 2023-09-23T14:57:50.083 INFO:tasks.workunit.client.0.smithi017.stdout: protected: True 2023-09-23T14:57:50.088 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/snap_protect 2023-09-23T14:57:50.089 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T14:57:50.137 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T14:57:50.185 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg 2023-09-23T14:57:50.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:49 smithi017 ceph-mon[118128]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T14:57:50.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:49 smithi017 ceph-mon[100189]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T14:57:50.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:49 smithi027 ceph-mon[105164]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T14:57:51.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:50 smithi027 ceph-mon[105164]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T14:57:51.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:50 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1234002520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:51.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:50 smithi027 ceph-mon[105164]: pgmap v95: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:51.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[100189]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T14:57:51.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1234002520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:51.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[100189]: pgmap v95: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:51.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[118128]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T14:57:51.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1234002520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:51.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:50 smithi017 ceph-mon[118128]: pgmap v95: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:51.984 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T14:57:51.992 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg_import 2023-09-23T14:57:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:51 smithi027 ceph-mon[105164]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T14:57:52.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:51 smithi017 ceph-mon[100189]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T14:57:52.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:51 smithi017 ceph-mon[118128]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T14:57:53.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:57:53 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:57:53.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:52 smithi027 ceph-mon[105164]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T14:57:53.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:52 smithi027 ceph-mon[105164]: pgmap v98: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:53.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:52 smithi027 ceph-mon[105164]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T14:57:53.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[100189]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T14:57:53.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[100189]: pgmap v98: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:53.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[100189]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T14:57:53.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[118128]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T14:57:53.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[118128]: pgmap v98: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 386 KiB/s rd, 49 KiB/s wr, 517 op/s 2023-09-23T14:57:53.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:52 smithi017 ceph-mon[118128]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T14:57:53.957 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T14:57:53.967 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T14:57:54.063 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:54.067 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T14:57:54.158 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:54.161 INFO:tasks.workunit.client.0.smithi017.stderr:+ tiered=0 2023-09-23T14:57:54.161 INFO:tasks.workunit.client.0.smithi017.stderr:+ ceph osd dump 2023-09-23T14:57:54.162 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep '^pool' 2023-09-23T14:57:54.162 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep ''\''rbd'\''' 2023-09-23T14:57:54.162 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep tier 2023-09-23T14:57:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:53 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2807420980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:53 smithi027 ceph-mon[105164]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T14:57:54.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:53 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2807420980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:54.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:53 smithi017 ceph-mon[100189]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T14:57:54.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:53 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2807420980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:54.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:53 smithi017 ceph-mon[118128]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T14:57:54.712 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_190949/sparse1 2023-09-23T14:57:54.718 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T14:57:54.718 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T14:57:54.718 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00502852 s, 209 MB/s 2023-09-23T14:57:54.718 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_190949/sparse2 2023-09-23T14:57:54.724 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T14:57:54.724 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T14:57:54.724 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00501622 s, 209 MB/s 2023-09-23T14:57:54.724 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_190949/sparse2 -s 2M 2023-09-23T14:57:54.739 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T14:57:54.781 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T14:57:54.781 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T14:57:54.783 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:54.784 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190949/sparse1 2023-09-23T14:57:54.850 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T14:57:54.851 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:54.851 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:54.853 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T14:57:54.854 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2023-09-23T14:57:54.854 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T14:57:54.901 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 2 MiB 2 2023-09-23T14:57:54.902 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2023-09-23T14:57:54.902 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2023-09-23T14:57:54.902 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2023-09-23T14:57:54.902 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:54.902 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:54.946 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.620c380c15f3 2023-09-23T14:57:54.946 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.620c380c15f3 2023-09-23T14:57:54.946 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2023-09-23T14:57:54.946 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2023-09-23T14:57:54.947 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:54.947 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.620c380c15f3\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:54.947 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2023-09-23T14:57:54.947 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:54.947 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:54.982 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:54.982 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:54.982 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:54.983 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:54.983 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:55.018 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T14:57:55.018 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T14:57:55.019 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T14:57:55.019 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.064 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:55.067 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.067 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.069 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse1 --format %b 2023-09-23T14:57:55.071 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T14:57:55.071 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse1.out --format %b 2023-09-23T14:57:55.072 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T14:57:55.072 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.072 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.072 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T14:57:55.072 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.073 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T14:57:55.165 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T14:57:55.169 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T14:57:55.209 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T14:57:55.209 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T14:57:55.212 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:55.212 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190949/sparse2 2023-09-23T14:57:55.275 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T14:57:55.276 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:55.276 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:55.279 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T14:57:55.279 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2023-09-23T14:57:55.279 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T14:57:55.317 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 2 MiB 2 2023-09-23T14:57:55.318 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2023-09-23T14:57:55.318 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2023-09-23T14:57:55.318 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2023-09-23T14:57:55.318 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:55.318 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:54 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:54 smithi027 ceph-mon[105164]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-09-23T14:57:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:54 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.356 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3b9d7a524a9e 2023-09-23T14:57:55.356 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3b9d7a524a9e 2023-09-23T14:57:55.357 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3b9d7a524a9e\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:55.357 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2023-09-23T14:57:55.357 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2023-09-23T14:57:55.358 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:55.358 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2023-09-23T14:57:55.358 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:55.359 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:55.396 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:55.397 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:55.397 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:55.397 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:55.397 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:55.431 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2023-09-23T14:57:55.431 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2023-09-23T14:57:55.432 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T14:57:55.432 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:55.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[100189]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-09-23T14:57:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[118128]: pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 10 KiB/s wr, 203 op/s 2023-09-23T14:57:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:54 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/327438896' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T14:57:55.469 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:55.472 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:55.472 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:55.475 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse2 --format %b 2023-09-23T14:57:55.475 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T14:57:55.476 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse2.out --format %b 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:55.477 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T14:57:55.572 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T14:57:55.576 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_190949/sparse1 -s 10M 2023-09-23T14:57:55.577 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T14:57:55.674 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:55.674 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:55.674 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:55.677 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T14:57:55.677 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2023-09-23T14:57:55.677 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T14:57:55.723 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 10 MiB 2 2023-09-23T14:57:55.724 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2023-09-23T14:57:55.724 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2023-09-23T14:57:55.724 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2023-09-23T14:57:55.724 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:55.724 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:55.771 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.622fb8c8855b 2023-09-23T14:57:55.772 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.622fb8c8855b 2023-09-23T14:57:55.772 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2023-09-23T14:57:55.772 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2023-09-23T14:57:55.773 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.622fb8c8855b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:55.773 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:55.779 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2023-09-23T14:57:55.779 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:55.779 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:55.821 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:55.821 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:55.822 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:55.822 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:55.822 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:55.856 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T14:57:55.856 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T14:57:55.856 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T14:57:55.856 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.895 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:55.898 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.899 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/sparse1 /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.906 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse1 --format %b 2023-09-23T14:57:55.907 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T14:57:55.908 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse1.out --format %b 2023-09-23T14:57:55.909 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T14:57:55.909 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.909 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:55.909 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T14:57:55.909 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse1.out 2023-09-23T14:57:55.910 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T14:57:55.995 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T14:57:55.998 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_190949/sparse2 oflag=append conv=notrunc 2023-09-23T14:57:56.009 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T14:57:56.009 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T14:57:56.009 INFO:tasks.workunit.client.0.smithi017.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0100255 s, 209 MB/s 2023-09-23T14:57:56.009 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T14:57:56.095 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:56.096 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:56.096 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:56.099 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T14:57:56.099 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2023-09-23T14:57:56.099 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T14:57:56.138 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 4 MiB 2 2023-09-23T14:57:56.138 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2023-09-23T14:57:56.139 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2023-09-23T14:57:56.139 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2023-09-23T14:57:56.139 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:56.139 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:56.176 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.629f623c5b70 2023-09-23T14:57:56.177 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.629f623c5b70 2023-09-23T14:57:56.177 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2023-09-23T14:57:56.177 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2023-09-23T14:57:56.177 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:56.178 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.629f623c5b70\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:56.178 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2023-09-23T14:57:56.178 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:56.178 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:56.215 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:56.215 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:56.216 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:56.216 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:56.216 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:56.249 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects='0 2023-09-23T14:57:56.249 INFO:tasks.workunit.client.0.smithi017.stderr:2 2023-09-23T14:57:56.249 INFO:tasks.workunit.client.0.smithi017.stderr:3' 2023-09-23T14:57:56.249 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2 3 2023-09-23T14:57:56.250 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T14:57:56.250 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:56.290 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T14:57:56.294 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_190949/sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:56.294 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_190949/sparse2 /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:56.297 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse2 --format %b 2023-09-23T14:57:56.298 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=6144 2023-09-23T14:57:56.298 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse2.out --format %b 2023-09-23T14:57:56.299 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=6144 2023-09-23T14:57:56.299 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:56.299 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T14:57:56.300 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T14:57:56.300 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse2.out 2023-09-23T14:57:56.300 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T14:57:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:55 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2835419188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:56.389 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T14:57:56.392 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T14:57:56.392 INFO:tasks.workunit.client.0.smithi017.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T14:57:56.393 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_190949/sparse1 sparse 2023-09-23T14:57:56.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:55 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2835419188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:56.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:55 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2835419188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:56.459 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T14:57:56.460 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:56.461 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T14:57:56.463 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T14:57:56.463 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T14:57:56.463 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T14:57:56.463 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:56.463 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:56.500 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3c24ffdde87d 2023-09-23T14:57:56.501 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3c24ffdde87d 2023-09-23T14:57:56.501 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3c24ffdde87d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:56.501 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T14:57:56.501 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T14:57:56.501 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:56.504 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T14:57:56.504 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:56.505 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:56.572 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T14:57:56.572 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T14:57:56.572 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T14:57:56.572 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T14:57:56.711 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T14:57:56.714 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T14:57:56.714 INFO:tasks.workunit.client.0.smithi017.stdout:zeros import through stdin to sparse image 2023-09-23T14:57:56.715 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T14:57:56.715 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - sparse 2023-09-23T14:57:56.762 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records in 2023-09-23T14:57:56.762 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records out 2023-09-23T14:57:56.762 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0474247 s, 88.4 MB/s 2023-09-23T14:57:56.772 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T14:57:56.773 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:56.775 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T14:57:56.776 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T14:57:56.776 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T14:57:56.776 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:56.776 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:56.813 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.62eac5a15c2c 2023-09-23T14:57:56.814 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.62eac5a15c2c 2023-09-23T14:57:56.814 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T14:57:56.814 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.62eac5a15c2c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:56.814 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T14:57:56.815 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:56.815 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T14:57:56.815 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:56.815 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:56.853 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:56.853 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:56.853 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:56.853 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:56.854 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:56.886 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects= 2023-09-23T14:57:56.886 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 2023-09-23T14:57:56.886 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T14:57:56.886 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T14:57:56.970 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:56.972 INFO:tasks.workunit.client.0.smithi017.stdout:zeros export to sparse file 2023-09-23T14:57:56.973 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros export to sparse file' 2023-09-23T14:57:56.973 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T14:57:57.017 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T14:57:57.020 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info sparse 2023-09-23T14:57:57.020 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep block_name_prefix 2023-09-23T14:57:57.020 INFO:tasks.workunit.client.0.smithi017.stderr:++ awk '{print $NF;}' 2023-09-23T14:57:57.069 INFO:tasks.workunit.client.0.smithi017.stderr:+ prefix=rbd_data.3c661ff41c77 2023-09-23T14:57:57.070 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T14:57:57.070 INFO:tasks.workunit.client.0.smithi017.stderr:++ get_image_data_pool sparse 2023-09-23T14:57:57.071 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T14:57:57.071 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T14:57:57.071 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep 'data_pool: ' 2023-09-23T14:57:57.071 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T14:57:57.109 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool= 2023-09-23T14:57:57.110 INFO:tasks.workunit.client.0.smithi017.stderr:++ '[' -z ']' 2023-09-23T14:57:57.110 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool=rbd 2023-09-23T14:57:57.110 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo rbd 2023-09-23T14:57:57.110 INFO:tasks.workunit.client.0.smithi017.stderr:+ rados -p rbd put rbd_data.3c661ff41c77.000000000000 - 2023-09-23T14:57:57.146 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T14:57:57.146 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T14:57:57.147 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0760197 s, 55.2 MB/s 2023-09-23T14:57:57.177 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T14:57:57.177 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T14:57:57.177 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T14:57:57.177 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T14:57:57.177 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T14:57:57.225 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.3c661ff41c77 2023-09-23T14:57:57.226 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.3c661ff41c77 2023-09-23T14:57:57.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T14:57:57.226 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T14:57:57.226 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.3c661ff41c77\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T14:57:57.227 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T14:57:57.230 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T14:57:57.230 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T14:57:57.231 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T14:57:57.269 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T14:57:57.269 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T14:57:57.269 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T14:57:57.270 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T14:57:57.270 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T14:57:57.306 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2023-09-23T14:57:57.306 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2023-09-23T14:57:57.306 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T14:57:57.306 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse 2023-09-23T14:57:57.307 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_190949/sparse': No such file or directory 2023-09-23T14:57:57.307 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:57.307 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse /tmp/rbd_import_export_190949/sparse 2023-09-23T14:57:57.343 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T14:57:57.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:56 smithi027 ceph-mon[105164]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-09-23T14:57:57.349 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_190949/sparse --format=%b 2023-09-23T14:57:57.350 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 = 0 ']' 2023-09-23T14:57:57.350 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T14:57:57.440 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T14:57:57.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:56 smithi017 ceph-mon[100189]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-09-23T14:57:57.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:56 smithi017 ceph-mon[118128]: pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.4 KiB/s wr, 143 op/s 2023-09-23T14:57:57.443 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_190949/sparse /tmp/rbd_import_export_190949/sparse1 /tmp/rbd_import_export_190949/sparse2 /tmp/rbd_import_export_190949/sparse3 2023-09-23T14:57:57.446 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_190949/sparse3': No such file or directory 2023-09-23T14:57:57.446 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T14:57:57.446 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo OK 2023-09-23T14:57:57.446 INFO:tasks.workunit.client.0.smithi017.stdout:OK 2023-09-23T14:57:57.447 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_190949 2023-09-23T14:57:57.447 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T14:57:57.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T14:57:57.481 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T14:57:57.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T14:57:57.771 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:57.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T14:57:57.799 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T14:57:57.799 DEBUG:teuthology.orchestra.run.smithi017:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T14:57:57.857 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T14:57:57.857 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T14:57:57.858 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T14:57:57.858 DEBUG:teuthology.parallel:result is None 2023-09-23T14:57:58.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:57:58 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:57:58] "GET /metrics HTTP/1.1" 200 239108 "" "Prometheus/2.43.0" 2023-09-23T14:57:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:58 smithi027 ceph-mon[105164]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T14:57:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:58 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/914804192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:58 smithi027 ceph-mon[105164]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T14:57:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:57:58 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[100189]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T14:57:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/914804192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[100189]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T14:57:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:57:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[118128]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T14:57:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/914804192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:57:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[118128]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 57 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 7.0 KiB/s wr, 135 op/s 2023-09-23T14:57:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:57:58 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:00 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2017933772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:00.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:00 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2017933772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:01.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:00 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2017933772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:01.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:01 smithi017 ceph-mon[100189]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 402 op/s 2023-09-23T14:58:01.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:01 smithi017 ceph-mon[118128]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 402 op/s 2023-09-23T14:58:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:01 smithi027 ceph-mon[105164]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.4 MiB/s wr, 402 op/s 2023-09-23T14:58:03.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:03 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:03] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:03.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:03 smithi017 ceph-mon[100189]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 975 KiB/s rd, 1.3 MiB/s wr, 356 op/s 2023-09-23T14:58:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:03 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2649395788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:03 smithi017 ceph-mon[118128]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 975 KiB/s rd, 1.3 MiB/s wr, 356 op/s 2023-09-23T14:58:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:03 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2649395788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:03 smithi027 ceph-mon[105164]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 975 KiB/s rd, 1.3 MiB/s wr, 356 op/s 2023-09-23T14:58:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:03 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2649395788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:05.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:05 smithi017 ceph-mon[100189]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:05.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:05 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1426202954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:05 smithi017 ceph-mon[118128]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:05 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1426202954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:05 smithi027 ceph-mon[105164]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:06.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:05 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1426202954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:07.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:07 smithi017 ceph-mon[100189]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:07 smithi017 ceph-mon[118128]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:07 smithi027 ceph-mon[105164]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 785 KiB/s rd, 1.1 MiB/s wr, 228 op/s 2023-09-23T14:58:08.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:08 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:08] "GET /metrics HTTP/1.1" 200 239365 "" "Prometheus/2.43.0" 2023-09-23T14:58:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:08 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2983907917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:08 smithi017 ceph-mon[100189]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 735 KiB/s rd, 1.0 MiB/s wr, 213 op/s 2023-09-23T14:58:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:08 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2983907917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:08 smithi017 ceph-mon[118128]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 735 KiB/s rd, 1.0 MiB/s wr, 213 op/s 2023-09-23T14:58:09.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:08 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2983907917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:09.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:08 smithi027 ceph-mon[105164]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 735 KiB/s rd, 1.0 MiB/s wr, 213 op/s 2023-09-23T14:58:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:10 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2990966977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:10 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2990966977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:10.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:10 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2990966977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:11 smithi027 ceph-mon[105164]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 947 KiB/s wr, 190 op/s 2023-09-23T14:58:11.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:11 smithi017 ceph-mon[100189]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 947 KiB/s wr, 190 op/s 2023-09-23T14:58:11.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:11 smithi017 ceph-mon[118128]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail; 655 KiB/s rd, 947 KiB/s wr, 190 op/s 2023-09-23T14:58:13.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:13 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:13] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[100189]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1559857334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[118128]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1559857334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:13.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:13 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:13 smithi027 ceph-mon[105164]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1559857334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:13 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:15 smithi017 ceph-mon[100189]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:15 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3662182554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:15 smithi017 ceph-mon[118128]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:15 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3662182554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:15 smithi027 ceph-mon[105164]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:15 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3662182554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:17 smithi017 ceph-mon[100189]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:17 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1558695920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:17 smithi017 ceph-mon[118128]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:17 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1558695920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:17 smithi027 ceph-mon[105164]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:17 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1558695920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:18.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:18 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:18] "GET /metrics HTTP/1.1" 200 239393 "" "Prometheus/2.43.0" 2023-09-23T14:58:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi027 2023-09-23T14:58:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:18 smithi027 ceph-mon[105164]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi027 2023-09-23T14:58:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[100189]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi027 2023-09-23T14:58:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:18 smithi017 ceph-mon[118128]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:20 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4047431451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:20 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4047431451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:20 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4047431451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:21 smithi017 ceph-mon[100189]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:21 smithi017 ceph-mon[118128]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:21.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:21 smithi027 ceph-mon[105164]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:22.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:22 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1463654868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:22.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:22 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1463654868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:22.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:22 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1463654868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:23.506 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:23 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:23] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:23.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:23 smithi027 ceph-mon[105164]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:23.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:23 smithi017 ceph-mon[100189]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:23.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:23 smithi017 ceph-mon[118128]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:25.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:25 smithi017 ceph-mon[100189]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:25.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:25 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1843374762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:25 smithi017 ceph-mon[118128]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:25 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1843374762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:25 smithi027 ceph-mon[105164]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:25 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1843374762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:27 smithi017 ceph-mon[100189]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:27 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/743689030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:27 smithi017 ceph-mon[118128]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:27 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/743689030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:27 smithi027 ceph-mon[105164]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:27 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/743689030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:28.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:28 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:28] "GET /metrics HTTP/1.1" 200 239393 "" "Prometheus/2.43.0" 2023-09-23T14:58:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:28 smithi027 ceph-mon[105164]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:28 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:28 smithi017 ceph-mon[100189]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:28 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:28 smithi017 ceph-mon[118128]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:28 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:29 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1420936152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:29 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1420936152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:29 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1420936152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:30 smithi027 ceph-mon[105164]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:30 smithi017 ceph-mon[100189]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:30 smithi017 ceph-mon[118128]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:32 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/5216281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:32 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/5216281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:32.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:32 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/5216281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:33 smithi017 ceph-mon[100189]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:33.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:33 smithi017 ceph-mon[118128]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:33 smithi027 ceph-mon[105164]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:33 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:33] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:34 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1363996306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:34 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1363996306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:34 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1363996306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:35 smithi027 ceph-mon[105164]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:35 smithi017 ceph-mon[100189]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:35 smithi017 ceph-mon[118128]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:37.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:37 smithi017 ceph-mon[100189]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:37 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/339598874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:37 smithi017 ceph-mon[118128]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:37 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/339598874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:37 smithi027 ceph-mon[105164]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:37 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/339598874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:38.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:38 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:38] "GET /metrics HTTP/1.1" 200 239348 "" "Prometheus/2.43.0" 2023-09-23T14:58:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:38 smithi027 ceph-mon[105164]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:39.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:38 smithi017 ceph-mon[100189]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:39.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:38 smithi017 ceph-mon[118128]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:39 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3671900051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:39 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3671900051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:39 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3671900051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:41.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:40 smithi027 ceph-mon[105164]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:41.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:40 smithi017 ceph-mon[100189]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:40 smithi017 ceph-mon[118128]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:41 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/752121354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:42.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:41 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/752121354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:42.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:41 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/752121354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:42 smithi027 ceph-mon[105164]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:43.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:43 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:43] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:43.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:42 smithi017 ceph-mon[100189]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:43.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:42 smithi017 ceph-mon[118128]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:44.286 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:43 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:44.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:43 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:44.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:43 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:44 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/860905515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:44 smithi027 ceph-mon[105164]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:45.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:44 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/860905515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:44 smithi017 ceph-mon[100189]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:44 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/860905515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:44 smithi017 ceph-mon[118128]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:46.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:46 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1754081805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:46 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1754081805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:46.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:46 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1754081805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:47.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:47 smithi027 ceph-mon[105164]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:47.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:47 smithi017 ceph-mon[100189]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:47 smithi017 ceph-mon[118128]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:48.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:48 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:48] "GET /metrics HTTP/1.1" 200 239345 "" "Prometheus/2.43.0" 2023-09-23T14:58:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:48 smithi027 ceph-mon[105164]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:48 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1209878044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:48 smithi017 ceph-mon[100189]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:48 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1209878044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:48 smithi017 ceph-mon[118128]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:48 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1209878044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:51 smithi017 ceph-mon[100189]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:51 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1926132625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:51 smithi017 ceph-mon[118128]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:51 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1926132625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:51 smithi027 ceph-mon[105164]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:51 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1926132625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:53.379 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:53 smithi027 conmon[108383]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:58:53.958 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:53.958 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:53.958 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:53.959 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:53.959 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:58:53.959 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:53 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:58:54.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:58:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:58:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:58:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:53 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:58:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:54 smithi027 ceph-mon[105164]: Upgrade: Updating mgr.x 2023-09-23T14:58:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:54 smithi027 ceph-mon[105164]: Deploying daemon mgr.x on smithi027 2023-09-23T14:58:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:54 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3053916125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:55.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:54 smithi027 systemd[1]: Stopping Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[100189]: Upgrade: Updating mgr.x 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[100189]: Deploying daemon mgr.x on smithi027 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3053916125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[118128]: Upgrade: Updating mgr.x 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[118128]: Deploying daemon mgr.x on smithi027 2023-09-23T14:58:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:54 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3053916125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:55.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:55 smithi027 ceph-mon[105164]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:55.904 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:55 smithi027 bash[159277]: 02c28602755772df4af84b754166f3d6a03a32882a5c09b0081e4a62939f8df3 2023-09-23T14:58:55.904 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:55 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T14:58:55.904 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:55 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T14:58:55.904 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:55 smithi027 systemd[1]: Stopped Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:58:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:55 smithi017 ceph-mon[100189]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:56.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:55 smithi017 ceph-mon[118128]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:56.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:55 smithi027 systemd[1]: Starting Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 bash[159630]: f93ae52f9b2baa4bf2cca72ea6afa0167f3b7d95d4a445c0611be1dd0109be12 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 ceph-mgr[159695]: -- 172.21.15.27:0/881188838 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a21c0bd1e0 con 0x55a21c095000 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 systemd[1]: Started Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 conmon[159671]: 2023-09-23T14:58:56.663+0000 7f75dd2b8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 conmon[159671]: 2023-09-23T14:58:56.763+0000 7f75dd2b8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:56 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2232555743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:56 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:56 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:56 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2232555743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:57.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2232555743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:57.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:57.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:57.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:56 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:58:57.237 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:56 smithi027 conmon[159671]: 2023-09-23T14:58:56.878+0000 7f75dd2b8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:58:57.238 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:57 smithi027 conmon[159671]: 2023-09-23T14:58:57.004+0000 7f75dd2b8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:58:57.238 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:57 smithi027 conmon[159671]: 2023-09-23T14:58:57.237+0000 7f75dd2b8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:58:58.015 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:57 smithi027 ceph-mon[105164]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:58.015 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:57 smithi027 conmon[159671]: 2023-09-23T14:58:57.739+0000 7f75dd2b8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:58:58.015 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:57 smithi027 conmon[159671]: 2023-09-23T14:58:57.904+0000 7f75dd2b8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:58:58.015 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:58 smithi027 conmon[159671]: 2023-09-23T14:58:58.014+0000 7f75dd2b8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:58:58.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:57 smithi017 ceph-mon[100189]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:57 smithi017 ceph-mon[118128]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:58.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:58 smithi027 conmon[159671]: 2023-09-23T14:58:58.141+0000 7f75dd2b8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:58:58.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:58:58 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:58:58] "GET /metrics HTTP/1.1" 200 239345 "" "Prometheus/2.43.0" 2023-09-23T14:58:58.960 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:58 smithi027 conmon[159671]: 2023-09-23T14:58:58.703+0000 7f75dd2b8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:58:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:58 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/975337346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:58 smithi027 ceph-mon[105164]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:58 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:58 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:58:58 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/975337346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[100189]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/975337346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[118128]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:58:58 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:58:59.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:59 smithi027 conmon[159671]: 2023-09-23T14:58:59.542+0000 7f75dd2b8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:58:59.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:58:59 smithi027 conmon[159671]: 2023-09-23T14:58:59.645+0000 7f75dd2b8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:59:01.017 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:00 smithi027 conmon[159671]: 2023-09-23T14:59:00.668+0000 7f75dd2b8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:01.017 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:00 smithi027 conmon[159671]: 2023-09-23T14:59:00.783+0000 7f75dd2b8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:59:01.017 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:01 smithi027 conmon[159671]: 2023-09-23T14:59:01.016+0000 7f75dd2b8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:01 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2267361153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2267361153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:01.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:01.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:01 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2267361153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:01.669 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:01 smithi027 conmon[159671]: 2023-09-23T14:59:01.403+0000 7f75dd2b8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:59:02.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:01 smithi027 conmon[159671]: 2023-09-23T14:59:01.668+0000 7f75dd2b8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:59:02.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:01 smithi027 conmon[159671]: 2023-09-23T14:59:01.860+0000 7f75dd2b8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:59:02.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:02 smithi027 conmon[159671]: 2023-09-23T14:59:02.197+0000 7f75dd2b8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:59:03.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:03 smithi027 conmon[159671]: 2023-09-23T14:59:03.070+0000 7f75dd2b8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:59:03.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:03 smithi027 conmon[159671]: 2023-09-23T14:59:03.234+0000 7f75dd2b8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:59:03.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:03 smithi027 conmon[159671]: 2023-09-23T14:59:03.515+0000 7f75dd2b8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:59:03.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:03 smithi027 conmon[159671]: 2023-09-23T14:59:03.683+0000 7f75dd2b8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:03.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:03 smithi027 ceph-mon[105164]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:03.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:03 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/173303303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:03 smithi017 ceph-mon[100189]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:03 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/173303303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:03 smithi017 ceph-mon[118128]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 81 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:03 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/173303303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:04.299 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:03 smithi027 conmon[159671]: 2023-09-23T14:59:03.969+0000 7f75dd2b8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 conmon[100620]: ignoring --setuser ceph since I am not root 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 conmon[100620]: ignoring --setgroup ceph since I am not root 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 ceph-mgr[100644]: -- 172.21.15.17:0/2662595570 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f4895089c0 con 0x55f4894e0800 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 conmon[100620]: 2023-09-23T14:59:04.710+0000 7f575dc481c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 conmon[100620]: 2023-09-23T14:59:04.804+0000 7f575dc481c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:04.912 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:04 smithi017 conmon[100620]: 2023-09-23T14:59:04.911+0000 7f575dc481c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:59:05.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:05 smithi017 conmon[100620]: 2023-09-23T14:59:05.020+0000 7f575dc481c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.034+0000 7f75dd2b8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STARTING 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: CherryPy Checker: 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: The Application mounted at '' has an empty config. 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Serving on http://:::9283 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STARTED 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STOPPING 2023-09-23T14:59:05.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:59:05.332 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STOPPED 2023-09-23T14:59:05.332 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STARTING 2023-09-23T14:59:05.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7539728700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7539728700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7539728700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7539728700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7539728700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7536f23700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7536f23700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7536f23700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7536f23700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: 2023-09-23T14:59:05.361+0000 7f7536f23700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Serving on http://:::9283 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:05 smithi027 conmon[159671]: [23/Sep/2023:14:59:05] ENGINE Bus STARTED 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: mgrmap e20: x(active, starting, since 0.00652386s) 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: Active manager daemon x restarted 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: Activating manager daemon x 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: mgrmap e21: x(active, starting, since 0.00522834s) 2023-09-23T14:59:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:05.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: Manager daemon x is now available 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:05.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:05 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.647 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:05 smithi017 conmon[100620]: 2023-09-23T14:59:05.228+0000 7f575dc481c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:59:05.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: mgrmap e20: x(active, starting, since 0.00652386s) 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: Active manager daemon x restarted 2023-09-23T14:59:05.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: Activating manager daemon x 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: mgrmap e21: x(active, starting, since 0.00522834s) 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:05.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 172.21.15.17:0/3460948250' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.14463 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: mgrmap e20: x(active, starting, since 0.00652386s) 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: Active manager daemon x restarted 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: Activating manager daemon x 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: mgrmap e21: x(active, starting, since 0.00522834s) 2023-09-23T14:59:05.651 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:05.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:05.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: Manager daemon x is now available 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: Manager daemon x is now available 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.655 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:05.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:05.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:05.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:05 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:05.923 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:05 smithi017 conmon[100620]: 2023-09-23T14:59:05.646+0000 7f575dc481c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:59:05.923 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:05 smithi017 conmon[100620]: 2023-09-23T14:59:05.760+0000 7f575dc481c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:59:06.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:05 smithi017 conmon[100620]: 2023-09-23T14:59:05.922+0000 7f575dc481c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:59:06.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:06 smithi017 conmon[100620]: 2023-09-23T14:59:06.035+0000 7f575dc481c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:59:06.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:06 smithi017 conmon[100620]: 2023-09-23T14:59:06.552+0000 7f575dc481c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:59:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:06 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4238230272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:06 smithi017 ceph-mon[100189]: mgrmap e22: x(active, since 1.00775s) 2023-09-23T14:59:06.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:06 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4238230272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:06.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:06 smithi017 ceph-mon[118128]: mgrmap e22: x(active, since 1.00775s) 2023-09-23T14:59:06.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:06 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4238230272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:06.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:06 smithi027 ceph-mon[105164]: mgrmap e22: x(active, since 1.00775s) 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[100189]: Deploying cephadm binary to smithi017 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[100189]: Deploying cephadm binary to smithi027 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[100189]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:07.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:07 smithi017 conmon[100620]: 2023-09-23T14:59:07.236+0000 7f575dc481c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:07 smithi017 conmon[100620]: 2023-09-23T14:59:07.430+0000 7f575dc481c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[118128]: Deploying cephadm binary to smithi017 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[118128]: Deploying cephadm binary to smithi027 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[118128]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:07.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:07 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:07 smithi027 ceph-mon[105164]: Deploying cephadm binary to smithi017 2023-09-23T14:59:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:07 smithi027 ceph-mon[105164]: Deploying cephadm binary to smithi027 2023-09-23T14:59:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:07 smithi027 ceph-mon[105164]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:07 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:07 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[100189]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[100189]: mgrmap e23: x(active, since 3s) 2023-09-23T14:59:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2910494469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:08.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:08 smithi017 conmon[100620]: 2023-09-23T14:59:08.422+0000 7f575dc481c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:08.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:08 smithi017 conmon[100620]: 2023-09-23T14:59:08.548+0000 7f575dc481c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:59:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[118128]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[118128]: mgrmap e23: x(active, since 3s) 2023-09-23T14:59:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:08 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2910494469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:08 smithi027 ceph-mon[105164]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:08 smithi027 ceph-mon[105164]: mgrmap e23: x(active, since 3s) 2023-09-23T14:59:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:08 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2910494469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:09.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:08 smithi017 conmon[100620]: 2023-09-23T14:59:08.804+0000 7f575dc481c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:09.682 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:09 smithi017 conmon[100620]: 2023-09-23T14:59:09.198+0000 7f575dc481c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:59:09.682 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:09 smithi017 conmon[100620]: 2023-09-23T14:59:09.318+0000 7f575dc481c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:59:09.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:07] ENGINE Bus STARTING 2023-09-23T14:59:09.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:07] ENGINE Serving on https://172.21.15.27:7150 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:07] ENGINE Serving on http://172.21.15.27:8765 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:07] ENGINE Bus STARTED 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:07] ENGINE Bus STARTING 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:07] ENGINE Serving on https://172.21.15.27:7150 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:07] ENGINE Serving on http://172.21.15.27:8765 2023-09-23T14:59:09.683 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:09 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:07] ENGINE Bus STARTED 2023-09-23T14:59:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:09 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:07] ENGINE Bus STARTING 2023-09-23T14:59:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:09 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:07] ENGINE Serving on https://172.21.15.27:7150 2023-09-23T14:59:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:09 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:07] ENGINE Serving on http://172.21.15.27:8765 2023-09-23T14:59:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:09 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:07] ENGINE Bus STARTED 2023-09-23T14:59:09.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:09 smithi017 conmon[100620]: 2023-09-23T14:59:09.680+0000 7f575dc481c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:59:10.411 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:10 smithi017 conmon[100620]: 2023-09-23T14:59:10.033+0000 7f575dc481c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:59:10.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[118128]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:10.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:10.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:10.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[100189]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:10 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:10 smithi027 ceph-mon[105164]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:10 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:10.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:10 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:10 smithi017 conmon[100620]: 2023-09-23T14:59:10.933+0000 7f575dc481c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:59:11.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:11 smithi017 conmon[100620]: 2023-09-23T14:59:11.106+0000 7f575dc481c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:11 smithi017 conmon[100620]: 2023-09-23T14:59:11.356+0000 7f575dc481c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:11 smithi017 conmon[100620]: 2023-09-23T14:59:11.499+0000 7f575dc481c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2516097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2516097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:11 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2516097415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:11.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:11 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:11 smithi017 conmon[100620]: 2023-09-23T14:59:11.776+0000 7f575dc481c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:59:12.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[100189]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:12.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi017 to 2501M 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[118128]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi017 to 2501M 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:12 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:12 smithi027 ceph-mon[105164]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:12.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:12 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi017 to 2501M 2023-09-23T14:59:12.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:12 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:12.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:12 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:13.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: 2023-09-23T14:59:12.832+0000 7f575dc481c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: [23/Sep/2023:14:59:12] ENGINE Bus STARTING 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: CherryPy Checker: 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: The Application mounted at '' has an empty config. 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: [23/Sep/2023:14:59:12] ENGINE Serving on http://:::9283 2023-09-23T14:59:13.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:12 smithi017 conmon[100620]: [23/Sep/2023:14:59:12] ENGINE Bus STARTED 2023-09-23T14:59:13.416 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:13 smithi027 conmon[159671]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:59:13] "GET /metrics HTTP/1.1" 200 30866 "" "Prometheus/2.43.0" 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: Standby manager daemon y started 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/657165328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: Standby manager daemon y started 2023-09-23T14:59:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/657165328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:13.793 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: Standby manager daemon y started 2023-09-23T14:59:13.793 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:13.794 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:13.794 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:13.794 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.17:0/2075421906' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:13.794 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/657165328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:14.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[100189]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[100189]: mgrmap e24: x(active, since 8s), standbys: y 2023-09-23T14:59:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[118128]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[118128]: mgrmap e24: x(active, since 8s), standbys: y 2023-09-23T14:59:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:14 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:14.798 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:14 smithi027 ceph-mon[105164]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:14.798 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:14 smithi027 ceph-mon[105164]: mgrmap e24: x(active, since 8s), standbys: y 2023-09-23T14:59:14.798 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:14 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:15.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:15 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4261051191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:15 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4261051191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:15.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:15 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4261051191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: Adjusting osd_memory_target on smithi027 to 2500M 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:16.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:16 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: Adjusting osd_memory_target on smithi027 to 2500M 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: Adjusting osd_memory_target on smithi027 to 2500M 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:16.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:16 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:17 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:17.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:17 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:18.471 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:18 smithi017 conmon[100620]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:59:18] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T14:59:18.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[100189]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:59:18.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[100189]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:59:18.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[100189]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:18.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2573306553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:18.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[118128]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:59:18.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[118128]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:59:18.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[118128]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:18.810 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:18 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2573306553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:18.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:18 smithi027 ceph-mon[105164]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T14:59:18.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:18 smithi027 ceph-mon[105164]: Reconfiguring daemon alertmanager.a on smithi017 2023-09-23T14:59:18.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:18 smithi027 ceph-mon[105164]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:18.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:18 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2573306553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:19.531 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:19 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:19.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:19 smithi027 ceph-mon[105164]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:19.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:19 smithi017 ceph-mon[100189]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:19.878 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:19 smithi017 conmon[175786]: ts=2023-09-23T14:59:19.548Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T14:59:19.878 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:19 smithi017 bash[200121]: ca695ff81f5c197ba9f57d58d9ba75ab69e147cf365e9365502c1d1c8eeb2d5a 2023-09-23T14:59:19.878 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:19 smithi017 ceph-mon[118128]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:20.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:20 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2463105712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:20.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:20 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:20.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:20 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2463105712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:20.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:20 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:20.694 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T14:59:20.694 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:20.694 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 systemd[1]: Starting Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:20.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:20 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2463105712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:20.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:20 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:21.192 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.927Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.927Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.928Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.17 port=9094 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.929Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 bash[200476]: 6f0275940b82f12ac4dc3f60554964fb377439d5260964abb25c076d140a571f 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 systemd[1]: Started Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.962Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.963Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.966Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T14:59:21.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:20 smithi017 conmon[200517]: ts=2023-09-23T14:59:20.966Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T14:59:21.660 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:21 smithi017 systemd[1]: Stopping Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:22.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[100189]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T14:59:22.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[100189]: Reconfiguring daemon node-exporter.a on smithi017 2023-09-23T14:59:22.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[100189]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:22.293 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 bash[200748]: af3d2ae00db6e819f00ca79cba753cfb702212344d6824572e86a26aac08fc1d 2023-09-23T14:59:22.293 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T14:59:22.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[118128]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T14:59:22.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[118128]: Reconfiguring daemon node-exporter.a on smithi017 2023-09-23T14:59:22.294 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:21 smithi017 ceph-mon[118128]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:21 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:21 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:21 smithi027 ceph-mon[105164]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T14:59:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:21 smithi027 ceph-mon[105164]: Reconfiguring daemon node-exporter.a on smithi017 2023-09-23T14:59:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:21 smithi027 ceph-mon[105164]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:22.692 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T14:59:22.692 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 systemd[1]: Stopped Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:22.693 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 systemd[1]: Starting Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:23.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[200517]: ts=2023-09-23T14:59:22.930Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000146582s 2023-09-23T14:59:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3400104252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:23.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3400104252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:22 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.843Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.843Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.844Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.844Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.844Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.844Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.844Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T14:59:23.195 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T14:59:23.196 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T14:59:23.197 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T14:59:23.198 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T14:59:23.199 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T14:59:23.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T14:59:23.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 conmon[201139]: ts=2023-09-23T14:59:22.845Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T14:59:23.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 bash[201098]: af1ee2972cfe39dd51babebb84f873468352b112f6cc69990ad9e08970e5386a 2023-09-23T14:59:23.200 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 14:59:22 smithi017 systemd[1]: Started Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:23.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:23 smithi027 conmon[159671]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:59:23] "GET /metrics HTTP/1.1" 200 33764 "" "Prometheus/2.43.0" 2023-09-23T14:59:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:22 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3400104252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:22 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:22 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:23 smithi027 ceph-mon[105164]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:24.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:23 smithi027 ceph-mon[105164]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:24.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:23 smithi027 ceph-mon[105164]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:24.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[100189]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:24.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[100189]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:24.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[100189]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[118128]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[118128]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:23 smithi017 ceph-mon[118128]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:25.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 systemd[1]: Stopping Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:26.118 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T14:59:26.118 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T14:59:26.118 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.856Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.857Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.857Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T14:59:26.119 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:25 smithi027 conmon[154350]: ts=2023-09-23T14:59:25.857Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T14:59:26.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:26 smithi017 ceph-mon[100189]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:26.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:26 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1027623122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:26.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:26 smithi017 ceph-mon[118128]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:26.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:26 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1027623122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:26.530 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:26 smithi027 bash[172955]: 5252779692deffd7b738741a5e766d2a4e33db427d6458df93351e364e9640f7 2023-09-23T14:59:26.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:26 smithi027 ceph-mon[105164]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:26.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:26 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1027623122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:26.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:26 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T14:59:26.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:26 smithi027 systemd[1]: Stopped Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:26.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:26 smithi027 systemd[1]: Starting Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:27.524 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 bash[173281]: b06f0b3b19f7e556a9e043d798ac1ec9bb2d3818ee431eb18425b01c3b8acb16 2023-09-23T14:59:27.524 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 systemd[1]: Started Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:27.524 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.141Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T14:59:27.524 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.141Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.141Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi027 (none))" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.141Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.141Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.143Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.143Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.144Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.144Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.147Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.147Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.923µs 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.147Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T14:59:27.525 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.147Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.165Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.165Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.165Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.498µs wal_replay_duration=18.132988ms wbl_replay_duration=150ns total_replay_duration=18.178551ms 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.168Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.168Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.168Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.185Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.565355ms db_storage=855ns remote_storage=1.129µs web_handler=406ns query_engine=814ns scrape=414.602µs scrape_sd=82.885µs notify=21.937µs notify_sd=21.814µs rules=16.697955ms tracing=5.78µs 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.185Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T14:59:27.526 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:27 smithi027 conmon[173322]: ts=2023-09-23T14:59:27.185Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T14:59:28.184 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:27 smithi027 systemd[1]: Stopping Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:28.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: Reconfiguring daemon node-exporter.b on smithi027 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1713227487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T14:59:28.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: Reconfiguring daemon node-exporter.b on smithi027 2023-09-23T14:59:28.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:28 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1713227487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:28.592 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:28 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T14:59:28.592 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:28 smithi027 bash[173559]: 67db5f1f10b695fbe8a6afd31e313e5f6dbdf2c904b46d84a3055adfe89be2e8 2023-09-23T14:59:28.592 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:28.592 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.592 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:28.592 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T14:59:28.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: Reconfiguring daemon node-exporter.b on smithi027 2023-09-23T14:59:28.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:28 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1713227487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:28.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:28 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T14:59:28.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:28 smithi027 systemd[1]: Stopped Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:28.850 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:28 smithi027 systemd[1]: Starting Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.219Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.219Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.219Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T14:59:29.600 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.220Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T14:59:29.601 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T14:59:29.602 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T14:59:29.603 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 conmon[173928]: ts=2023-09-23T14:59:29.221Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T14:59:29.604 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 bash[173887]: 71fdd501b2ad3b0b035dd3c55ef7ed42de92d538027d90aac449e2e4eaab9068 2023-09-23T14:59:29.605 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 14:59:29 smithi027 systemd[1]: Started Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T14:59:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: Reconfiguring daemon grafana.a on smithi027 2023-09-23T14:59:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:30 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1053650290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: Reconfiguring daemon grafana.a on smithi027 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1053650290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T14:59:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: Reconfiguring daemon grafana.a on smithi027 2023-09-23T14:59:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:30 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1053650290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:31.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:30 smithi017 conmon[200517]: ts=2023-09-23T14:59:30.932Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002864288s 2023-09-23T14:59:31.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:31 smithi027 systemd[1]: Stopping Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:32.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:32 smithi027 bash[174401]: dec828213828f457ad587ecd0199a0fcf8a5af9922e54a095be3c5ff67830dce 2023-09-23T14:59:32.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:32 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T14:59:32.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:32 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T14:59:32.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:32 smithi027 systemd[1]: Stopped Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:32.600 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:32 smithi027 systemd[1]: Starting Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:32 smithi027 ceph-mon[105164]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:32 smithi017 ceph-mon[100189]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:32 smithi017 ceph-mon[118128]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:33.325 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPING 2023-09-23T14:59:33.325 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:59:33.325 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPED 2023-09-23T14:59:33.325 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTING 2023-09-23T14:59:33.325 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Serving on http://:::9283 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTED 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPING 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPED 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTING 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1846731745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.326 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:33 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:33.327 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 bash[174736]: 4653950d68ab094923207bb5fe72d48a86a334bb7b3a7915dcdbcb321e2af267 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 systemd[1]: Started Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203265618Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203482558Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203515888Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.20352995Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203542055Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203553977Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203567348Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203578694Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203596357Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.20360869Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203625246Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T14:59:33.328 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203638947Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203649115Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=settings t=2023-09-23T14:59:33.203660205Z level=info msg="App mode production" 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=sqlstore t=2023-09-23T14:59:33.203739183Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=sqlstore t=2023-09-23T14:59:33.203770112Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=migrator t=2023-09-23T14:59:33.222892445Z level=info msg="Starting DB migrations" 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=migrator t=2023-09-23T14:59:33.228018693Z level=info msg="migrations completed" performed=0 skipped=481 duration=500.914µs 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=sqlstore t=2023-09-23T14:59:33.228462499Z level=info msg="Created default organization" 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=plugin.loader t=2023-09-23T14:59:33.295669793Z level=info msg="Plugin registered" pluginID=input 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=plugin.loader t=2023-09-23T14:59:33.325504724Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=plugin.loader t=2023-09-23T14:59:33.325525253Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T14:59:33.329 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=secrets t=2023-09-23T14:59:33.325579523Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T14:59:33.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=query_data t=2023-09-23T14:59:33.327863723Z level=info msg="Query Service initialization" 2023-09-23T14:59:33.599 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[174777]: logger=live.push_http t=2023-09-23T14:59:33.332395912Z level=info msg="Live Push Gateway initialization" 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Serving on http://:::9283 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTED 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPING 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STOPPED 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTING 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Serving on http://:::9283 2023-09-23T14:59:33.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:33 smithi027 conmon[159671]: [23/Sep/2023:14:59:33] ENGINE Bus STARTED 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1846731745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:33.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1846731745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:33 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=infra.usagestats.collector t=2023-09-23T14:59:34.122834678Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=provisioning.datasources t=2023-09-23T14:59:34.23967305Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=provisioning.datasources t=2023-09-23T14:59:34.2396989Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=provisioning.datasources t=2023-09-23T14:59:34.250051745Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=provisioning.alerting t=2023-09-23T14:59:34.415360595Z level=info msg="starting to provision alerting" 2023-09-23T14:59:34.415 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=provisioning.alerting t=2023-09-23T14:59:34.415389816Z level=info msg="finished to provision alerting" 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.416 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:34 smithi027 ceph-mon[105164]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:34.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[118128]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi017.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027:9095"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi027:3000"}]: dispatch 2023-09-23T14:59:34.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:34 smithi017 ceph-mon[100189]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T14:59:34.849 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=grafanaStorageLogger t=2023-09-23T14:59:34.415552012Z level=info msg="storage starting" 2023-09-23T14:59:34.849 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=ngalert.state.manager t=2023-09-23T14:59:34.415633969Z level=info msg="Warming state cache for startup" 2023-09-23T14:59:34.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=http.server t=2023-09-23T14:59:34.417460146Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T14:59:34.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=ngalert.state.manager t=2023-09-23T14:59:34.484264236Z level=info msg="State cache has been initialized" states=0 duration=68.629343ms 2023-09-23T14:59:34.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=ticker t=2023-09-23T14:59:34.484339502Z level=info msg=starting first_tick=2023-09-23T14:59:40Z 2023-09-23T14:59:34.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 14:59:34 smithi027 conmon[174777]: logger=ngalert.multiorg.alertmanager t=2023-09-23T14:59:34.484332949Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: Upgrade: Updating mgr.y 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: Deploying daemon mgr.y on smithi017 2023-09-23T14:59:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3615035627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: Upgrade: Updating mgr.y 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: Deploying daemon mgr.y on smithi017 2023-09-23T14:59:35.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3615035627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:35.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:35.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:35 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: Upgrade: Updating mgr.y 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: Deploying daemon mgr.y on smithi017 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3615035627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:35 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:36.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:36 smithi017 systemd[1]: Stopping Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:37.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:36 smithi017 bash[201892]: 2de58bf12b45e54b232225eef2ec58c3b832da780aaacab04f0e0448c4ec8dc3 2023-09-23T14:59:37.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:36 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T14:59:37.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T14:59:37.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 systemd[1]: Stopped Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:37.143 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 systemd[1]: Starting Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:38.121 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 ceph-mgr[202339]: -- 172.21.15.17:0/76075610 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562feff011e0 con 0x562fefed8800 2023-09-23T14:59:38.121 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 bash[202274]: a6d14007bbf4b2a57f2ecea427a32212afcb103592de8ae59c6d10b2cba06121 2023-09-23T14:59:38.121 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 systemd[1]: Started Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:37 smithi017 conmon[202315]: 2023-09-23T14:59:37.923+0000 7f62c6a15200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:38 smithi017 conmon[202315]: 2023-09-23T14:59:38.022+0000 7f62c6a15200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[100189]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3087238977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.122 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[118128]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3087238977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.123 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:38 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:38.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:38 smithi017 conmon[202315]: 2023-09-23T14:59:38.142+0000 7f62c6a15200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:59:38.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:38 smithi017 conmon[202315]: 2023-09-23T14:59:38.274+0000 7f62c6a15200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:59:38.467 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:38 smithi027 ceph-mon[105164]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:38.467 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:38 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3087238977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:38.467 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:38 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.468 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:38 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:38.468 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:38 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:38.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:38 smithi027 conmon[159671]: ::ffff:172.21.15.27 - - [23/Sep/2023:14:59:38] "GET /metrics HTTP/1.1" 200 33764 "" "Prometheus/2.43.0" 2023-09-23T14:59:38.850 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:38 smithi027 conmon[173322]: ts=2023-09-23T14:59:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T14:59:38.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:38 smithi017 conmon[202315]: 2023-09-23T14:59:38.499+0000 7f62c6a15200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:59:39.293 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:39 smithi017 conmon[202315]: 2023-09-23T14:59:39.012+0000 7f62c6a15200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:59:39.293 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:39 smithi017 conmon[202315]: 2023-09-23T14:59:39.170+0000 7f62c6a15200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:59:39.293 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:39 smithi017 conmon[202315]: 2023-09-23T14:59:39.292+0000 7f62c6a15200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:59:39.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:39 smithi017 conmon[202315]: 2023-09-23T14:59:39.440+0000 7f62c6a15200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:59:40.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:40 smithi017 conmon[202315]: 2023-09-23T14:59:40.033+0000 7f62c6a15200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:59:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:40 smithi017 ceph-mon[100189]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:40 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1779691631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:40 smithi017 ceph-mon[118128]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:40 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1779691631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:40 smithi027 ceph-mon[105164]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:40.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:40 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1779691631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:41.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:40 smithi017 conmon[202315]: 2023-09-23T14:59:40.878+0000 7f62c6a15200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:59:41.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:40 smithi017 conmon[202315]: 2023-09-23T14:59:40.986+0000 7f62c6a15200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:41 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:41 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:42.331 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:42 smithi017 conmon[202315]: 2023-09-23T14:59:42.048+0000 7f62c6a15200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:42.331 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:42 smithi017 conmon[202315]: 2023-09-23T14:59:42.167+0000 7f62c6a15200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:59:42.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:42 smithi017 conmon[202315]: 2023-09-23T14:59:42.410+0000 7f62c6a15200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:43.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:42 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/935920224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:43.038 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:42 smithi017 conmon[202315]: 2023-09-23T14:59:42.769+0000 7f62c6a15200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:59:43.038 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:43 smithi017 conmon[202315]: 2023-09-23T14:59:43.036+0000 7f62c6a15200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:59:43.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:42 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/935920224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:42 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/935920224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:43.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:43 smithi017 conmon[202315]: 2023-09-23T14:59:43.229+0000 7f62c6a15200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:59:43.942 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:43 smithi017 conmon[200517]: ts=2023-09-23T14:59:43.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi017.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.17:8443: connect: connection refused" 2023-09-23T14:59:43.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:43 smithi017 conmon[202315]: 2023-09-23T14:59:43.583+0000 7f62c6a15200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:59:44.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:43 smithi017 ceph-mon[100189]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:44.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:43 smithi017 ceph-mon[118128]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:43 smithi027 ceph-mon[105164]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:44.654 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:44 smithi017 conmon[202315]: 2023-09-23T14:59:44.488+0000 7f62c6a15200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:59:44.654 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:44 smithi017 conmon[202315]: 2023-09-23T14:59:44.654+0000 7f62c6a15200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:59:45.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:44 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2731833075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:45.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:44 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:45.169 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:44 smithi017 conmon[202315]: 2023-09-23T14:59:44.909+0000 7f62c6a15200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:59:45.169 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:45 smithi017 conmon[202315]: 2023-09-23T14:59:45.068+0000 7f62c6a15200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:45.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:44 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2731833075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:45.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:44 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:44 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2731833075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:44 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.137+0000 7f75d0966700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: ignoring --setuser ceph since I am not root 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: ignoring --setgroup ceph since I am not root 2023-09-23T14:59:45.331 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mgr[159695]: -- 172.21.15.27:0/2040733299 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5626ade2f1e0 con 0x5626ade06c00 2023-09-23T14:59:45.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:45 smithi017 conmon[202315]: 2023-09-23T14:59:45.356+0000 7f62c6a15200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:59:45.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.330+0000 7f9262ef6200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T14:59:45.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.426+0000 7f9262ef6200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:45.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.535+0000 7f9262ef6200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T14:59:45.976 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.647+0000 7f9262ef6200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T14:59:45.976 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:45 smithi027 conmon[159671]: 2023-09-23T14:59:45.854+0000 7f9262ef6200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T14:59:46.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: from='mgr.25486 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T14:59:46.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:45 smithi027 ceph-mon[105164]: mgrmap e25: y(active, starting, since 0.00578021s) 2023-09-23T14:59:46.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: from='mgr.25486 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T14:59:46.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[100189]: mgrmap e25: y(active, starting, since 0.00578021s) 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 172.21.15.27:0/586654799' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: from='mgr.25486 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T14:59:46.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:45 smithi017 ceph-mon[118128]: mgrmap e25: y(active, starting, since 0.00578021s) 2023-09-23T14:59:46.590 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:46 smithi027 conmon[159671]: 2023-09-23T14:59:46.326+0000 7f9262ef6200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T14:59:46.590 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:46 smithi027 conmon[159671]: 2023-09-23T14:59:46.480+0000 7f9262ef6200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T14:59:46.591 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:46 smithi027 conmon[159671]: 2023-09-23T14:59:46.589+0000 7f9262ef6200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.429+0000 7f62c6a15200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STARTING 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: CherryPy Checker: 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: The Application mounted at '' has an empty config. 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Serving on http://:::9283 2023-09-23T14:59:46.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STARTED 2023-09-23T14:59:46.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STOPPING 2023-09-23T14:59:46.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T14:59:46.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STOPPED 2023-09-23T14:59:46.694 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STARTING 2023-09-23T14:59:46.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:46 smithi027 conmon[159671]: 2023-09-23T14:59:46.709+0000 7f9262ef6200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6220eab700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6220eab700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6220eab700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6220eab700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6220eab700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6223eb1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6223eb1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6223eb1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6223eb1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: 2023-09-23T14:59:46.747+0000 7f6223eb1700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Serving on http://:::9283 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:46 smithi017 conmon[202315]: [23/Sep/2023:14:59:46] ENGINE Bus STARTED 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: Active manager daemon y restarted 2023-09-23T14:59:47.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: Activating manager daemon y 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: mgrmap e26: y(active, starting, since 0.00544943s) 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:47.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: Manager daemon y is now available 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: Active manager daemon y restarted 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: Activating manager daemon y 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: mgrmap e26: y(active, starting, since 0.00544943s) 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:47.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:47.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: Manager daemon y is now available 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:47.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:46 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:47.240 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: Active manager daemon y restarted 2023-09-23T14:59:47.240 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: Activating manager daemon y 2023-09-23T14:59:47.240 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T14:59:47.240 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: mgrmap e26: y(active, starting, since 0.00544943s) 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T14:59:47.241 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: Manager daemon y is now available 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T14:59:47.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T14:59:47.243 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T14:59:47.243 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:46 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T14:59:47.243 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:47 smithi027 conmon[159671]: 2023-09-23T14:59:47.238+0000 7f9262ef6200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T14:59:48.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 14:59:47 smithi017 conmon[200517]: ts=2023-09-23T14:59:47.840Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2023-09-23T14:59:48.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3801928791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:48.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[100189]: mgrmap e27: y(active, since 1.00895s) 2023-09-23T14:59:48.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:47] ENGINE Bus STARTING 2023-09-23T14:59:48.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:47] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:59:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3801928791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[118128]: mgrmap e27: y(active, since 1.00895s) 2023-09-23T14:59:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:47] ENGINE Bus STARTING 2023-09-23T14:59:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:47 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:47] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:59:48.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:48 smithi027 conmon[159671]: 2023-09-23T14:59:48.115+0000 7f9262ef6200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T14:59:48.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:48 smithi027 conmon[159671]: 2023-09-23T14:59:48.223+0000 7f9262ef6200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T14:59:48.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:47 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3801928791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:48.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:47 smithi027 ceph-mon[105164]: mgrmap e27: y(active, since 1.00895s) 2023-09-23T14:59:48.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:47 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:47] ENGINE Bus STARTING 2023-09-23T14:59:48.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:47 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:47] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T14:59:49.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:49 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:47] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T14:59:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:49 smithi027 ceph-mon[105164]: [23/Sep/2023:14:59:47] ENGINE Bus STARTED 2023-09-23T14:59:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:49 smithi027 ceph-mon[105164]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:49.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:49 smithi027 conmon[159671]: 2023-09-23T14:59:49.279+0000 7f9262ef6200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:49.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:49 smithi027 conmon[159671]: 2023-09-23T14:59:49.412+0000 7f9262ef6200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T14:59:49.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:47] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T14:59:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[100189]: [23/Sep/2023:14:59:47] ENGINE Bus STARTED 2023-09-23T14:59:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[100189]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:47] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T14:59:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[118128]: [23/Sep/2023:14:59:47] ENGINE Bus STARTED 2023-09-23T14:59:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:49 smithi017 ceph-mon[118128]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:50.047 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:49 smithi027 conmon[159671]: 2023-09-23T14:59:49.655+0000 7f9262ef6200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T14:59:50.321 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:50 smithi027 conmon[159671]: 2023-09-23T14:59:50.046+0000 7f9262ef6200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T14:59:50.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:50 smithi027 conmon[159671]: 2023-09-23T14:59:50.320+0000 7f9262ef6200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:50 smithi027 conmon[159671]: 2023-09-23T14:59:50.513+0000 7f9262ef6200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: mgrmap e28: y(active, since 3s) 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1964889302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:50 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: mgrmap e28: y(active, since 3s) 2023-09-23T14:59:50.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1964889302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: mgrmap e28: y(active, since 3s) 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1964889302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:50 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:50 smithi027 conmon[159671]: 2023-09-23T14:59:50.865+0000 7f9262ef6200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T14:59:51.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[100189]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[118128]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:51 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:51 smithi027 conmon[159671]: 2023-09-23T14:59:51.715+0000 7f9262ef6200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T14:59:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:51 smithi027 ceph-mon[105164]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:51 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:51 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:51 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:51 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:52.130 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:51 smithi027 conmon[159671]: 2023-09-23T14:59:51.877+0000 7f9262ef6200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T14:59:52.552 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:52 smithi027 conmon[159671]: 2023-09-23T14:59:52.129+0000 7f9262ef6200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T14:59:52.552 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:52 smithi027 conmon[159671]: 2023-09-23T14:59:52.269+0000 7f9262ef6200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T14:59:52.552 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:52 smithi027 conmon[159671]: 2023-09-23T14:59:52.551+0000 7f9262ef6200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T14:59:52.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:52 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3982944571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:52 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3982944571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:52 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3982944571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:53.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:53 smithi027 ceph-mon[105164]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: 2023-09-23T14:59:53.608+0000 7f9262ef6200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: [23/Sep/2023:14:59:53] ENGINE Bus STARTING 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: CherryPy Checker: 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: The Application mounted at '' has an empty config. 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: [23/Sep/2023:14:59:53] ENGINE Serving on http://:::9283 2023-09-23T14:59:53.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 14:59:53 smithi027 conmon[159671]: [23/Sep/2023:14:59:53] ENGINE Bus STARTED 2023-09-23T14:59:53.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:53 smithi017 ceph-mon[100189]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:53.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:53 smithi017 ceph-mon[118128]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:54.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: Standby manager daemon x started 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:54 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/875689450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: Standby manager daemon x started 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/875689450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: Standby manager daemon x started 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/749417812' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:54.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T14:59:54.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T14:59:54.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:54 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/875689450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: mgrmap e29: y(active, since 8s), standbys: x 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:55 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: mgrmap e29: y(active, since 8s), standbys: x 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T14:59:55.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: mgrmap e29: y(active, since 8s), standbys: x 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:55.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:55 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T14:59:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:56 smithi027 ceph-mon[105164]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:56 smithi027 ceph-mon[105164]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:56.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:56 smithi017 ceph-mon[100189]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:56.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:56 smithi017 ceph-mon[100189]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:56 smithi017 ceph-mon[118128]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T14:59:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:56 smithi017 ceph-mon[118128]: Reconfiguring daemon prometheus.a on smithi027 2023-09-23T14:59:57.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:57 smithi027 ceph-mon[105164]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:57.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:57 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4216902255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:57.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:57 smithi017 ceph-mon[100189]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:57 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4216902255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:57 smithi017 ceph-mon[118128]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:57 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4216902255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T14:59:58.236 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:57 smithi027 systemd[1]: Stopping Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.235Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.236Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T14:59:58.600 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.236Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T14:59:58.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.236Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T14:59:58.601 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 conmon[173322]: ts=2023-09-23T14:59:58.236Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T14:59:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:58 smithi027 ceph-mon[105164]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:58 smithi027 bash[178761]: b06f0b3b19f7e556a9e043d798ac1ec9bb2d3818ee431eb18425b01c3b8acb16 2023-09-23T14:59:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T14:59:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 systemd[1]: Stopped Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 systemd[1]: Starting Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T14:59:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:58 smithi017 ceph-mon[100189]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:58 smithi017 ceph-mon[118128]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T14:59:59.976 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 bash[179082]: bbda5037d08246b2c7c00e55e081d85c1c359abf576cad47204f5c2296ea44e1 2023-09-23T14:59:59.976 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.644Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.644Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.644Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi027 (none))" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.645Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.645Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 systemd[1]: Started Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.646Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.647Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.650Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.651Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.652Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.652Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.76µs 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.652Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T14:59:59.977 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.652Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.669Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.672Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.673Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.673Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=31.261µs wal_replay_duration=21.556403ms wbl_replay_duration=123ns total_replay_duration=21.608001ms 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.678Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.678Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.678Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.693Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.856819ms db_storage=987ns remote_storage=1.5µs web_handler=480ns query_engine=785ns scrape=490.144µs scrape_sd=80.203µs notify=12.174µs notify_sd=9.981µs rules=13.941776ms tracing=5.55µs 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.693Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T14:59:59.978 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 14:59:59 smithi027 conmon[179123]: ts=2023-09-23T14:59:59.693Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE Bus STOPPING 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE Bus STOPPED 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE Bus STARTING 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE Serving on http://:::9283 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 14:59:59 smithi017 conmon[202315]: [23/Sep/2023:14:59:59] ENGINE Bus STARTED 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/3994374081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3994374081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 14:59:59 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3994374081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi027.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T15:00:00.251 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:00.252 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 14:59:59 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:00 smithi027 ceph-mon[105164]: overall HEALTH_OK 2023-09-23T15:00:01.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:00 smithi027 ceph-mon[105164]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:01.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:00 smithi017 ceph-mon[100189]: overall HEALTH_OK 2023-09-23T15:00:01.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:00 smithi017 ceph-mon[100189]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:00 smithi017 ceph-mon[118128]: overall HEALTH_OK 2023-09-23T15:00:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:00 smithi017 ceph-mon[118128]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: Upgrade: Updating mgr.x 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: Deploying daemon mgr.x on smithi027 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/825462534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:01 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: Upgrade: Updating mgr.x 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: Deploying daemon mgr.x on smithi027 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/825462534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: Upgrade: Updating mgr.x 2023-09-23T15:00:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:00:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:00:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: Deploying daemon mgr.x on smithi027 2023-09-23T15:00:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/825462534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:01 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:02.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:02 smithi027 systemd[1]: Stopping Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:00:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:02 smithi027 ceph-mon[105164]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:03.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:02 smithi017 ceph-mon[100189]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:02 smithi017 ceph-mon[118128]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:03.716 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:03 smithi027 bash[179758]: f93ae52f9b2baa4bf2cca72ea6afa0167f3b7d95d4a445c0611be1dd0109be12 2023-09-23T15:00:03.716 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:03 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T15:00:03.717 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:03 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T15:00:03.717 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:03 smithi027 systemd[1]: Stopped Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:00:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:04 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2172837317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:04.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:03 smithi027 systemd[1]: Starting Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:00:04.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 ceph-mgr[180174]: -- 172.21.15.27:0/2089180629 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559483c951e0 con 0x559483c6d000 2023-09-23T15:00:04.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 bash[180109]: a039252b558eec6c98d5dab85884591460425fa1563f239bb9625aa1be4eef6d 2023-09-23T15:00:04.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 systemd[1]: Started Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:00:04.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:04 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2172837317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:04 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2172837317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:04.805 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 conmon[180150]: 2023-09-23T15:00:04.456+0000 7f570c9da200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T15:00:04.805 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 conmon[180150]: 2023-09-23T15:00:04.556+0000 7f570c9da200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:04.805 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 conmon[180150]: 2023-09-23T15:00:04.675+0000 7f570c9da200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T15:00:04.805 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:04 smithi027 conmon[180150]: 2023-09-23T15:00:04.804+0000 7f570c9da200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T15:00:05.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:05 smithi027 conmon[180150]: 2023-09-23T15:00:05.028+0000 7f570c9da200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[100189]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:05 smithi017 ceph-mon[118128]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:05 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:05 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:05 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:05 smithi027 ceph-mon[105164]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:05 smithi027 conmon[180150]: 2023-09-23T15:00:05.536+0000 7f570c9da200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T15:00:05.696 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:05 smithi027 conmon[180150]: 2023-09-23T15:00:05.695+0000 7f570c9da200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T15:00:06.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:05 smithi027 conmon[180150]: 2023-09-23T15:00:05.808+0000 7f570c9da200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T15:00:06.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:05 smithi027 conmon[180150]: 2023-09-23T15:00:05.942+0000 7f570c9da200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T15:00:06.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:06 smithi027 conmon[180150]: 2023-09-23T15:00:06.506+0000 7f570c9da200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T15:00:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:06 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/4107239306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:06 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/4107239306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:06.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:06 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/4107239306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:07 smithi027 ceph-mon[105164]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:07.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:07 smithi027 conmon[180150]: 2023-09-23T15:00:07.377+0000 7f570c9da200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T15:00:07.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:07 smithi027 conmon[180150]: 2023-09-23T15:00:07.485+0000 7f570c9da200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T15:00:07.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:07 smithi017 ceph-mon[100189]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:07 smithi017 ceph-mon[118128]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:08.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:00:08] "GET /metrics HTTP/1.1" 200 33760 "" "Prometheus/2.43.0" 2023-09-23T15:00:08.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:08 smithi027 conmon[180150]: 2023-09-23T15:00:08.502+0000 7f570c9da200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:08.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:08 smithi027 conmon[180150]: 2023-09-23T15:00:08.624+0000 7f570c9da200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T15:00:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:08 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:08 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:08 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:08 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:08.851 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:00:08 smithi027 conmon[179123]: ts=2023-09-23T15:00:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:00:09.123 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:08 smithi027 conmon[180150]: 2023-09-23T15:00:08.871+0000 7f570c9da200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:09.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:08 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:09.426 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:09 smithi027 conmon[180150]: 2023-09-23T15:00:09.243+0000 7f570c9da200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T15:00:09.692 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:09 smithi027 conmon[180150]: 2023-09-23T15:00:09.505+0000 7f570c9da200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T15:00:10.041 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:09 smithi027 ceph-mon[105164]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:10.041 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:09 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2547248181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:10.041 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:09 smithi027 conmon[180150]: 2023-09-23T15:00:09.691+0000 7f570c9da200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T15:00:10.041 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:10 smithi027 conmon[180150]: 2023-09-23T15:00:10.039+0000 7f570c9da200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T15:00:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:09 smithi017 ceph-mon[118128]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:09 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2547248181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:10.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:09 smithi017 ceph-mon[100189]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:10.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:09 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2547248181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:11.233 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:10 smithi027 conmon[180150]: 2023-09-23T15:00:10.924+0000 7f570c9da200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T15:00:11.234 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:11 smithi027 conmon[180150]: 2023-09-23T15:00:11.088+0000 7f570c9da200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T15:00:11.490 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:11 smithi027 conmon[180150]: 2023-09-23T15:00:11.348+0000 7f570c9da200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T15:00:11.490 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:11 smithi027 conmon[180150]: 2023-09-23T15:00:11.488+0000 7f570c9da200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/2838896950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:11.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:11 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:11 smithi027 conmon[180150]: 2023-09-23T15:00:11.773+0000 7f570c9da200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/2838896950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/2838896950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:11 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T15:00:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:00:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:12 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: 2023-09-23T15:00:12.839+0000 7f570c9da200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: [23/Sep/2023:15:00:12] ENGINE Bus STARTING 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: CherryPy Checker: 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: The Application mounted at '' has an empty config. 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: [23/Sep/2023:15:00:12] ENGINE Serving on http://:::9283 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:12 smithi027 conmon[180150]: [23/Sep/2023:15:00:12] ENGINE Bus STARTED 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:00:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:12 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T15:00:14.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: Upgrade: Setting container_image for all mgr 2023-09-23T15:00:14.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: Upgrade: It appears safe to stop mon.a 2023-09-23T15:00:14.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: Standby manager daemon x restarted 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: Standby manager daemon x started 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:14.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[100189]: from='client.? 172.21.15.17:0/1085873930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: Upgrade: Setting container_image for all mgr 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: Upgrade: It appears safe to stop mon.a 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: Standby manager daemon x restarted 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: Standby manager daemon x started 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:14.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:14.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:14.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:14.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:00:14.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:00:14.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:14.027 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:13 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1085873930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: Upgrade: Setting container_image for all mgr 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: Upgrade: It appears safe to stop mon.a 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: Standby manager daemon x restarted 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: Standby manager daemon x started 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/21324357' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:00:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:00:14.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:14.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:13 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1085873930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[100189]: Upgrade: Updating mon.a 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[100189]: Deploying daemon mon.a on smithi017 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[100189]: mgrmap e30: y(active, since 27s), standbys: x 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:14 smithi017 systemd[1]: Stopping Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[118128]: Upgrade: Updating mon.a 2023-09-23T15:00:14.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[118128]: Deploying daemon mon.a on smithi017 2023-09-23T15:00:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:14 smithi017 ceph-mon[118128]: mgrmap e30: y(active, since 27s), standbys: x 2023-09-23T15:00:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:14 smithi027 ceph-mon[105164]: Upgrade: Updating mon.a 2023-09-23T15:00:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:14 smithi027 ceph-mon[105164]: Deploying daemon mon.a on smithi017 2023-09-23T15:00:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:14 smithi027 ceph-mon[105164]: mgrmap e30: y(active, since 27s), standbys: x 2023-09-23T15:00:15.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[100165]: 2023-09-23T15:00:15.094+0000 7f64eaf14700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[100165]: 2023-09-23T15:00:15.094+0000 7f64eaf14700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE Bus STOPPING 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE Bus STOPPED 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE Bus STARTING 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE Serving on http://:::9283 2023-09-23T15:00:15.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:15 smithi017 conmon[202315]: [23/Sep/2023:15:00:15] ENGINE Bus STARTED 2023-09-23T15:00:15.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:15 smithi017 bash[215185]: 54ba4e8b4801d1a4f4db77268365ee916418be42aad1a8bcb363e076147935eb 2023-09-23T15:00:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:15 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T15:00:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:15 smithi017 systemd[1]: Stopped Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:00:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 systemd[1]: Starting Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:00:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: pidfile_write: ignore empty --pid-file 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 bash[215581]: 0012d0aac43233a9a64ca3d3bc47c1628b9f60f0c0536b122dfb01d4efccfaae 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: load: jerasure load: lrc 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: RocksDB version: 7.9.2 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Git sha 0 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: DB SUMMARY 2023-09-23T15:00:16.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: DB Session ID: KSB66OR0UZ5WOI4IBGPC 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: CURRENT file: CURRENT 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1170 Bytes 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000033.sst 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000031.log size: 370659 ; 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.error_if_exists: 0 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.create_if_missing: 0 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.paranoid_checks: 1 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T15:00:16.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.env: 0x5593b6b49720 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.info_log: 0x5593ba08d420 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.statistics: (nil) 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.use_fsync: 0 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_log_file_size: 0 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 systemd[1]: Started Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:00:16.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_fallocate: 1 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.use_direct_reads: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.db_log_dir: 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.wal_dir: 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T15:00:16.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.write_buffer_manager: 0x5593b931c5a0 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T15:00:16.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.unordered_write: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.row_cache: None 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.wal_filter: None 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.two_write_queues: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T15:00:16.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.wal_compression: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.atomic_flush: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.log_readahead_size: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T15:00:16.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T15:00:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T15:00:16.951 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_background_jobs: 2 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_background_compactions: -1 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_subcompactions: 1 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T15:00:16.952 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_open_files: -1 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_background_flushes: -1 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Compression algorithms supported: 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kZSTD supported: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kXpressCompression supported: 0 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T15:00:16.953 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kZlibCompression supported: 1 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kSnappyCompression supported: 1 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kLZ4Compression supported: 1 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: kBZip2Compression supported: 0 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.merge_operator: 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_filter: None 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T15:00:16.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5593ba08d520) 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks: 1 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: pin_top_level_index_and_filter: 1 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_type: 0 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_index_type: 0 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_shortening: 1 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T15:00:16.955 INFO:journalctl@ceph.mon.a.smithi017.stdout: checksum: 4 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: no_block_cache: 0 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache: 0x5593b939f350 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_name: BinnedLRUCache 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_options: 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: capacity : 536870912 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_shard_bits : 4 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: strict_capacity_limit : 0 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: high_pri_pool_ratio: 0.000 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_cache_compressed: (nil) 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: persistent_cache: (nil) 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size: 4096 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_size_deviation: 10 2023-09-23T15:00:16.956 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_restart_interval: 16 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: index_block_restart_interval: 1 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_block_size: 4096 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: partition_filters: 0 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: use_delta_encoding: 1 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: filter_policy: bloomfilter 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: whole_key_filtering: 1 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: verify_compression: 0 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: read_amp_bytes_per_bit: 0 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: format_version: 5 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_index_compression: 1 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: block_align: 0 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_auto_readahead_size: 262144 2023-09-23T15:00:16.957 INFO:journalctl@ceph.mon.a.smithi017.stdout: prepopulate_block_cache: 0 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout: initial_auto_readahead_size: 8192 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression: NoCompression 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.num_levels: 7 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T15:00:16.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T15:00:16.959 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T15:00:16.960 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T15:00:16.961 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T15:00:16.962 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.table_properties_collectors: 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.inplace_update_support: 0 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T15:00:16.963 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.bloom_locality: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.max_successive_merges: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.ttl: 2592000 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enable_blob_files: false 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.min_blob_size: 0 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T15:00:16.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 33.sst 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 35, last_sequence is 12575, log_number is 31,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 31 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 242a97b8-636c-478c-a7eb-30aa3083b328 2023-09-23T15:00:16.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481216613263, "job": 1, "event": "recovery_started", "wal_files": [31]} 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #31 mode 2 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481216615221, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 36, "file_size": 357341, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12567, "largest_seqno": 12772, "table_properties": {"data_size": 355435, "index_size": 574, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 325, "raw_key_size": 3008, "raw_average_key_size": 26, "raw_value_size": 352900, "raw_average_value_size": 3095, "num_data_blocks": 25, "num_entries": 114, "num_filter_entries": 114, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695481216, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "242a97b8-636c-478c-a7eb-30aa3083b328", "db_session_id": "KSB66OR0UZ5WOI4IBGPC", "orig_file_number": 36, "seqno_to_time_mapping": "N/A"}} 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481216615294, "job": 1, "event": "recovery_finished"} 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:5047] Creating manifest 38 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000031.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5593b943a000 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: DB pointer 0x5593b9428000 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** DB Stats ** 2023-09-23T15:00:16.966 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: L0 1/0 348.97 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 298.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T15:00:16.967 INFO:journalctl@ceph.mon.a.smithi017.stdout: L6 1/0 6.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Sum 2/0 6.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 298.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 298.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 298.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:00:16.968 INFO:journalctl@ceph.mon.a.smithi017.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: Cumulative compaction: 0.00 GB write, 76.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: Interval compaction: 0.00 GB write, 76.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache BinnedLRUCache@0x5593b939f350#7 capacity: 512.00 MB usage: 1.02 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.38 KB,7.15256e-05%) IndexBlock(1,0.64 KB,0.00012219%) Misc(1,0.00 KB,0%) 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T15:00:16.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: starting mon.a rank 0 at public addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] at bind addrs [v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???) e3 preinit fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).mds e9 new map 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).mds e9 print_map 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: e9 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: legacy client fscid: 1 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: Filesystem 'foo' (1) 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: fs_name foo 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: epoch 8 2023-09-23T15:00:16.970 INFO:journalctl@ceph.mon.a.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: created 2023-09-23T14:56:31.343039+0000 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: modified 2023-09-23T14:56:42.940454+0000 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: tableserver 0 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: root 0 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_timeout 60 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: session_autoclose 300 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_file_size 1099511627776 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: required_client_features {} 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure 0 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: last_failure_osd_epoch 0 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:00:16.971 INFO:journalctl@ceph.mon.a.smithi017.stdout: max_mds 1 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: in 0 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: up {0=14676} 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: failed 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: damaged 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: stopped 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: data_pools [4] 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: metadata_pool 3 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: inline_data disabled 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: balancer 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: bal_rank_mask -1 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: standby_count_wanted 1 2023-09-23T15:00:16.972 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi017.ygjhom{0:14676} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: Standby daemons: 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout: [mds.foo.smithi027.ynevql{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).osd e90 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).osd e90 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expand map: {default=false} 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta from 'false' to 'false' 2023-09-23T15:00:16.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expanded map: {default=false} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expand map: {default=info} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta from 'info' to 'info' 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expanded map: {default=info} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expand map: {default=daemon} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expand map: {default=debug} 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T15:00:16.974 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:16 smithi017 ceph-mon[215646]: expand_channel_meta expanded map: {default=debug} 2023-09-23T15:00:18.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:00:18] "GET /metrics HTTP/1.1" 200 33693 "" "Prometheus/2.43.0" 2023-09-23T15:00:19.101 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:00:18 smithi027 conmon[179123]: ts=2023-09-23T15:00:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:19 smithi027 conmon[180150]: ignoring --setuser ceph since I am not root 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:19 smithi027 conmon[180150]: ignoring --setgroup ceph since I am not root 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mgr[180174]: -- 172.21.15.27:0/2307535293 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563c67efb1e0 con 0x563c67ed3000 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:19 smithi027 conmon[180150]: 2023-09-23T15:00:19.863+0000 7fa8ef6f7200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:19 smithi027 conmon[180150]: 2023-09-23T15:00:19.960+0000 7fa8ef6f7200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:19.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1887589300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3124922742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: mon.a calling monitor election 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T15:00:19.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T15:00:19.963 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T15:00:19.963 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: overall HEALTH_OK 2023-09-23T15:00:19.963 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:19 smithi027 ceph-mon[105164]: from='mgr.15816 ' entity='' 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1887589300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3124922742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: mon.a calling monitor election 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: overall HEALTH_OK 2023-09-23T15:00:19.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[215646]: from='mgr.15816 ' entity='' 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:19 smithi017 conmon[202315]: ignoring --setuser ceph since I am not root 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:19 smithi017 conmon[202315]: ignoring --setgroup ceph since I am not root 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mgr[202339]: -- 172.21.15.17:0/1443887304 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55dc5a4a51e0 con 0x55dc5a47c800 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:19 smithi017 conmon[202315]: 2023-09-23T15:00:19.867+0000 7fdcff65d200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:19 smithi017 conmon[202315]: 2023-09-23T15:00:19.963+0000 7fdcff65d200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:19.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1887589300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='mgr.15816 172.21.15.17:0/3539421366' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/3124922742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: mon.a calling monitor election 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: osdmap e90: 8 total, 8 up, 8 in 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: mgrmap e30: y(active, since 33s), standbys: x 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: mgrmap e31: y(active, since 33s), standbys: x 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: overall HEALTH_OK 2023-09-23T15:00:19.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:19 smithi017 ceph-mon[118128]: from='mgr.15816 ' entity='' 2023-09-23T15:00:20.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:20 smithi027 conmon[180150]: 2023-09-23T15:00:20.069+0000 7fa8ef6f7200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T15:00:20.350 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:20 smithi027 conmon[180150]: 2023-09-23T15:00:20.181+0000 7fa8ef6f7200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T15:00:20.392 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:20 smithi017 conmon[202315]: 2023-09-23T15:00:20.072+0000 7fdcff65d200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T15:00:20.392 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:20 smithi017 conmon[202315]: 2023-09-23T15:00:20.184+0000 7fdcff65d200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T15:00:20.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:20 smithi017 conmon[202315]: 2023-09-23T15:00:20.392+0000 7fdcff65d200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T15:00:20.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:20 smithi027 conmon[180150]: 2023-09-23T15:00:20.389+0000 7fa8ef6f7200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T15:00:21.020 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T15:00:21.124 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:20 smithi027 conmon[180150]: 2023-09-23T15:00:20.861+0000 7fa8ef6f7200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T15:00:21.124 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:21 smithi027 conmon[180150]: 2023-09-23T15:00:21.014+0000 7fa8ef6f7200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T15:00:21.124 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:21 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1743604416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:21.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1743604416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:21.146 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:20 smithi017 conmon[202315]: 2023-09-23T15:00:20.875+0000 7fdcff65d200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T15:00:21.146 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:21 smithi017 conmon[202315]: 2023-09-23T15:00:21.030+0000 7fdcff65d200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T15:00:21.146 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:21 smithi017 conmon[202315]: 2023-09-23T15:00:21.144+0000 7fdcff65d200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T15:00:21.146 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:21 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1743604416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:00:21.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:21 smithi017 conmon[202315]: 2023-09-23T15:00:21.264+0000 7fdcff65d200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T15:00:21.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:21 smithi027 conmon[180150]: 2023-09-23T15:00:21.123+0000 7fa8ef6f7200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T15:00:21.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:21 smithi027 conmon[180150]: 2023-09-23T15:00:21.239+0000 7fa8ef6f7200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T15:00:21.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T15:00:22.096 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:00:22.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:21 smithi027 conmon[180150]: 2023-09-23T15:00:21.752+0000 7fa8ef6f7200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T15:00:22.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:21 smithi017 conmon[202315]: 2023-09-23T15:00:21.787+0000 7fdcff65d200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T15:00:22.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:22 smithi027 conmon[180150]: 2023-09-23T15:00:22.532+0000 7fa8ef6f7200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T15:00:22.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:22 smithi027 conmon[180150]: 2023-09-23T15:00:22.634+0000 7fa8ef6f7200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T15:00:22.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:22 smithi017 conmon[202315]: 2023-09-23T15:00:22.620+0000 7fdcff65d200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T15:00:22.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:22 smithi017 conmon[202315]: 2023-09-23T15:00:22.725+0000 7fdcff65d200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T15:00:23.861 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:23 smithi017 conmon[202315]: 2023-09-23T15:00:23.740+0000 7fdcff65d200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:23.861 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:23 smithi017 conmon[202315]: 2023-09-23T15:00:23.860+0000 7fdcff65d200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T15:00:23.953 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:23 smithi027 conmon[180150]: 2023-09-23T15:00:23.615+0000 7fa8ef6f7200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:23.953 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:23 smithi027 conmon[180150]: 2023-09-23T15:00:23.729+0000 7fa8ef6f7200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T15:00:23.954 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:23 smithi027 conmon[180150]: 2023-09-23T15:00:23.952+0000 7fa8ef6f7200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:24.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:24 smithi017 conmon[202315]: 2023-09-23T15:00:24.087+0000 7fdcff65d200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T15:00:24.564 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:24 smithi027 conmon[180150]: 2023-09-23T15:00:24.308+0000 7fa8ef6f7200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T15:00:24.703 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:24 smithi017 conmon[202315]: 2023-09-23T15:00:24.445+0000 7fdcff65d200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T15:00:24.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:24 smithi027 conmon[180150]: 2023-09-23T15:00:24.563+0000 7fa8ef6f7200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T15:00:24.849 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:24 smithi027 conmon[180150]: 2023-09-23T15:00:24.747+0000 7fa8ef6f7200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T15:00:25.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:24 smithi017 conmon[202315]: 2023-09-23T15:00:24.702+0000 7fdcff65d200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T15:00:25.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:24 smithi017 conmon[202315]: 2023-09-23T15:00:24.888+0000 7fdcff65d200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T15:00:25.349 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:25 smithi027 conmon[180150]: 2023-09-23T15:00:25.084+0000 7fa8ef6f7200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T15:00:25.692 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:25 smithi017 conmon[202315]: 2023-09-23T15:00:25.228+0000 7fdcff65d200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T15:00:26.335 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:25 smithi027 conmon[180150]: 2023-09-23T15:00:25.921+0000 7fa8ef6f7200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T15:00:26.336 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:26 smithi027 conmon[180150]: 2023-09-23T15:00:26.082+0000 7fa8ef6f7200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T15:00:26.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:26 smithi017 conmon[202315]: 2023-09-23T15:00:26.071+0000 7fdcff65d200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T15:00:26.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:26 smithi017 conmon[202315]: 2023-09-23T15:00:26.233+0000 7fdcff65d200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T15:00:26.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:26 smithi027 conmon[180150]: 2023-09-23T15:00:26.334+0000 7fa8ef6f7200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T15:00:26.599 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:26 smithi027 conmon[180150]: 2023-09-23T15:00:26.475+0000 7fa8ef6f7200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:26.916 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:26 smithi017 conmon[202315]: 2023-09-23T15:00:26.488+0000 7fdcff65d200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T15:00:26.916 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:26 smithi017 conmon[202315]: 2023-09-23T15:00:26.630+0000 7fdcff65d200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T15:00:27.099 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:26 smithi027 conmon[180150]: 2023-09-23T15:00:26.756+0000 7fa8ef6f7200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T15:00:27.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:26 smithi017 conmon[202315]: 2023-09-23T15:00:26.915+0000 7fdcff65d200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: 2023-09-23T15:00:27.812+0000 7fa8ef6f7200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: [23/Sep/2023:15:00:27] ENGINE Bus STARTING 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: CherryPy Checker: 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: The Application mounted at '' has an empty config. 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: [23/Sep/2023:15:00:27] ENGINE Serving on http://:::9283 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:00:27 smithi027 conmon[180150]: [23/Sep/2023:15:00:27] ENGINE Bus STARTED 2023-09-23T15:00:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: Standby manager daemon x restarted 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: Standby manager daemon x started 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:27 smithi027 ceph-mon[105164]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:28.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 conmon[105140]: 2023-09-23T15:00:28.005+0000 7fefd0fe8700 -1 mon.b@1(peon).osd e91 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: Standby manager daemon x restarted 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: Standby manager daemon x started 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[215646]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:27 smithi017 conmon[202315]: 2023-09-23T15:00:27.989+0000 7fdcff65d200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T15:00:28.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: [23/Sep/2023:15:00:28] ENGINE Bus STARTING 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: Standby manager daemon x restarted 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: Standby manager daemon x started 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:27 smithi017 ceph-mon[118128]: from='mgr.? 172.21.15.27:0/594551117' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T15:00:28.160 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[118104]: 2023-09-23T15:00:28.005+0000 7f836b5c9700 -1 mon.c@2(peon).osd e91 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T15:00:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc5f1fe700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc5f1fe700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc5f1fe700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc5f1fe700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc5f1fe700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc599b3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc599b3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc599b3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc599b3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.161+0000 7fdc599b3700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: CherryPy Checker: 2023-09-23T15:00:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: The Application mounted at '' has an empty config. 2023-09-23T15:00:28.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: 2023-09-23T15:00:28.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: [23/Sep/2023:15:00:28] ENGINE Serving on http://:::9283 2023-09-23T15:00:28.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:28 smithi017 conmon[202315]: [23/Sep/2023:15:00:28] ENGINE Bus STARTED 2023-09-23T15:00:29.035 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T15:00:29.035 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (68s) - 4m 21.2M - 0.25.0 c8568f914cd2 6f0275940b82 2023-09-23T15:00:29.035 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (56s) - 4m 82.3M - 9.4.7 2c41d148cca3 4653950d68ab 2023-09-23T15:00:29.035 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (3m) - 3m 16.4M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T15:00:29.035 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (3m) - 3m 16.7M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443,9283,8765 running (25s) - 8m 43.1M - 18.2.0-460-gbb0daeff e32c2a64c126 a039252b558e 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (51s) - 10m 473M - 18.2.0-460-gbb0daeff e32c2a64c126 a6d14007bbf4 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (10m) - 10m 100M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 54ba4e8b4801 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 running (8m) - 8m 70.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 38ba724092d3 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (8m) - 8m 71.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 4d08e21f44ed 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (66s) - 4m 8472k - 1.5.0 0da6a335fe13 af1ee2972cfe 2023-09-23T15:00:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (60s) - 4m 15.0M - 1.5.0 0da6a335fe13 71fdd501b2ad 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (8m) - 8m 77.9M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 bb8e8a0efb36 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (7m) - 7m 91.5M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1542a5a16945 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (7m) - 7m 77.8M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 d6cc1ec2d677 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (7m) - 7m 66.1M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1e310e53ac98 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (6m) - 6m 69.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (6m) - 6m 80.4M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (6m) - 6m 77.4M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T15:00:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (5m) - 5m 89.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T15:00:29.038 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (29s) - 5m 39.5M - 2.43.0 a07b618ecd1d bbda5037d082 2023-09-23T15:00:29.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:29 smithi017 conmon[202315]: 2023-09-23T15:00:29.015+0000 7fdcb8de7700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T15:00:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: mgrmap e32: y(active, since 41s), standbys: x 2023-09-23T15:00:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: Active manager daemon y restarted 2023-09-23T15:00:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: Activating manager daemon y 2023-09-23T15:00:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: mgrmap e33: y(active, starting, since 0.00666823s), standbys: x 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:00:29.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:00:29.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: Manager daemon y is now available 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:29.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T15:00:29.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T15:00:29.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: mgrmap e32: y(active, since 41s), standbys: x 2023-09-23T15:00:29.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: Active manager daemon y restarted 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: Activating manager daemon y 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: mgrmap e33: y(active, starting, since 0.00666823s), standbys: x 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:29.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:00:29.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:00:29.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:00:29.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:00:29.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T15:00:29.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: Manager daemon y is now available 2023-09-23T15:00:29.203 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:29.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T15:00:29.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.204 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: mgrmap e32: y(active, since 41s), standbys: x 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: Active manager daemon y restarted 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: Activating manager daemon y 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: mgrmap e33: y(active, starting, since 0.00666823s), standbys: x 2023-09-23T15:00:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi017.ygjhom"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi027.ynevql"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: Manager daemon y is now available 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:29.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T15:00:29.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: failed to encode map e91 with expected crc 2023-09-23T15:00:29.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T15:00:29.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T15:00:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:30 smithi027 ceph-mon[105164]: mgrmap e34: y(active, since 1.00963s), standbys: x 2023-09-23T15:00:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:30 smithi027 ceph-mon[105164]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:30 smithi027 ceph-mon[105164]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[215646]: mgrmap e34: y(active, since 1.00963s), standbys: x 2023-09-23T15:00:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[215646]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[215646]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[118128]: mgrmap e34: y(active, since 1.00963s), standbys: x 2023-09-23T15:00:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[118128]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:30 smithi017 ceph-mon[118128]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:30.486 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:00:31.016 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[215646]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T15:00:31.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[215646]: [23/Sep/2023:15:00:30] ENGINE Bus STARTING 2023-09-23T15:00:31.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[215646]: [23/Sep/2023:15:00:30] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T15:00:31.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[118128]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T15:00:31.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[118128]: [23/Sep/2023:15:00:30] ENGINE Bus STARTING 2023-09-23T15:00:31.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:31 smithi017 ceph-mon[118128]: [23/Sep/2023:15:00:30] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T15:00:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:31 smithi027 ceph-mon[105164]: mgrmap e35: y(active, since 2s), standbys: x 2023-09-23T15:00:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:31 smithi027 ceph-mon[105164]: [23/Sep/2023:15:00:30] ENGINE Bus STARTING 2023-09-23T15:00:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:31 smithi027 ceph-mon[105164]: [23/Sep/2023:15:00:30] ENGINE Serving on https://172.21.15.17:7150 2023-09-23T15:00:32.574 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: [23/Sep/2023:15:00:30] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: [23/Sep/2023:15:00:30] ENGINE Bus STARTED 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: [23/Sep/2023:15:00:30] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: [23/Sep/2023:15:00:30] ENGINE Bus STARTED 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.784 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: [23/Sep/2023:15:00:30] ENGINE Serving on http://172.21.15.17:8765 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: [23/Sep/2023:15:00:30] ENGINE Bus STARTED 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.839 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:32 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T15:00:33.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:33 smithi027 ceph-mon[105164]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:33.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:33 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:33 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:33 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.839 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:00:33.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[215646]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[118128]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:33 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:34 smithi017 ceph-mon[118128]: from='client.25889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:34 smithi017 ceph-mon[118128]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:00:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:34 smithi017 ceph-mon[215646]: from='client.25889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:34 smithi017 ceph-mon[215646]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:00:34.914 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:34 smithi027 ceph-mon[105164]: from='client.25889 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:00:34.914 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:34 smithi027 ceph-mon[105164]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:00:35.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[215646]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[118128]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:35 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:35 smithi027 ceph-mon[105164]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:35 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:35 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:35 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:35 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:36 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.conf 2023-09-23T15:00:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.conf 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:36 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:37.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: Updating smithi017:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: Updating smithi027:/etc/ceph/ceph.client.admin.keyring 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: Updating smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: Updating smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:00:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:00:38.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:00:38] "GET /metrics HTTP/1.1" 200 31075 "" "Prometheus/2.43.0" 2023-09-23T15:00:39.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:00:38 smithi027 conmon[179123]: ts=2023-09-23T15:00:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:00:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:39 smithi017 ceph-mon[215646]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:39 smithi017 ceph-mon[118128]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:39 smithi027 ceph-mon[105164]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:41 smithi017 ceph-mon[215646]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:41 smithi017 ceph-mon[118128]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:41 smithi027 ceph-mon[105164]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:43.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:43 smithi017 ceph-mon[215646]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:43 smithi017 ceph-mon[118128]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:43 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:43 smithi027 ceph-mon[105164]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:43 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:45.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:45 smithi017 ceph-mon[215646]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:45.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:45 smithi017 ceph-mon[118128]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:45 smithi027 ceph-mon[105164]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:47.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:47 smithi017 ceph-mon[215646]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:47 smithi017 ceph-mon[118128]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:47 smithi027 ceph-mon[105164]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:48.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:00:48] "GET /metrics HTTP/1.1" 200 33758 "" "Prometheus/2.43.0" 2023-09-23T15:00:49.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:00:48 smithi027 conmon[179123]: ts=2023-09-23T15:00:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:00:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:49 smithi017 ceph-mon[215646]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:49 smithi017 ceph-mon[118128]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:49 smithi027 ceph-mon[105164]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:51 smithi017 ceph-mon[215646]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:51 smithi017 ceph-mon[118128]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:51 smithi027 ceph-mon[105164]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:53 smithi017 ceph-mon[215646]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:53 smithi017 ceph-mon[118128]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:53 smithi027 ceph-mon[105164]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:55 smithi017 ceph-mon[215646]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:55 smithi017 ceph-mon[118128]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:55 smithi027 ceph-mon[105164]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:57 smithi017 ceph-mon[215646]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:57 smithi017 ceph-mon[118128]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:57 smithi027 ceph-mon[105164]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:58.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:00:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:00:58] "GET /metrics HTTP/1.1" 200 33758 "" "Prometheus/2.43.0" 2023-09-23T15:00:58.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:58.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:58 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:00:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:00:58 smithi027 conmon[179123]: ts=2023-09-23T15:00:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:00:59.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:00:59 smithi017 ceph-mon[215646]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:00:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:00:59 smithi017 ceph-mon[118128]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:00:59 smithi027 ceph-mon[105164]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:01.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:01 smithi017 ceph-mon[215646]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:01.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:01 smithi017 ceph-mon[118128]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:01 smithi027 ceph-mon[105164]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:03 smithi017 ceph-mon[215646]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:03 smithi017 ceph-mon[118128]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:04.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:03 smithi027 ceph-mon[105164]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:05.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:05 smithi017 ceph-mon[215646]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:05 smithi017 ceph-mon[118128]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:05 smithi027 ceph-mon[105164]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:07 smithi017 ceph-mon[215646]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:07 smithi017 ceph-mon[118128]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:07 smithi027 ceph-mon[105164]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:08.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:08] "GET /metrics HTTP/1.1" 200 33759 "" "Prometheus/2.43.0" 2023-09-23T15:01:09.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:08 smithi027 conmon[179123]: ts=2023-09-23T15:01:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:09.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:09 smithi017 ceph-mon[215646]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:09.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:09 smithi017 ceph-mon[118128]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:10.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:09 smithi027 ceph-mon[105164]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:11.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:11 smithi017 ceph-mon[215646]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:11.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:11 smithi017 ceph-mon[118128]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:11 smithi027 ceph-mon[105164]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:13 smithi017 ceph-mon[215646]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:13 smithi017 ceph-mon[118128]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:13 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:13 smithi027 ceph-mon[105164]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:13 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:15 smithi017 ceph-mon[215646]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:15 smithi017 ceph-mon[118128]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:15 smithi027 ceph-mon[105164]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:18.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:17 smithi017 ceph-mon[215646]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:18.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:17 smithi017 ceph-mon[118128]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:17 smithi027 ceph-mon[105164]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:18.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:18] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T15:01:19.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:18 smithi027 conmon[179123]: ts=2023-09-23T15:01:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:19 smithi027 ceph-mon[105164]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:19 smithi017 ceph-mon[215646]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:19 smithi017 ceph-mon[118128]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:21 smithi027 ceph-mon[105164]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:21 smithi017 ceph-mon[215646]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:21 smithi017 ceph-mon[118128]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:23 smithi027 ceph-mon[105164]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:24.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:23 smithi017 ceph-mon[215646]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:23 smithi017 ceph-mon[118128]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:25 smithi027 ceph-mon[105164]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:26.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:25 smithi017 ceph-mon[215646]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:25 smithi017 ceph-mon[118128]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:28.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:27 smithi017 ceph-mon[215646]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:28.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:27 smithi017 ceph-mon[118128]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:27 smithi027 ceph-mon[105164]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:28] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T15:01:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:29.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:28 smithi027 conmon[179123]: ts=2023-09-23T15:01:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:28 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:29 smithi027 ceph-mon[105164]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:30.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:29 smithi017 ceph-mon[215646]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:30.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:29 smithi017 ceph-mon[118128]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:31 smithi027 ceph-mon[105164]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:31 smithi017 ceph-mon[215646]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:31 smithi017 ceph-mon[118128]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:33 smithi027 ceph-mon[105164]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:33 smithi017 ceph-mon[215646]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:33 smithi017 ceph-mon[118128]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:35.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T15:01:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:35 smithi027 ceph-mon[105164]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:36.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:35 smithi017 ceph-mon[215646]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:35 smithi017 ceph-mon[118128]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:36.205 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:37 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:37 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:38.120 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:38.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:38] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T15:01:38.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T15:01:39.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:38 smithi027 conmon[179123]: ts=2023-09-23T15:01:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:39.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[215646]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[215646]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[215646]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[118128]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[118128]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[118128]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:39 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:39.387 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:39 smithi027 ceph-mon[105164]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:39 smithi027 ceph-mon[105164]: from='client.34164 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:39 smithi027 ceph-mon[105164]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:39 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.099 INFO:teuthology.orchestra.run.smithi017.stdout:wait for majority of mons to upgrade 2023-09-23T15:01:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[215646]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[118128]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:41.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:41 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T15:01:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:41 smithi027 ceph-mon[105164]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:41 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:41 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[215646]: Upgrade: It appears safe to stop mon.c 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:42.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:42.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:01:42.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:01:42.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T15:01:42.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:42 smithi017 ceph-mon[118128]: Upgrade: It appears safe to stop mon.c 2023-09-23T15:01:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T15:01:43.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:42 smithi027 ceph-mon[105164]: Upgrade: It appears safe to stop mon.c 2023-09-23T15:01:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1233905263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:43 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:44.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:43 smithi017 ceph-mon[118128]: from='client.? 172.21.15.17:0/1233905263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:43 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:43 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/1233905263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: Upgrade: Updating mon.c 2023-09-23T15:01:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:01:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[215646]: Deploying daemon mon.c on smithi017 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: Upgrade: Updating mon.c 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:45 smithi017 ceph-mon[118128]: Deploying daemon mon.c on smithi017 2023-09-23T15:01:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: Upgrade: Updating mon.c 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:45 smithi027 ceph-mon[105164]: Deploying daemon mon.c on smithi017 2023-09-23T15:01:46.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:46 smithi017 systemd[1]: Stopping Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:01:46.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:46 smithi017 conmon[118104]: 2023-09-23T15:01:46.346+0000 7f8370dd4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:01:46.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:46 smithi017 conmon[118104]: 2023-09-23T15:01:46.346+0000 7f8370dd4700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T15:01:47.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 bash[222461]: 4d08e21f44ed510bf43c3f09b79d99b9a67912c5a83ae9fa30b0d7f9f3c0da23 2023-09-23T15:01:47.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T15:01:47.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 systemd[1]: Stopped Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:01:47.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 systemd[1]: Starting Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:01:48.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:48] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T15:01:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T15:01:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T15:01:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: pidfile_write: ignore empty --pid-file 2023-09-23T15:01:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 bash[222813]: 7f42350c36be2dea8073ca37fbfdf2fb1e4fa380d38de7adf8ddcfd2cfb07f9a 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 systemd[1]: Started Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: load: jerasure load: lrc 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: RocksDB version: 7.9.2 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Git sha 0 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: DB SUMMARY 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: DB Session ID: 0CGGL1IW1DQQ3L9MF1EE 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: CURRENT file: CURRENT 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T15:01:48.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1095 Bytes 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 4000283 ; 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.error_if_exists: 0 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.create_if_missing: 0 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.paranoid_checks: 1 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.env: 0x55654accb720 2023-09-23T15:01:48.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.info_log: 0x55654cab7420 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.statistics: (nil) 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.use_fsync: 0 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_log_file_size: 0 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T15:01:48.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_fallocate: 1 2023-09-23T15:01:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T15:01:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T15:01:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.use_direct_reads: 0 2023-09-23T15:01:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T15:01:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T15:01:48.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.db_log_dir: 2023-09-23T15:01:48.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.wal_dir: 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.write_buffer_manager: 0x55654bd465a0 2023-09-23T15:01:48.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.unordered_write: 0 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T15:01:48.200 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.row_cache: None 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.wal_filter: None 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.two_write_queues: 0 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.wal_compression: 0 2023-09-23T15:01:48.201 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.atomic_flush: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.log_readahead_size: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T15:01:48.202 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_background_jobs: 2 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_background_compactions: -1 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_subcompactions: 1 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T15:01:48.203 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_open_files: -1 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_background_flushes: -1 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Compression algorithms supported: 2023-09-23T15:01:48.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kZSTD supported: 0 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kXpressCompression supported: 0 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kZlibCompression supported: 1 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kSnappyCompression supported: 1 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kLZ4Compression supported: 1 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: kBZip2Compression supported: 0 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T15:01:48.205 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.merge_operator: 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_filter: None 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T15:01:48.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55654cab7520) 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks: 1 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: pin_top_level_index_and_filter: 1 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_type: 0 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_index_type: 0 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_shortening: 1 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: checksum: 4 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: no_block_cache: 0 2023-09-23T15:01:48.207 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache: 0x55654bdc9350 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_name: BinnedLRUCache 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_options: 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: capacity : 536870912 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_shard_bits : 4 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: strict_capacity_limit : 0 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: high_pri_pool_ratio: 0.000 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_cache_compressed: (nil) 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: persistent_cache: (nil) 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size: 4096 2023-09-23T15:01:48.208 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_size_deviation: 10 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_restart_interval: 16 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: index_block_restart_interval: 1 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_block_size: 4096 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: partition_filters: 0 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: use_delta_encoding: 1 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: filter_policy: bloomfilter 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: whole_key_filtering: 1 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: verify_compression: 0 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: read_amp_bytes_per_bit: 0 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: format_version: 5 2023-09-23T15:01:48.209 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_index_compression: 1 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout: block_align: 0 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_auto_readahead_size: 262144 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout: prepopulate_block_cache: 0 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout: initial_auto_readahead_size: 8192 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression: NoCompression 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T15:01:48.210 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.num_levels: 7 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:01:48.211 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T15:01:48.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T15:01:48.213 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T15:01:48.214 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T15:01:48.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.table_properties_collectors: 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.inplace_update_support: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.bloom_locality: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.max_successive_merges: 0 2023-09-23T15:01:48.216 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.ttl: 2592000 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enable_blob_files: false 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.min_blob_size: 0 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T15:01:48.217 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13132, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T15:01:48.218 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: fceac21a-a61e-4cf1-b12d-eae3a55e2da8 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481307862976, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481307873277, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2512187, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13127, "largest_seqno": 14658, "table_properties": {"data_size": 2505787, "index_size": 3909, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 12899, "raw_average_key_size": 22, "raw_value_size": 2493494, "raw_average_value_size": 4413, "num_data_blocks": 178, "num_entries": 565, "num_filter_entries": 565, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695481307, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "fceac21a-a61e-4cf1-b12d-eae3a55e2da8", "db_session_id": "0CGGL1IW1DQQ3L9MF1EE", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481307873379, "job": 1, "event": "recovery_finished"} 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55654be64000 2023-09-23T15:01:48.219 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: DB pointer 0x55654be52000 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** DB Stats ** 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:01:48.220 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: L0 1/0 2.40 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: L6 1/0 6.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: Sum 2/0 8.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 572.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** Compaction Stats [default] ** 2023-09-23T15:01:48.221 INFO:journalctl@ceph.mon.c.smithi017.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 572.5 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T15:01:48.222 INFO:journalctl@ceph.mon.c.smithi017.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: Cumulative compaction: 0.00 GB write, 183.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: Interval compaction: 0.00 GB write, 183.22 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache BinnedLRUCache@0x55654bdc9350#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: starting mon.c rank 2 at public addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] at bind addrs [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???) e3 preinit fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:01:48.223 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).mds e9 new map 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).mds e9 print_map 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: e9 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: legacy client fscid: 1 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: Filesystem 'foo' (1) 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: fs_name foo 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: epoch 8 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T15:01:48.224 INFO:journalctl@ceph.mon.c.smithi017.stdout: created 2023-09-23T14:56:31.343039+0000 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: modified 2023-09-23T14:56:42.940454+0000 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: tableserver 0 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: root 0 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_timeout 60 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: session_autoclose 300 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_file_size 1099511627776 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: required_client_features {} 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure 0 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: last_failure_osd_epoch 0 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:01:48.225 INFO:journalctl@ceph.mon.c.smithi017.stdout: max_mds 1 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: in 0 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: up {0=14676} 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: failed 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: damaged 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: stopped 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: data_pools [4] 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: metadata_pool 3 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: inline_data disabled 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: balancer 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: bal_rank_mask -1 2023-09-23T15:01:48.226 INFO:journalctl@ceph.mon.c.smithi017.stdout: standby_count_wanted 1 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi017.ygjhom{0:14676} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: Standby daemons: 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout: [mds.foo.smithi027.ynevql{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:01:48.227 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expand map: {default=false} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta from 'false' to 'false' 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expanded map: {default=false} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expand map: {default=info} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta from 'info' to 'info' 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expanded map: {default=info} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expand map: {default=daemon} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T15:01:48.228 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expand map: {default=debug} 2023-09-23T15:01:48.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T15:01:48.229 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:47 smithi017 ceph-mon[222878]: expand_channel_meta expanded map: {default=debug} 2023-09-23T15:01:48.752 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T15:01:49.051 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:48 smithi027 conmon[179123]: ts=2023-09-23T15:01:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:49.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: mon.c calling monitor election 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: mon.a calling monitor election 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:01:49.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: mgrmap e35: y(active, since 80s), standbys: x 2023-09-23T15:01:49.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: overall HEALTH_OK 2023-09-23T15:01:49.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:49.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:49 smithi027 ceph-mon[105164]: from='client.? 172.21.15.17:0/3053057835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:49.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: mon.c calling monitor election 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: mon.a calling monitor election 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:01:49.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: mgrmap e35: y(active, since 80s), standbys: x 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: overall HEALTH_OK 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3053057835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: mon.c calling monitor election 2023-09-23T15:01:49.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: mon.a calling monitor election 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: monmap e3: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: mgrmap e35: y(active, since 80s), standbys: x 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: overall HEALTH_OK 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:49.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:49.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3053057835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:01:49.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T15:01:50.203 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:51.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:51 smithi027 ceph-mon[105164]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:51.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:51 smithi017 ceph-mon[215646]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:51.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:51 smithi017 ceph-mon[222878]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:52.466 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (2m) - 6m 29.2M - 0.25.0 c8568f914cd2 6f0275940b82 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (2m) - 5m 84.8M - 9.4.7 2c41d148cca3 4653950d68ab 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (5m) - 5m 17.7M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (5m) - 5m 17.3M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443,9283,8765 running (108s) - 9m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 a039252b558e 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (2m) - 11m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 a6d14007bbf4 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (96s) - 11m 45.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0012d0aac432 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 running (10m) - 10m 80.5M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 38ba724092d3 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4s) - 10m 25.1M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7f42350c36be 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (2m) - 6m 18.7M - 1.5.0 0da6a335fe13 af1ee2972cfe 2023-09-23T15:01:52.467 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (2m) - 6m 17.3M - 1.5.0 0da6a335fe13 71fdd501b2ad 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (9m) - 9m 83.8M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 bb8e8a0efb36 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (9m) - 9m 97.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1542a5a16945 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (8m) - 8m 81.6M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 d6cc1ec2d677 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (8m) - 8m 72.0M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1e310e53ac98 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (8m) - 8m 74.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (7m) - 7m 83.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (7m) - 7m 81.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (7m) - 7m 94.2M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T15:01:52.468 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (113s) - 6m 52.8M - 2.43.0 a07b618ecd1d bbda5037d082 2023-09-23T15:01:53.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:53.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:53.147 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.149 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:52 smithi017 conmon[202315]: 2023-09-23T15:01:52.884+0000 7fdcb8de7700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T15:01:53.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T15:01:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: from='client.34188 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:53 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:53.775 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:55.129 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:55 smithi017 ceph-mon[215646]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:55.129 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:55 smithi017 ceph-mon[222878]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:55 smithi027 ceph-mon[105164]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:55.960 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:56.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T15:01:57.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:01:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:56 smithi027 ceph-mon[105164]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:56 smithi027 ceph-mon[105164]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:56 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:56 smithi027 ceph-mon[105164]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[215646]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[215646]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[215646]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[222878]: from='client.34194 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[222878]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:01:57.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:57.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:56 smithi017 ceph-mon[222878]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:58.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:01:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:01:58] "GET /metrics HTTP/1.1" 200 33754 "" "Prometheus/2.43.0" 2023-09-23T15:01:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:01:58.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:01:59.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:01:58 smithi027 conmon[179123]: ts=2023-09-23T15:01:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:01:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:01:59 smithi027 ceph-mon[105164]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:59.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:01:59 smithi017 ceph-mon[215646]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:01:59.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:01:59 smithi017 ceph-mon[222878]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:00 smithi027 ceph-mon[105164]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T15:02:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:00 smithi017 ceph-mon[215646]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T15:02:00.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:00 smithi017 ceph-mon[222878]: mgrmap e36: y(active, since 91s), standbys: x 2023-09-23T15:02:01.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:01 smithi027 ceph-mon[105164]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:01.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:01 smithi017 ceph-mon[215646]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:01.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:01 smithi017 ceph-mon[222878]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:03.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:03 smithi017 ceph-mon[215646]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:03 smithi017 ceph-mon[222878]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:03 smithi027 ceph-mon[105164]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:05.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:05 smithi017 ceph-mon[215646]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:05 smithi017 ceph-mon[222878]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:05 smithi027 ceph-mon[105164]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:07 smithi017 ceph-mon[215646]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:07 smithi017 ceph-mon[222878]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:07 smithi027 ceph-mon[105164]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:08.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:08] "GET /metrics HTTP/1.1" 200 33749 "" "Prometheus/2.43.0" 2023-09-23T15:02:09.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:08 smithi027 conmon[179123]: ts=2023-09-23T15:02:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:09.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:09 smithi017 ceph-mon[215646]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:09.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:09 smithi017 ceph-mon[222878]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:09 smithi027 ceph-mon[105164]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:11.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:11 smithi017 ceph-mon[215646]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:11.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:11 smithi017 ceph-mon[222878]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:11 smithi027 ceph-mon[105164]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T15:02:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:13 smithi017 ceph-mon[215646]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:13.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:13 smithi017 ceph-mon[222878]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:13.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:13 smithi027 ceph-mon[105164]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:13 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:15 smithi017 ceph-mon[215646]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:15 smithi017 ceph-mon[222878]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:15 smithi027 ceph-mon[105164]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:17 smithi017 ceph-mon[215646]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:17 smithi017 ceph-mon[222878]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:17 smithi027 ceph-mon[105164]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:18.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:18] "GET /metrics HTTP/1.1" 200 33749 "" "Prometheus/2.43.0" 2023-09-23T15:02:19.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:18 smithi027 conmon[179123]: ts=2023-09-23T15:02:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:19 smithi017 ceph-mon[215646]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:19 smithi017 ceph-mon[222878]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:19 smithi027 ceph-mon[105164]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:21 smithi017 ceph-mon[215646]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:21 smithi017 ceph-mon[222878]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:21 smithi027 ceph-mon[105164]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:23 smithi017 ceph-mon[215646]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:23 smithi017 ceph-mon[222878]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:23 smithi027 ceph-mon[105164]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:25 smithi017 ceph-mon[215646]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:25 smithi017 ceph-mon[222878]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:25 smithi027 ceph-mon[105164]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:27 smithi017 ceph-mon[215646]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:27 smithi017 ceph-mon[222878]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:27 smithi027 ceph-mon[105164]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:28.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:28] "GET /metrics HTTP/1.1" 200 33749 "" "Prometheus/2.43.0" 2023-09-23T15:02:28.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:28.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:28 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:29.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:28 smithi027 conmon[179123]: ts=2023-09-23T15:02:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:29.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:29 smithi017 ceph-mon[215646]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:29.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:29 smithi017 ceph-mon[222878]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:29 smithi027 ceph-mon[105164]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:31 smithi027 ceph-mon[105164]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:31 smithi017 ceph-mon[215646]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:31 smithi017 ceph-mon[222878]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:33 smithi027 ceph-mon[105164]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:34.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:33 smithi017 ceph-mon[215646]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:33 smithi017 ceph-mon[222878]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:35 smithi027 ceph-mon[105164]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:36.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:35 smithi017 ceph-mon[215646]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:35 smithi017 ceph-mon[222878]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:38.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:37 smithi017 ceph-mon[215646]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:38.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:37 smithi017 ceph-mon[222878]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:37 smithi027 ceph-mon[105164]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:38.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:38] "GET /metrics HTTP/1.1" 200 33750 "" "Prometheus/2.43.0" 2023-09-23T15:02:39.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:38 smithi027 conmon[179123]: ts=2023-09-23T15:02:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:39 smithi027 ceph-mon[105164]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:39 smithi017 ceph-mon[215646]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:39 smithi017 ceph-mon[222878]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:41 smithi027 ceph-mon[105164]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:41 smithi017 ceph-mon[215646]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:42.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:41 smithi017 ceph-mon[222878]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:43 smithi027 ceph-mon[105164]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:43 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:44.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:43 smithi017 ceph-mon[215646]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:43 smithi017 ceph-mon[222878]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:45 smithi027 ceph-mon[105164]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:45 smithi017 ceph-mon[215646]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:45 smithi017 ceph-mon[222878]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:47 smithi017 ceph-mon[215646]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:48.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:47 smithi017 ceph-mon[222878]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:47 smithi027 ceph-mon[105164]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:48.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:48] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.43.0" 2023-09-23T15:02:49.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:48 smithi027 conmon[179123]: ts=2023-09-23T15:02:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:49 smithi027 ceph-mon[105164]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:50.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:49 smithi017 ceph-mon[215646]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:49 smithi017 ceph-mon[222878]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:51 smithi027 ceph-mon[105164]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:51 smithi017 ceph-mon[215646]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:52.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:51 smithi017 ceph-mon[222878]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:53 smithi027 ceph-mon[105164]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:54.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:53 smithi017 ceph-mon[215646]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:53 smithi017 ceph-mon[222878]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:55 smithi027 ceph-mon[105164]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:55 smithi017 ceph-mon[215646]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:56.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:55 smithi017 ceph-mon[222878]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:57 smithi017 ceph-mon[215646]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:02:58.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:57 smithi017 ceph-mon[222878]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:58.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:02:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:57 smithi027 ceph-mon[105164]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:02:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:57 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:02:58.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:02:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:02:58] "GET /metrics HTTP/1.1" 200 33763 "" "Prometheus/2.43.0" 2023-09-23T15:02:59.073 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:59.073 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:58 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:02:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:02:58 smithi027 conmon[179123]: ts=2023-09-23T15:02:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:02:59.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T15:02:59.737 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:02:59 smithi017 ceph-mon[222878]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:02:59 smithi027 ceph-mon[105164]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:00.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:00.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:02:59 smithi017 ceph-mon[215646]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:01.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:01.833 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:01.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:01.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.835 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:01.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:01.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:01.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:02.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:01 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:02.249 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:02.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T15:03:03.293 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:03.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:03.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:03.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T15:03:03.502 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[222878]: Upgrade: It appears safe to stop mon.b 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T15:03:03.503 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:03 smithi017 ceph-mon[215646]: Upgrade: It appears safe to stop mon.b 2023-09-23T15:03:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T15:03:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T15:03:03.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:03 smithi027 ceph-mon[105164]: Upgrade: It appears safe to stop mon.b 2023-09-23T15:03:04.773 INFO:teuthology.orchestra.run.smithi017.stdout:wait for all mons to upgrade 2023-09-23T15:03:04.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: Upgrade: Updating mon.b 2023-09-23T15:03:04.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: Deploying daemon mon.b on smithi027 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[215646]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: Upgrade: Updating mon.b 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:04.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:04.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: Deploying daemon mon.b on smithi027 2023-09-23T15:03:04.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:04 smithi017 ceph-mon[222878]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: Upgrade: Updating mon.b 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: Deploying daemon mon.b on smithi027 2023-09-23T15:03:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:04 smithi027 ceph-mon[105164]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:05.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T15:03:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:05 smithi027 systemd[1]: Stopping Ceph mon.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:03:05.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:05.871 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:05 smithi027 conmon[105140]: 2023-09-23T15:03:05.602+0000 7fefd67f3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:03:05.871 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:05 smithi027 conmon[105140]: 2023-09-23T15:03:05.602+0000 7fefd67f3700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T15:03:06.328 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:05 smithi027 bash[194808]: 38ba724092d303bc23f824a4bf3d5bf8444028bc5ecfff5c041963ac2bd47973 2023-09-23T15:03:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T15:03:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 systemd[1]: Stopped Ceph mon.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:03:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 systemd[1]: Starting Ceph mon.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:03:07.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: pidfile_write: ignore empty --pid-file 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 bash[195168]: 0f598c97295ecd84227db90daafe9d4c00bc70a6151107bdeacef5b1c35eb71b 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: load: jerasure load: lrc 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: RocksDB version: 7.9.2 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Git sha 0 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T15:03:07.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: DB SUMMARY 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: DB Session ID: 67FHRD26Q9U2FV35VIEM 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: CURRENT file: CURRENT 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1095 Bytes 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 5626813 ; 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.error_if_exists: 0 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.create_if_missing: 0 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.paranoid_checks: 1 2023-09-23T15:03:07.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.env: 0x561fd9be7720 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.info_log: 0x561fdbcf9420 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.statistics: (nil) 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.use_fsync: 0 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_log_file_size: 0 2023-09-23T15:03:07.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_fallocate: 1 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.use_direct_reads: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T15:03:07.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.db_log_dir: 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.wal_dir: 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T15:03:07.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.write_buffer_manager: 0x561fdaf885a0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.unordered_write: 0 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T15:03:07.356 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.row_cache: None 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.wal_filter: None 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.two_write_queues: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.wal_compression: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.atomic_flush: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T15:03:07.357 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.log_readahead_size: 0 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_background_jobs: 2 2023-09-23T15:03:07.358 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_background_compactions: -1 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_subcompactions: 1 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_open_files: -1 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T15:03:07.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_background_flushes: -1 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Compression algorithms supported: 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kZSTD supported: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kXpressCompression supported: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kZlibCompression supported: 1 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kSnappyCompression supported: 1 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kLZ4Compression supported: 1 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: kBZip2Compression supported: 0 2023-09-23T15:03:07.360 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.merge_operator: 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_filter: None 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561fdbcf9520) 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout: cache_index_and_filter_blocks: 1 2023-09-23T15:03:07.361 INFO:journalctl@ceph.mon.b.smithi027.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: pin_top_level_index_and_filter: 1 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: index_type: 0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: data_block_index_type: 0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: index_shortening: 1 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: checksum: 4 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: no_block_cache: 0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_cache: 0x561fdb00b1f0 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_cache_name: BinnedLRUCache 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_cache_options: 2023-09-23T15:03:07.362 INFO:journalctl@ceph.mon.b.smithi027.stdout: capacity : 536870912 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: num_shard_bits : 4 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: strict_capacity_limit : 0 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: high_pri_pool_ratio: 0.000 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_cache_compressed: (nil) 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: persistent_cache: (nil) 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_size: 4096 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_size_deviation: 10 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_restart_interval: 16 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: index_block_restart_interval: 1 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: metadata_block_size: 4096 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: partition_filters: 0 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: use_delta_encoding: 1 2023-09-23T15:03:07.363 INFO:journalctl@ceph.mon.b.smithi027.stdout: filter_policy: bloomfilter 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: whole_key_filtering: 1 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: verify_compression: 0 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: read_amp_bytes_per_bit: 0 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: format_version: 5 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: enable_index_compression: 1 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: block_align: 0 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: max_auto_readahead_size: 262144 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: prepopulate_block_cache: 0 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: initial_auto_readahead_size: 8192 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T15:03:07.364 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression: NoCompression 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.num_levels: 7 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T15:03:07.365 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T15:03:07.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T15:03:07.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:03:07.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T15:03:07.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T15:03:07.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T15:03:07.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T15:03:07.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.table_properties_collectors: 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.inplace_update_support: 0 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T15:03:07.370 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.bloom_locality: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.max_successive_merges: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.ttl: 2592000 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T15:03:07.371 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enable_blob_files: false 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.min_blob_size: 0 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13208, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T15:03:07.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: df9d4009-41d3-4eb8-b427-d230947bb0f7 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481386963470, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 systemd[1]: Started Ceph mon.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481386981627, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 3497209, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13203, "largest_seqno": 16087, "table_properties": {"data_size": 3486787, "index_size": 6521, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2885, "raw_key_size": 24165, "raw_average_key_size": 21, "raw_value_size": 3464081, "raw_average_value_size": 3014, "num_data_blocks": 296, "num_entries": 1149, "num_filter_entries": 1149, "num_deletions": 253, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695481386, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "df9d4009-41d3-4eb8-b427-d230947bb0f7", "db_session_id": "67FHRD26Q9U2FV35VIEM", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695481386981725, "job": 1, "event": "recovery_finished"} 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x561fdb0a6000 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: DB pointer 0x561fdb094000 2023-09-23T15:03:07.373 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: ** DB Stats ** 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: ** Compaction Stats [default] ** 2023-09-23T15:03:07.374 INFO:journalctl@ceph.mon.b.smithi027.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: L0 1/0 3.34 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 357.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: L6 1/0 6.29 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: Sum 2/0 9.63 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 357.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 357.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: ** Compaction Stats [default] ** 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 357.4 0.01 0.00 1 0.009 0 0 0.0 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T15:03:07.375 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Flush(GB): cumulative 0.003, interval 0.003 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Cumulative compaction: 0.00 GB write, 144.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Interval compaction: 0.00 GB write, 144.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Block cache BinnedLRUCache@0x561fdb00b1f0#7 capacity: 512.00 MB usage: 9.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,3.03 KB,0.000578165%) IndexBlock(1,6.58 KB,0.00125468%) Misc(1,0.00 KB,0%) 2023-09-23T15:03:07.376 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: starting mon.b rank 1 at public addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] at bind addrs [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???) e3 preinit fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).mds e9 new map 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).mds e9 print_map 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: e9 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: legacy client fscid: 1 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: Filesystem 'foo' (1) 2023-09-23T15:03:07.377 INFO:journalctl@ceph.mon.b.smithi027.stdout: fs_name foo 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: epoch 8 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: created 2023-09-23T14:56:31.343039+0000 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: modified 2023-09-23T14:56:42.940454+0000 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: tableserver 0 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: root 0 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: session_timeout 60 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: session_autoclose 300 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: max_file_size 1099511627776 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: required_client_features {} 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: last_failure 0 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: last_failure_osd_epoch 0 2023-09-23T15:03:07.378 INFO:journalctl@ceph.mon.b.smithi027.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: max_mds 1 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: in 0 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: up {0=14676} 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: failed 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: damaged 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: stopped 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: data_pools [4] 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: metadata_pool 3 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: inline_data disabled 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: balancer 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: bal_rank_mask -1 2023-09-23T15:03:07.379 INFO:journalctl@ceph.mon.b.smithi027.stdout: standby_count_wanted 1 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: [mds.foo.smithi017.ygjhom{0:14676} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.17:6834/2177953552,v1:172.21.15.17:6835/2177953552] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: Standby daemons: 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout: [mds.foo.smithi027.ynevql{-1:24544} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.27:6832/3657677765,v1:172.21.15.27:6833/3657677765] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expand map: {default=false} 2023-09-23T15:03:07.380 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta from 'false' to 'false' 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expanded map: {default=false} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expand map: {default=info} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta from 'info' to 'info' 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expanded map: {default=info} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expand map: {default=daemon} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expand map: {default=debug} 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T15:03:07.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:06 smithi027 ceph-mon[195233]: expand_channel_meta expanded map: {default=debug} 2023-09-23T15:03:07.401 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T15:03:07.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T15:03:08.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:08] "GET /metrics HTTP/1.1" 200 33756 "" "Prometheus/2.43.0" 2023-09-23T15:03:08.299 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:09.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:08 smithi027 conmon[179123]: ts=2023-09-23T15:03:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:03:12.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:11 smithi017 conmon[202315]: 2023-09-23T15:03:11.994+0000 7fdcb8de7700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T15:03:12.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: overall HEALTH_OK 2023-09-23T15:03:12.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.a calling monitor election 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.c calling monitor election 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.b calling monitor election 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2694906709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: overall HEALTH_OK 2023-09-23T15:03:12.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.a calling monitor election 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: monmap e4: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: overall HEALTH_OK 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: overall HEALTH_OK 2023-09-23T15:03:12.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.a calling monitor election 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.c calling monitor election 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.b calling monitor election 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2694906709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: overall HEALTH_OK 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.a calling monitor election 2023-09-23T15:03:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: monmap e4: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: overall HEALTH_OK 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: overall HEALTH_OK 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.a calling monitor election 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.c calling monitor election 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.b calling monitor election 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2694906709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: overall HEALTH_OK 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.a calling monitor election 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: monmap e4: 3 mons at {a=[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0],b=[v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0],c=[v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]} removed_ranks: {} 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: fsmap foo:1 {0=foo.smithi017.ygjhom=up:active} 1 up:standby 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: osdmap e91: 8 total, 8 up, 8 in 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: mgrmap e36: y(active, since 2m), standbys: x 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: overall HEALTH_OK 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (3m) - 7m 29.2M - 0.25.0 c8568f914cd2 6f0275940b82 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (3m) - 7m 91.5M - 9.4.7 2c41d148cca3 4653950d68ab 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (6m) - 6m 18.1M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (6m) - 6m 17.5M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443,9283,8765 running (3m) - 11m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 a039252b558e 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (3m) - 13m 503M - 18.2.0-460-gbb0daeff e32c2a64c126 a6d14007bbf4 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (2m) - 13m 52.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0012d0aac432 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 starting - - - 2048M 2023-09-23T15:03:12.958 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (85s) - 11m 38.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7f42350c36be 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (3m) - 7m 18.8M - 1.5.0 0da6a335fe13 af1ee2972cfe 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (3m) - 7m 18.1M - 1.5.0 0da6a335fe13 71fdd501b2ad 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (11m) - 10m 87.2M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 bb8e8a0efb36 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (10m) - 10m 100M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1542a5a16945 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (10m) - 10m 83.6M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 d6cc1ec2d677 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (9m) - 9m 75.1M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 1e310e53ac98 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (9m) - 9m 78.4M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (9m) - 9m 86.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (8m) - 8m 84.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (8m) - 8m 97.7M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T15:03:12.959 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (3m) - 7m 55.7M - 2.43.0 a07b618ecd1d bbda5037d082 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[215646]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[222878]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:13.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:13.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T15:03:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:13 smithi027 ceph-mon[195233]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:13.964 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:14 smithi017 ceph-mon[215646]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:14 smithi017 ceph-mon[222878]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:14 smithi027 ceph-mon[195233]: from='client.44120 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:15 smithi027 ceph-mon[195233]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:15.628 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:15.662 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:15 smithi017 ceph-mon[215646]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:15.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:15 smithi017 ceph-mon[222878]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:16.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T15:03:16.556 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:03:16.723 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.723 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.723 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:16.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:16.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:16 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.754 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:16.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:16 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:17.757 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:17 smithi027 ceph-mon[195233]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:17.757 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:17 smithi017 ceph-mon[215646]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:17.758 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:17 smithi017 ceph-mon[222878]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:18.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:18] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-23T15:03:19.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:18 smithi027 conmon[179123]: ts=2023-09-23T15:03:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:03:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:19.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:19.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:19.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:19.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:19.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:20.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:20 smithi017 ceph-mon[215646]: Reconfiguring mon.a (monmap changed)... 2023-09-23T15:03:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:20 smithi017 ceph-mon[215646]: Reconfiguring daemon mon.a on smithi017 2023-09-23T15:03:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:20 smithi017 ceph-mon[222878]: Reconfiguring mon.a (monmap changed)... 2023-09-23T15:03:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:20 smithi017 ceph-mon[222878]: Reconfiguring daemon mon.a on smithi017 2023-09-23T15:03:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:20 smithi027 ceph-mon[195233]: Reconfiguring mon.a (monmap changed)... 2023-09-23T15:03:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:20 smithi027 ceph-mon[195233]: Reconfiguring daemon mon.a on smithi017 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:21.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:22.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:22.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T15:03:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: Reconfiguring daemon mgr.y on smithi017 2023-09-23T15:03:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:23.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: Reconfiguring mon.c (monmap changed)... 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: Reconfiguring daemon mon.c on smithi017 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:03:23.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: Reconfiguring mon.c (monmap changed)... 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: Reconfiguring daemon mon.c on smithi017 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:03:23.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: Reconfiguring mon.c (monmap changed)... 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: Reconfiguring daemon mon.c on smithi017 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:03:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:24.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:24 smithi017 ceph-mon[222878]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T15:03:24.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:24 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.0 on smithi017 2023-09-23T15:03:24.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:24 smithi017 ceph-mon[215646]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T15:03:24.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:24 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.0 on smithi017 2023-09-23T15:03:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:24 smithi027 ceph-mon[195233]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T15:03:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:24 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.0 on smithi017 2023-09-23T15:03:25.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[215646]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:25.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:25.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[222878]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:03:25.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:25 smithi027 ceph-mon[195233]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:03:26.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.1 on smithi017 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:03:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.1 on smithi017 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:03:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.1 on smithi017 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:03:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:27.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[215646]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[215646]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.2 on smithi017 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[222878]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[222878]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T15:03:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.2 on smithi017 2023-09-23T15:03:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:27 smithi027 ceph-mon[195233]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:27 smithi027 ceph-mon[195233]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T15:03:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:27 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.2 on smithi017 2023-09-23T15:03:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:28.347 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:28] "GET /metrics HTTP/1.1" 200 33755 "" "Prometheus/2.43.0" 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[215646]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.3 on smithi017 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:28.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[222878]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T15:03:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:03:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.3 on smithi017 2023-09-23T15:03:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:29.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:28 smithi027 conmon[179123]: ts=2023-09-23T15:03:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:03:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:28 smithi027 ceph-mon[195233]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T15:03:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:03:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:28 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.3 on smithi017 2023-09-23T15:03:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:29.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[215646]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[222878]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:29.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:29 smithi027 ceph-mon[195233]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi017.ygjhom", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:31.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: Reconfiguring mds.foo.smithi017.ygjhom (monmap changed)... 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: Reconfiguring daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:31.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:31.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: Reconfiguring mds.foo.smithi017.ygjhom (monmap changed)... 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: Reconfiguring daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: Reconfiguring mds.foo.smithi017.ygjhom (monmap changed)... 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: Reconfiguring daemon mds.foo.smithi017.ygjhom on smithi017 2023-09-23T15:03:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T15:03:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T15:03:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: Reconfiguring mon.b (monmap changed)... 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: Reconfiguring daemon mon.b on smithi027 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: Reconfiguring mon.b (monmap changed)... 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: Reconfiguring daemon mon.b on smithi027 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:32.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: Reconfiguring mon.b (monmap changed)... 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: Reconfiguring daemon mon.b on smithi027 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T15:03:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:32 smithi027 ceph-mon[195233]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T15:03:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:32 smithi027 ceph-mon[195233]: Reconfiguring daemon mgr.x on smithi027 2023-09-23T15:03:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[215646]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[215646]: Reconfiguring daemon mgr.x on smithi027 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[222878]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[222878]: Reconfiguring daemon mgr.x on smithi027 2023-09-23T15:03:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T15:03:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T15:03:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:33 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.4 on smithi027 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.4 on smithi027 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T15:03:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T15:03:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:34.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:33 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.4 on smithi027 2023-09-23T15:03:35.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T15:03:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T15:03:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.5 on smithi027 2023-09-23T15:03:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:34 smithi027 ceph-mon[195233]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.5 on smithi027 2023-09-23T15:03:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[215646]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.5 on smithi027 2023-09-23T15:03:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:34 smithi017 ceph-mon[222878]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:36.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T15:03:36.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T15:03:36.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:36.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:36 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.6 on smithi027 2023-09-23T15:03:36.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.6 on smithi027 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T15:03:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T15:03:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:36 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.6 on smithi027 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[215646]: Reconfiguring daemon osd.7 on smithi027 2023-09-23T15:03:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:37 smithi017 ceph-mon[222878]: Reconfiguring daemon osd.7 on smithi027 2023-09-23T15:03:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:37 smithi027 ceph-mon[195233]: Reconfiguring daemon osd.7 on smithi027 2023-09-23T15:03:38.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:38] "GET /metrics HTTP/1.1" 200 33753 "" "Prometheus/2.43.0" 2023-09-23T15:03:38.849 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:38 smithi027 conmon[179123]: ts=2023-09-23T15:03:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: Reconfiguring mds.foo.smithi027.ynevql (monmap changed)... 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: Reconfiguring daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T15:03:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:38 smithi027 ceph-mon[195233]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: Reconfiguring mds.foo.smithi027.ynevql (monmap changed)... 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: Reconfiguring daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[215646]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: Reconfiguring mds.foo.smithi027.ynevql (monmap changed)... 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi027.ynevql", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: Reconfiguring daemon mds.foo.smithi027.ynevql on smithi027 2023-09-23T15:03:39.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:38 smithi017 ceph-mon[222878]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:40.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:40.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:40.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:40.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:03:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:03:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:03:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:03:41.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:40 smithi027 ceph-mon[195233]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:40 smithi017 ceph-mon[215646]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:40 smithi017 ceph-mon[222878]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:43 smithi017 ceph-mon[215646]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:43 smithi017 ceph-mon[222878]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:43 smithi027 ceph-mon[195233]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:45.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:45 smithi017 ceph-mon[215646]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:45.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:45 smithi017 ceph-mon[222878]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:45 smithi027 ceph-mon[195233]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:47 smithi017 ceph-mon[215646]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:47 smithi017 ceph-mon[222878]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:47 smithi027 ceph-mon[195233]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:48.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:48] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-23T15:03:49.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:48 smithi027 conmon[179123]: ts=2023-09-23T15:03:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:03:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:49 smithi017 ceph-mon[215646]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:49 smithi017 ceph-mon[222878]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:50.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:49 smithi027 ceph-mon[195233]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:51.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:51 smithi017 ceph-mon[215646]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:51 smithi017 ceph-mon[222878]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:51 smithi027 ceph-mon[195233]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:53 smithi017 ceph-mon[215646]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:53 smithi017 ceph-mon[222878]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:53 smithi027 ceph-mon[195233]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:55 smithi017 ceph-mon[215646]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:55 smithi017 ceph-mon[222878]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:55 smithi027 ceph-mon[195233]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:57 smithi017 ceph-mon[215646]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:57 smithi017 ceph-mon[222878]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:57 smithi027 ceph-mon[195233]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:03:58.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:03:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:03:58] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-23T15:03:58.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:58.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:03:59.100 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:03:58 smithi027 conmon[179123]: ts=2023-09-23T15:03:58.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:04:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:03:59 smithi027 ceph-mon[195233]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:03:59 smithi017 ceph-mon[215646]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:03:59 smithi017 ceph-mon[222878]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:01 smithi027 ceph-mon[195233]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:02.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:01 smithi017 ceph-mon[215646]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:01 smithi017 ceph-mon[222878]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:03 smithi027 ceph-mon[195233]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:03 smithi017 ceph-mon[215646]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:03 smithi017 ceph-mon[222878]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:06.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:05 smithi027 ceph-mon[195233]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:06.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:05 smithi017 ceph-mon[215646]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:06.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:05 smithi017 ceph-mon[222878]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:08.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:07 smithi017 ceph-mon[215646]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:08.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:07 smithi017 ceph-mon[222878]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:07 smithi027 ceph-mon[195233]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:08.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:08] "GET /metrics HTTP/1.1" 200 33757 "" "Prometheus/2.43.0" 2023-09-23T15:04:09.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:04:08 smithi027 conmon[179123]: ts=2023-09-23T15:04:08.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:04:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:09 smithi027 ceph-mon[195233]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:10.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:09 smithi017 ceph-mon[215646]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:09 smithi017 ceph-mon[222878]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:11 smithi027 ceph-mon[195233]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:12.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:11 smithi017 ceph-mon[215646]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:12.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:11 smithi017 ceph-mon[222878]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:13 smithi027 ceph-mon[195233]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:14.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:13 smithi017 ceph-mon[215646]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:14.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:13 smithi017 ceph-mon[222878]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:14.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:15 smithi027 ceph-mon[195233]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:16.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:15 smithi017 ceph-mon[215646]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:15 smithi017 ceph-mon[222878]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:17.981 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:17 smithi017 ceph-mon[215646]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:17.982 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:17 smithi017 ceph-mon[222878]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:17 smithi027 ceph-mon[195233]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:18.412 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:18] "GET /metrics HTTP/1.1" 200 33751 "" "Prometheus/2.43.0" 2023-09-23T15:04:18.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T15:04:18.796 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:04:19.099 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:04:18 smithi027 conmon[179123]: ts=2023-09-23T15:04:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi027.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.17\", device_class=\"ssd\", hostname=\"smithi017\", instance=\"smithi017:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.17\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T15:04:20.065 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:19 smithi017 ceph-mon[215646]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:20.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:19 smithi017 ceph-mon[222878]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:19 smithi027 ceph-mon[195233]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:20.508 INFO:teuthology.orchestra.run.smithi017.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:04:21.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T15:04:21.583 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:04:21.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[215646]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:21.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[215646]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:04:21.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[215646]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:04:21.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:21.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:21.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[222878]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:21.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[222878]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:04:21.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[222878]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:04:21.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:21.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:21.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:21 smithi027 ceph-mon[195233]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:21.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:21 smithi027 ceph-mon[195233]: from='client.34224 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:04:21.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:21 smithi027 ceph-mon[195233]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:04:21.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:21.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:23.136 INFO:teuthology.orchestra.run.smithi017.stdout:wait for half of osds to upgrade 2023-09-23T15:04:23.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[222878]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[215646]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:23.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:23 smithi027 ceph-mon[195233]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:24.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:24.504 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:25.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:25.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:26 smithi027 ceph-mon[195233]: Upgrade: Setting container_image for all mon 2023-09-23T15:04:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:26 smithi027 ceph-mon[195233]: Upgrade: Setting container_image for all crash 2023-09-23T15:04:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:26 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:26 smithi027 ceph-mon[195233]: Upgrade: osd.0 is safe to restart 2023-09-23T15:04:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1064062798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[215646]: Upgrade: Setting container_image for all mon 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[215646]: Upgrade: Setting container_image for all crash 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[215646]: Upgrade: osd.0 is safe to restart 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1064062798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[222878]: Upgrade: Setting container_image for all mon 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[222878]: Upgrade: Setting container_image for all crash 2023-09-23T15:04:26.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T15:04:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[222878]: Upgrade: osd.0 is safe to restart 2023-09-23T15:04:26.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1064062798' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[222878]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[215646]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:27.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:04:27.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:27 smithi027 ceph-mon[195233]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T15:04:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:28.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:28] "GET /metrics HTTP/1.1" 200 33751 "" "Prometheus/2.43.0" 2023-09-23T15:04:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[215646]: Upgrade: Updating osd.0 2023-09-23T15:04:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[215646]: Deploying daemon osd.0 on smithi017 2023-09-23T15:04:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:28.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[222878]: Upgrade: Updating osd.0 2023-09-23T15:04:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[222878]: Deploying daemon osd.0 on smithi017 2023-09-23T15:04:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:28 smithi027 ceph-mon[195233]: Upgrade: Updating osd.0 2023-09-23T15:04:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:28 smithi027 ceph-mon[195233]: Deploying daemon osd.0 on smithi017 2023-09-23T15:04:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:29.690 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:29 smithi017 systemd[1]: Stopping Ceph osd.0 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:04:29.691 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:29 smithi017 conmon[124373]: 2023-09-23T15:04:29.624+0000 7fa0d3e51700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:04:29.691 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:29 smithi017 conmon[124373]: 2023-09-23T15:04:29.624+0000 7fa0d3e51700 -1 osd.0 91 *** Got signal Terminated *** 2023-09-23T15:04:29.691 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:29 smithi017 conmon[124373]: 2023-09-23T15:04:29.624+0000 7fa0d3e51700 -1 osd.0 91 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:04:29.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[215646]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4136015774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[215646]: osd.0 marked itself down and dead 2023-09-23T15:04:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[222878]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4136015774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:29 smithi017 ceph-mon[222878]: osd.0 marked itself down and dead 2023-09-23T15:04:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:29 smithi027 ceph-mon[195233]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4136015774' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:29 smithi027 ceph-mon[195233]: osd.0 marked itself down and dead 2023-09-23T15:04:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:04:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[215646]: failed to encode map e92 with expected crc 2023-09-23T15:04:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[215646]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T15:04:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[215646]: failed to encode map e92 with expected crc 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[215646]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[222878]: failed to encode map e92 with expected crc 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[222878]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[222878]: failed to encode map e92 with expected crc 2023-09-23T15:04:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:30 smithi017 ceph-mon[222878]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T15:04:30.944 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:30 smithi017 bash[238649]: bb8e8a0efb36426ef13e589545cc862faa7a6dac8182130dd589db79a701289c 2023-09-23T15:04:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:30 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:04:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:30 smithi027 ceph-mon[195233]: failed to encode map e92 with expected crc 2023-09-23T15:04:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:30 smithi027 ceph-mon[195233]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T15:04:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:30 smithi027 ceph-mon[195233]: failed to encode map e92 with expected crc 2023-09-23T15:04:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:30 smithi027 ceph-mon[195233]: osdmap e93: 8 total, 7 up, 8 in 2023-09-23T15:04:31.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: pgmap v125: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4091914758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: pgmap v125: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: failed to encode map e93 with expected crc 2023-09-23T15:04:31.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4091914758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: pgmap v125: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: failed to encode map e93 with expected crc 2023-09-23T15:04:32.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4091914758' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:32.443 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:32 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T15:04:32.443 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:32 smithi017 systemd[1]: Stopped Ceph osd.0 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:04:32.443 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:32 smithi017 systemd[1]: Starting Ceph osd.0 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:04:33.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:04:33.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T15:04:33.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T15:04:33.694 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T15:04:33.694 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T15:04:33.694 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T15:04:33.694 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T15:04:33.694 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:33 smithi017 bash[239301]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T15:04:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:33 smithi027 ceph-mon[195233]: pgmap v127: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1347440808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:34.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:33 smithi017 ceph-mon[215646]: pgmap v127: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1347440808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:33 smithi017 ceph-mon[222878]: pgmap v127: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1347440808' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:34.692 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:34 smithi017 ceph-osd[239790]: -- 172.21.15.17:0/4061705914 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5575841364e0 con 0x557584109400 2023-09-23T15:04:34.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:34 smithi017 bash[239726]: 17a5480512e73459fc01874233b5e9c364916159607356e6c38cd77108c0bb4d 2023-09-23T15:04:34.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:34 smithi017 systemd[1]: Started Ceph osd.0 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:04:35.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:34 smithi017 conmon[239766]: 2023-09-23T15:04:34.907+0000 7f628fb0d640 -1 Falling back to public interface 2023-09-23T15:04:35.424 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:35 smithi027 ceph-mon[195233]: pgmap v128: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:35.424 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.425 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.425 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:35.425 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:35 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:04:35.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[215646]: pgmap v128: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[222878]: pgmap v128: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:35 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:04:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2608966993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2608966993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:36.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2608966993' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:37.943 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:37 smithi017 conmon[239766]: 2023-09-23T15:04:37.686+0000 7f628fb0d640 -1 osd.0 91 log_to_monitors true 2023-09-23T15:04:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[215646]: pgmap v129: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:37.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[222878]: pgmap v129: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:37.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:37 smithi027 ceph-mon[195233]: pgmap v129: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:38] "GET /metrics HTTP/1.1" 200 33758 "" "Prometheus/2.43.0" 2023-09-23T15:04:38.852 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:04:38 smithi017 conmon[239766]: 2023-09-23T15:04:38.703+0000 7f627ef76700 -1 osd.0 91 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:04:38.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[215646]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T15:04:38.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.853 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4173895398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:38.853 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[222878]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T15:04:38.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:38.854 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4173895398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:38 smithi027 ceph-mon[195233]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T15:04:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:38 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4173895398' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: pgmap v130: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: osdmap e94: 8 total, 7 up, 8 in 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:04:39.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[215646]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: pgmap v130: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T15:04:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: osdmap e94: 8 total, 7 up, 8 in 2023-09-23T15:04:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:04:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: failed to encode map e94 with expected crc 2023-09-23T15:04:39.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:39 smithi017 ceph-mon[222878]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: pgmap v130: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 85 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: osdmap e94: 8 total, 7 up, 8 in 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: from='osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:39 smithi027 ceph-mon[195233]: failed to encode map e94 with expected crc 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: failed to encode map e95 with expected crc 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: failed to encode map e95 with expected crc 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239] boot 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3354314664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[215646]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: failed to encode map e95 with expected crc 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: failed to encode map e95 with expected crc 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239] boot 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3354314664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:40 smithi017 ceph-mon[222878]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T15:04:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:04:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: failed to encode map e95 with expected crc 2023-09-23T15:04:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: failed to encode map e95 with expected crc 2023-09-23T15:04:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: osd.0 [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239] boot 2023-09-23T15:04:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: osdmap e95: 8 total, 8 up, 8 in 2023-09-23T15:04:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T15:04:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3354314664' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:40 smithi027 ceph-mon[195233]: osdmap e96: 8 total, 8 up, 8 in 2023-09-23T15:04:41.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:41 smithi017 conmon[202315]: 2023-09-23T15:04:41.513+0000 7fdcb8de7700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T15:04:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: pgmap v133: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:42.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: pgmap v133: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: pgmap v133: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: failed to encode map e96 with expected crc 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:04:42.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:04:42.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:42 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:42 smithi027 ceph-mon[195233]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T15:04:43.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:42 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:43.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:42 smithi017 ceph-mon[215646]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T15:04:43.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:42 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:43.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:42 smithi017 ceph-mon[222878]: Upgrade: unsafe to stop osd(s) at this time (7 PGs are or would become offline) 2023-09-23T15:04:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:43 smithi027 ceph-mon[195233]: pgmap v135: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3084545737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[215646]: pgmap v135: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3084545737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:44.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[222878]: pgmap v135: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T15:04:44.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3084545737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:44.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:44.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:44 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-09-23T15:04:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:44 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:04:45.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:44 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-09-23T15:04:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:44 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:04:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:44 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded) 2023-09-23T15:04:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:44 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:04:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:45 smithi027 ceph-mon[195233]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3653750864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:46.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:45 smithi017 ceph-mon[215646]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3653750864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:45 smithi017 ceph-mon[222878]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:46.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3653750864' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:47 smithi017 ceph-mon[215646]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3860113188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:47 smithi017 ceph-mon[222878]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3860113188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:47 smithi027 ceph-mon[195233]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3860113188' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:48.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:48] "GET /metrics HTTP/1.1" 200 33673 "" "Prometheus/2.43.0" 2023-09-23T15:04:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:49 smithi027 ceph-mon[195233]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:50.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:49 smithi017 ceph-mon[215646]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:50.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:49 smithi017 ceph-mon[222878]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3684655461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:51.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3684655461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:51.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3684655461' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:51 smithi027 ceph-mon[195233]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:51 smithi017 ceph-mon[215646]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:52.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:51 smithi017 ceph-mon[222878]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4193844391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:53.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4193844391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:53.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4193844391' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:53 smithi027 ceph-mon[195233]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:53 smithi017 ceph-mon[215646]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:53 smithi017 ceph-mon[222878]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:54 smithi027 ceph-mon[195233]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1385452763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:54 smithi017 ceph-mon[215646]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1385452763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:54 smithi017 ceph-mon[222878]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1385452763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:57.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[215646]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:57.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[215646]: Upgrade: osd.1 is safe to restart 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/324040343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[222878]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[222878]: Upgrade: osd.1 is safe to restart 2023-09-23T15:04:57.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/324040343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:57 smithi027 ceph-mon[195233]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:57 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T15:04:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:57 smithi027 ceph-mon[195233]: Upgrade: osd.1 is safe to restart 2023-09-23T15:04:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/324040343' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:04:58.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:04:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:04:58] "GET /metrics HTTP/1.1" 200 33673 "" "Prometheus/2.43.0" 2023-09-23T15:04:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: Upgrade: Updating osd.1 2023-09-23T15:04:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: Deploying daemon osd.1 on smithi017 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: Upgrade: Updating osd.1 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:04:59.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:59.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: Deploying daemon osd.1 on smithi017 2023-09-23T15:04:59.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:59.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:04:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: Upgrade: Updating osd.1 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: Deploying daemon osd.1 on smithi017 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:04:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:04:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:00.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1892946589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1892946589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1892946589' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:01.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:01 smithi017 ceph-mon[215646]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:01 smithi017 ceph-mon[222878]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:01.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:00 smithi017 systemd[1]: Stopping Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:05:01.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:01 smithi017 conmon[134049]: 2023-09-23T15:05:01.046+0000 7f72db0ef700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:05:01.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:01 smithi017 conmon[134049]: 2023-09-23T15:05:01.046+0000 7f72db0ef700 -1 osd.1 96 *** Got signal Terminated *** 2023-09-23T15:05:01.194 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:01 smithi017 conmon[134049]: 2023-09-23T15:05:01.046+0000 7f72db0ef700 -1 osd.1 96 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:05:01.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:01 smithi027 ceph-mon[195233]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:02 smithi027 ceph-mon[195233]: osd.1 marked itself down and dead 2023-09-23T15:05:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/633602984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:02 smithi017 ceph-mon[215646]: osd.1 marked itself down and dead 2023-09-23T15:05:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/633602984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:02 smithi017 ceph-mon[222878]: osd.1 marked itself down and dead 2023-09-23T15:05:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/633602984' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:02.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:02 smithi017 bash[246212]: 1542a5a16945d3b08390f82a857566ef1a542ab617021bc1e1a0ca8480d8811b 2023-09-23T15:05:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:03 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:03 smithi027 ceph-mon[195233]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:03 smithi027 ceph-mon[195233]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:03 smithi027 ceph-mon[195233]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T15:05:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:03 smithi027 ceph-mon[195233]: pgmap v146: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:03.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:03.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[215646]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[215646]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[215646]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[215646]: pgmap v146: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[222878]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[222878]: failed to encode map e97 with expected crc 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[222878]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T15:05:03.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:03 smithi017 ceph-mon[222878]: pgmap v146: 73 pgs: 9 stale+active+clean, 64 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2610904526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2610904526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: osdmap e98: 8 total, 7 up, 8 in 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: failed to encode map e98 with expected crc 2023-09-23T15:05:04.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2610904526' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:04.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:04 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T15:05:04.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:04 smithi017 systemd[1]: Stopped Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:05:04.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:04 smithi017 systemd[1]: Starting Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:05:05.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:05 smithi027 ceph-mon[195233]: pgmap v148: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:05 smithi017 ceph-mon[215646]: pgmap v148: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:05 smithi017 ceph-mon[222878]: pgmap v148: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:05.942 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T15:05:05.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:05 smithi017 bash[246908]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T15:05:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:06 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:06.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:06 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:06 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 6/84 objects degraded (7.143%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:07.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[215646]: pgmap v149: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3379818031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:07.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:06 smithi017 ceph-osd[247418]: -- 172.21.15.17:0/709121667 <== mon.0 v2:172.21.15.17:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562da2dfc4e0 con 0x562da2dcec00 2023-09-23T15:05:07.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:06 smithi017 bash[247353]: 1c41de2c6f27e7c172002e402cd3117eb8939903b224dac97ab1f7186360cbb8 2023-09-23T15:05:07.193 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:06 smithi017 systemd[1]: Started Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:05:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[222878]: pgmap v149: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3379818031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:07 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:07 smithi027 ceph-mon[195233]: pgmap v149: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3379818031' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:07 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:07 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:07.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:07 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:07.693 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:07 smithi017 conmon[247394]: 2023-09-23T15:05:07.433+0000 7f5b1c82d640 -1 Falling back to public interface 2023-09-23T15:05:08.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:08] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.43.0" 2023-09-23T15:05:09.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:09 smithi017 ceph-mon[215646]: pgmap v150: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1507258283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:09 smithi017 ceph-mon[222878]: pgmap v150: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1507258283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:09 smithi027 ceph-mon[195233]: pgmap v150: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1507258283' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:10.615 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:10 smithi017 conmon[247394]: 2023-09-23T15:05:10.203+0000 7f5b1c82d640 -1 osd.1 96 log_to_monitors true 2023-09-23T15:05:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:10.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.944 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:05:10 smithi017 conmon[247394]: 2023-09-23T15:05:10.652+0000 7f5b09c92700 -1 osd.1 96 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:05:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:10.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:10.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:10.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:05:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:11.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: pgmap v151: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:11.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: pgmap v151: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:11.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: failed to encode map e99 with expected crc 2023-09-23T15:05:11.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: failed to encode map e99 with expected crc 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2468004701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: failed to encode map e99 with expected crc 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: failed to encode map e99 with expected crc 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:05:11.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T15:05:11.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:11.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:11.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2468004701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: pgmap v151: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: failed to encode map e99 with expected crc 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: failed to encode map e99 with expected crc 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: osdmap e99: 8 total, 7 up, 8 in 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: from='osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2468004701' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[215646]: failed to encode map e100 with expected crc 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[215646]: osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547] boot 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[215646]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[222878]: failed to encode map e100 with expected crc 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[222878]: osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547] boot 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[222878]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T15:05:12.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:05:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:12 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:12 smithi027 ceph-mon[195233]: failed to encode map e100 with expected crc 2023-09-23T15:05:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:12 smithi027 ceph-mon[195233]: osd.1 [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547] boot 2023-09-23T15:05:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:12 smithi027 ceph-mon[195233]: osdmap e100: 8 total, 8 up, 8 in 2023-09-23T15:05:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: pgmap v154: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/551763602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: pgmap v154: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: failed to encode map e101 with expected crc 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:13.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/551763602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: pgmap v154: 73 pgs: 6 active+undersized+degraded, 26 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T15:05:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: osdmap e101: 8 total, 8 up, 8 in 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: failed to encode map e101 with expected crc 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/551763602' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:14.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:14 smithi017 conmon[202315]: 2023-09-23T15:05:14.143+0000 7fdcb8de7700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T15:05:15.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:14 smithi027 ceph-mon[195233]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T15:05:15.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[215646]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: pgmap v156: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:15.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:14 smithi017 ceph-mon[222878]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2023-09-23T15:05:16.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:15 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:16.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4001166766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:15 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4001166766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:15 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4001166766' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[215646]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[222878]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T15:05:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:17 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:05:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:17 smithi027 ceph-mon[195233]: pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:17 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T15:05:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:17 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:05:18.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:18] "GET /metrics HTTP/1.1" 200 33673 "" "Prometheus/2.43.0" 2023-09-23T15:05:18.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2929172084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2929172084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2929172084' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:19 smithi027 ceph-mon[195233]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:19 smithi017 ceph-mon[215646]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:19 smithi017 ceph-mon[222878]: pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:20 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4259106785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:21.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:20 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4259106785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:21.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:20 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4259106785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:21 smithi027 ceph-mon[195233]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:22.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:21 smithi017 ceph-mon[222878]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:21 smithi017 ceph-mon[215646]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1533771394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:23.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1533771394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:23.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1533771394' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:23 smithi027 ceph-mon[195233]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:24.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:23 smithi017 ceph-mon[215646]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:24.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:23 smithi017 ceph-mon[222878]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:25 smithi027 ceph-mon[195233]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3627363606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:26.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:25 smithi017 ceph-mon[215646]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3627363606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:25 smithi017 ceph-mon[222878]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3627363606' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:28.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:27 smithi017 ceph-mon[215646]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:28.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2811654703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:28.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:27 smithi017 ceph-mon[222878]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:28.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2811654703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:27 smithi027 ceph-mon[195233]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2811654703' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:28.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:28] "GET /metrics HTTP/1.1" 200 33673 "" "Prometheus/2.43.0" 2023-09-23T15:05:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[215646]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[222878]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:29 smithi027 ceph-mon[195233]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[215646]: Upgrade: osd.2 is safe to restart 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4231183760' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[222878]: Upgrade: osd.2 is safe to restart 2023-09-23T15:05:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4231183760' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:30 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T15:05:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:30 smithi027 ceph-mon[195233]: Upgrade: osd.2 is safe to restart 2023-09-23T15:05:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4231183760' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: Upgrade: Updating osd.2 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[215646]: Deploying daemon osd.2 on smithi017 2023-09-23T15:05:31.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: Upgrade: Updating osd.2 2023-09-23T15:05:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:05:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:31 smithi017 ceph-mon[222878]: Deploying daemon osd.2 on smithi017 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: Upgrade: Updating osd.2 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:31 smithi027 ceph-mon[195233]: Deploying daemon osd.2 on smithi017 2023-09-23T15:05:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3590765728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3590765728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3590765728' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:33.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:33 smithi017 systemd[1]: Stopping Ceph osd.2 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:05:33.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:33 smithi017 conmon[142966]: 2023-09-23T15:05:33.600+0000 7fda54d3d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:05:33.694 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:33 smithi017 conmon[142966]: 2023-09-23T15:05:33.600+0000 7fda54d3d700 -1 osd.2 101 *** Got signal Terminated *** 2023-09-23T15:05:33.694 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:33 smithi017 conmon[142966]: 2023-09-23T15:05:33.600+0000 7fda54d3d700 -1 osd.2 101 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:05:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:33 smithi027 ceph-mon[195233]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:34.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:33 smithi027 ceph-mon[195233]: osd.2 marked itself down and dead 2023-09-23T15:05:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:33 smithi017 ceph-mon[215646]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:33 smithi017 ceph-mon[215646]: osd.2 marked itself down and dead 2023-09-23T15:05:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:33 smithi017 ceph-mon[222878]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:33 smithi017 ceph-mon[222878]: osd.2 marked itself down and dead 2023-09-23T15:05:34.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:34 smithi017 bash[254485]: d6cc1ec2d6779acf75618139732ec47de9488646c7601e164bfcda2070c15a66 2023-09-23T15:05:35.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:35.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2325470680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:35.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2325470680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:35.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: failed to encode map e102 with expected crc 2023-09-23T15:05:35.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2325470680' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: pgmap v167: 73 pgs: 14 peering, 1 stale+active+clean, 58 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[222878]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: pgmap v167: 73 pgs: 14 peering, 1 stale+active+clean, 58 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:35.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:35 smithi017 ceph-mon[215646]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: pgmap v167: 73 pgs: 14 peering, 1 stale+active+clean, 58 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: osdmap e103: 8 total, 7 up, 8 in 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:35 smithi027 ceph-mon[195233]: failed to encode map e103 with expected crc 2023-09-23T15:05:36.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:36 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T15:05:36.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:36 smithi017 systemd[1]: Stopped Ceph osd.2 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:05:36.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:36 smithi017 systemd[1]: Starting Ceph osd.2 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:05:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/660169792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:37.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/660169792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:37.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/660169792' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:37.692 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T15:05:37.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:37 smithi017 bash[255165]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T15:05:38.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:37 smithi017 ceph-mon[215646]: pgmap v169: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:38.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:37 smithi017 ceph-mon[222878]: pgmap v169: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:37 smithi027 ceph-mon[195233]: pgmap v169: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:38.334 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:38] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.43.0" 2023-09-23T15:05:38.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:38 smithi017 ceph-osd[255655]: -- 172.21.15.17:0/1805536858 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5631023aa4e0 con 0x56310237d400 2023-09-23T15:05:38.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:38 smithi017 bash[255590]: 49117382ccb9167dcbc15867e49e748d7987ce5bf36b9a6c48611971edd93dab 2023-09-23T15:05:38.693 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:38 smithi017 systemd[1]: Started Ceph osd.2 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:05:39.192 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:38 smithi017 conmon[255631]: 2023-09-23T15:05:38.927+0000 7f316ed89640 -1 Falling back to public interface 2023-09-23T15:05:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:39 smithi027 ceph-mon[195233]: pgmap v170: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2152730266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[215646]: pgmap v170: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2152730266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[222878]: pgmap v170: 73 pgs: 14 peering, 5 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2152730266' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:40.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:40 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:40.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:40 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:05:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:40 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:40 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:05:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:40 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:05:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:40 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:05:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:41 smithi027 ceph-mon[195233]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2486471756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:41.943 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:41 smithi017 conmon[255631]: 2023-09-23T15:05:41.722+0000 7f316ed89640 -1 osd.2 101 log_to_monitors true 2023-09-23T15:05:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[215646]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2486471756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[222878]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:41.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2486471756' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:42.942 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:05:42 smithi017 conmon[255631]: 2023-09-23T15:05:42.717+0000 7f315f1f4700 -1 osd.2 101 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T15:05:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: pgmap v172: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: failed to encode map e104 with expected crc 2023-09-23T15:05:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: failed to encode map e104 with expected crc 2023-09-23T15:05:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T15:05:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: osdmap e104: 8 total, 7 up, 8 in 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4127288938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: pgmap v172: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: failed to encode map e104 with expected crc 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: failed to encode map e104 with expected crc 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T15:05:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: osdmap e104: 8 total, 7 up, 8 in 2023-09-23T15:05:43.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:43.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:43.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:43.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:43.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4127288938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: pgmap v172: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: failed to encode map e104 with expected crc 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: failed to encode map e104 with expected crc 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: osdmap e104: 8 total, 7 up, 8 in 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4127288938' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298] boot 2023-09-23T15:05:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T15:05:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:05:45.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:45.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298] boot 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: failed to encode map e105 with expected crc 2023-09-23T15:05:45.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: osd.2 [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298] boot 2023-09-23T15:05:45.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: osdmap e105: 8 total, 8 up, 8 in 2023-09-23T15:05:45.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T15:05:45.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:45 smithi017 conmon[202315]: 2023-09-23T15:05:45.433+0000 7fdcb8de7700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 10 peering, 8 active+undersized, 54 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:46.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:46.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 10 peering, 8 active+undersized, 54 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: pgmap v175: 73 pgs: 1 active+undersized+degraded, 10 peering, 8 active+undersized, 54 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: osdmap e106: 8 total, 8 up, 8 in 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: failed to encode map e106 with expected crc 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:05:46.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:05:46.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:46 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:46 smithi027 ceph-mon[195233]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T15:05:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/945248331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:46 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[215646]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/945248331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[222878]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T15:05:47.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/945248331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:47.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:46 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:05:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:47 smithi017 ceph-mon[215646]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 10 peering, 4 active+undersized, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:48.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:47 smithi017 ceph-mon[222878]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 10 peering, 4 active+undersized, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:47 smithi027 ceph-mon[195233]: pgmap v177: 73 pgs: 1 active+undersized+degraded, 10 peering, 4 active+undersized, 58 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:48.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:48] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-23T15:05:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3975446718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3975446718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:49.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3975446718' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:49 smithi027 ceph-mon[195233]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 10 peering, 1 active+undersized, 61 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:50.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:49 smithi017 ceph-mon[215646]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 10 peering, 1 active+undersized, 61 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:49 smithi017 ceph-mon[222878]: pgmap v178: 73 pgs: 1 active+undersized+degraded, 10 peering, 1 active+undersized, 61 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T15:05:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1214497392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:51.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1214497392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:51.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1214497392' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:51 smithi027 ceph-mon[195233]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:51 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T15:05:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:51 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:05:52.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[215646]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T15:05:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:05:52.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[222878]: pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:52.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2023-09-23T15:05:52.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:51 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:05:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:53 smithi027 ceph-mon[195233]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/144453024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:53 smithi017 ceph-mon[215646]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/144453024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:53 smithi017 ceph-mon[222878]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:54.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/144453024' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:55 smithi027 ceph-mon[195233]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3917498785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:56.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:55 smithi017 ceph-mon[215646]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:56.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3917498785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:56.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:55 smithi017 ceph-mon[222878]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:56.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3917498785' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:58.020 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:57 smithi017 ceph-mon[215646]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1125628168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:58.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:57 smithi017 ceph-mon[222878]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:58.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1125628168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:57 smithi027 ceph-mon[195233]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1125628168' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:05:58.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:05:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:05:58] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[215646]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[222878]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:05:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:59 smithi027 ceph-mon[195233]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:05:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:05:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:05:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:05:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2455347624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:00.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2455347624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2455347624' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:01.225 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[215646]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:01.225 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[215646]: Upgrade: osd.3 is safe to restart 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[222878]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:01 smithi017 ceph-mon[222878]: Upgrade: osd.3 is safe to restart 2023-09-23T15:06:01.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:01 smithi027 ceph-mon[195233]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:01.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:01 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:01 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T15:06:01.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:01 smithi027 ceph-mon[195233]: Upgrade: osd.3 is safe to restart 2023-09-23T15:06:03.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: Upgrade: Updating osd.3 2023-09-23T15:06:03.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:03.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:06:03.148 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: Deploying daemon osd.3 on smithi017 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/738774600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: Upgrade: Updating osd.3 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: Deploying daemon osd.3 on smithi017 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:03.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/738774600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: Upgrade: Updating osd.3 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: Deploying daemon osd.3 on smithi017 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/738774600' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:05.160 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:04 smithi017 systemd[1]: Stopping Ceph osd.3 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:06:05.160 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:04 smithi017 conmon[152758]: 2023-09-23T15:06:04.948+0000 7ff307197700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:06:05.160 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:04 smithi017 conmon[152758]: 2023-09-23T15:06:04.948+0000 7ff307197700 -1 osd.3 106 *** Got signal Terminated *** 2023-09-23T15:06:05.160 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:04 smithi017 conmon[152758]: 2023-09-23T15:06:04.948+0000 7ff307197700 -1 osd.3 106 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:06:05.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[215646]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/380649033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[215646]: osd.3 marked itself down and dead 2023-09-23T15:06:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[222878]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/380649033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:05 smithi017 ceph-mon[222878]: osd.3 marked itself down and dead 2023-09-23T15:06:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:05 smithi027 ceph-mon[195233]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/380649033' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:05 smithi027 ceph-mon[195233]: osd.3 marked itself down and dead 2023-09-23T15:06:06.160 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:05 smithi017 bash[262092]: 1e310e53ac9813fde866eb9eec3e75ea990d794f3605d1204bd60546460ad9a7 2023-09-23T15:06:06.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[215646]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:06 smithi017 ceph-mon[222878]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:06 smithi027 ceph-mon[195233]: failed to encode map e107 with expected crc 2023-09-23T15:06:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: pgmap v188: 73 pgs: 6 peering, 10 stale+active+clean, 57 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1823372388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: pgmap v188: 73 pgs: 6 peering, 10 stale+active+clean, 57 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1823372388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: pgmap v188: 73 pgs: 6 peering, 10 stale+active+clean, 57 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: osdmap e108: 8 total, 7 up, 8 in 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: failed to encode map e108 with expected crc 2023-09-23T15:06:07.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1823372388' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:08.021 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:07 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T15:06:08.021 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:07 smithi017 systemd[1]: Stopped Ceph osd.3 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:06:08.021 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:07 smithi017 systemd[1]: Starting Ceph osd.3 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:06:08.021 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:08] "GET /metrics HTTP/1.1" 200 33877 "" "Prometheus/2.43.0" 2023-09-23T15:06:09.171 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:06:09.171 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T15:06:09.171 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T15:06:09.172 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T15:06:09.172 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T15:06:09.172 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T15:06:09.172 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T15:06:09.172 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:08 smithi017 bash[262751]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T15:06:09.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:09 smithi017 ceph-mon[215646]: pgmap v190: 73 pgs: 6 peering, 9 stale+active+clean, 3 active+undersized, 55 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:09 smithi017 ceph-mon[222878]: pgmap v190: 73 pgs: 6 peering, 9 stale+active+clean, 3 active+undersized, 55 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:09 smithi027 ceph-mon[195233]: pgmap v190: 73 pgs: 6 peering, 9 stale+active+clean, 3 active+undersized, 55 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3748985407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3748985407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:10.444 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:09 smithi017 bash[263193]: 3f1a3bdcd680e96e885d29acb5e2f243b764d37811cc873a004ae5d71046fe0e 2023-09-23T15:06:10.444 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:09 smithi017 systemd[1]: Started Ceph osd.3 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:06:10.444 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:09 smithi017 ceph-osd[263258]: -- 172.21.15.17:0/825904523 <== mon.2 v2:172.21.15.17:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559ee55604e0 con 0x559ee5533000 2023-09-23T15:06:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:10 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3748985407' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:10.943 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:10 smithi017 conmon[263234]: 2023-09-23T15:06:10.502+0000 7fe1eb605640 -1 Falling back to public interface 2023-09-23T15:06:11.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:11 smithi017 ceph-mon[215646]: pgmap v191: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:11.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:11 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:06:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:11 smithi017 ceph-mon[222878]: pgmap v191: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:11 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:06:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:11 smithi027 ceph-mon[195233]: pgmap v191: 73 pgs: 3 active+undersized+degraded, 6 peering, 22 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:11.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:11 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T15:06:12.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/41710136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:12.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/41710136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/41710136' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:13.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:13.632 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:13.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:13.633 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:13 smithi017 conmon[263234]: 2023-09-23T15:06:13.295+0000 7fe1eb605640 -1 osd.3 106 log_to_monitors true 2023-09-23T15:06:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: pgmap v192: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:14.193 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:06:14 smithi017 conmon[263234]: 2023-09-23T15:06:14.125+0000 7fe1dca72700 -1 osd.3 106 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:06:14.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4248393505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T15:06:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T15:06:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:14.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[215646]: failed to encode map e109 with expected crc 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4248393505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:14.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:14 smithi017 ceph-mon[222878]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4248393505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: osdmap e109: 8 total, 7 up, 8 in 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: from='osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:06:15.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:14 smithi027 ceph-mon[195233]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743] boot 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: failed to encode map e109 with expected crc 2023-09-23T15:06:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:06:15.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743] boot 2023-09-23T15:06:15.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T15:06:15.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:15 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:06:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: failed to encode map e109 with expected crc 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: pgmap v194: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: failed to encode map e109 with expected crc 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: failed to encode map e109 with expected crc 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: osd.3 [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743] boot 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T15:06:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:15 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T15:06:16.299 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/976372714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: failed to encode map e110 with expected crc 2023-09-23T15:06:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T15:06:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:16.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/976372714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: failed to encode map e110 with expected crc 2023-09-23T15:06:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: failed to encode map e110 with expected crc 2023-09-23T15:06:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: failed to encode map e110 with expected crc 2023-09-23T15:06:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T15:06:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/976372714' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T15:06:17.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T15:06:17.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: pgmap v196: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:17.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.638 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:17 smithi017 conmon[202315]: 2023-09-23T15:06:17.466+0000 7fdcb8de7700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T15:06:17.747 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:17.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: pgmap v196: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:17.941 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: failed to encode map e111 with expected crc 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:17.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: pgmap v196: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: failed to encode map e111 with expected crc 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:18.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:18.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:18.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:18] "GET /metrics HTTP/1.1" 200 33880 "" "Prometheus/2.43.0" 2023-09-23T15:06:18.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[215646]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:18.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[215646]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T15:06:18.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[222878]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:18.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[222878]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T15:06:18.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:18 smithi027 ceph-mon[195233]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T15:06:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:18 smithi027 ceph-mon[195233]: Upgrade: unsafe to stop osd(s) at this time (10 PGs are or would become offline) 2023-09-23T15:06:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:19.522 INFO:teuthology.orchestra.run.smithi017.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:06:19.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:19 smithi017 ceph-mon[215646]: pgmap v198: 73 pgs: 3 peering, 3 active+undersized+degraded, 23 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:19.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:19 smithi017 ceph-mon[222878]: pgmap v198: 73 pgs: 3 peering, 3 active+undersized+degraded, 23 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:19.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:19 smithi027 ceph-mon[195233]: pgmap v198: 73 pgs: 3 peering, 3 active+undersized+degraded, 23 active+undersized, 44 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T15:06:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:20.232 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T15:06:20.598 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:20 smithi017 ceph-mon[215646]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:20.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:20 smithi017 ceph-mon[215646]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:06:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:20 smithi017 ceph-mon[222878]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:20.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:20 smithi017 ceph-mon[222878]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:06:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:20 smithi027 ceph-mon[195233]: from='client.44450 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:20 smithi027 ceph-mon[195233]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[215646]: pgmap v199: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[222878]: pgmap v199: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-09-23T15:06:21.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:21 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:06:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:21 smithi027 ceph-mon[195233]: pgmap v199: 73 pgs: 3 peering, 70 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:21 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2023-09-23T15:06:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:21 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) - 10m 29.2M - 0.25.0 c8568f914cd2 6f0275940b82 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (6m) - 10m 90.8M - 9.4.7 2c41d148cca3 4653950d68ab 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (9m) - 9m 19.0M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (9m) - 9m 18.3M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443,9283,8765 running (6m) - 14m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 a039252b558e 2023-09-23T15:06:22.246 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) - 16m 513M - 18.2.0-460-gbb0daeff e32c2a64c126 a6d14007bbf4 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (6m) - 16m 84.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0012d0aac432 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 running (3m) - 14m 52.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0f598c97295e 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4m) - 14m 63.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7f42350c36be 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (6m) - 10m 19.3M - 1.5.0 0da6a335fe13 af1ee2972cfe 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (6m) - 10m 18.8M - 1.5.0 0da6a335fe13 71fdd501b2ad 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (108s) - 14m 59.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 17a5480512e7 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (75s) - 13m 59.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 1c41de2c6f27 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (44s) - 13m 49.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 49117382ccb9 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (12s) - 12m 12.7M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 3f1a3bdcd680 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (12m) - 12m 94.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T15:06:22.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (12m) - 12m 98.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T15:06:22.248 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (11m) - 11m 96.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T15:06:22.248 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (11m) - 11m 111M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T15:06:22.248 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (6m) - 11m 62.3M - 2.43.0 a07b618ecd1d bbda5037d082 2023-09-23T15:06:22.924 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T15:06:23.282 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:23.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:23 smithi017 ceph-mon[215646]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:23.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:23 smithi017 ceph-mon[215646]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:23.839 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:23 smithi017 ceph-mon[222878]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:23.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:23 smithi017 ceph-mon[222878]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:23 smithi027 ceph-mon[195233]: pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:23 smithi027 ceph-mon[195233]: from='client.34548 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:24.988 INFO:teuthology.orchestra.run.smithi017.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:alertmanager.a smithi017 *:9093,9094 running (7m) - 10m 29.2M - 0.25.0 c8568f914cd2 6f0275940b82 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:grafana.a smithi027 *:3000 running (6m) - 10m 90.8M - 9.4.7 2c41d148cca3 4653950d68ab 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi017.ygjhom smithi017 running (9m) - 9m 19.0M - 17.2.6-1123-g61431f6b 0db14aa01094 087379bd75f6 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mds.foo.smithi027.ynevql smithi027 running (9m) - 9m 18.3M - 17.2.6-1123-g61431f6b 0db14aa01094 70bed72cca66 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.x smithi027 *:8443,9283,8765 running (6m) - 14m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 a039252b558e 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mgr.y smithi017 *:8443,9283,8765 running (6m) - 16m 513M - 18.2.0-460-gbb0daeff e32c2a64c126 a6d14007bbf4 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a smithi017 running (6m) - 16m 84.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0012d0aac432 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b smithi027 running (3m) - 14m 52.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 0f598c97295e 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c smithi017 running (4m) - 14m 63.6M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 7f42350c36be 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.a smithi017 *:9100 running (7m) - 10m 19.3M - 1.5.0 0da6a335fe13 af1ee2972cfe 2023-09-23T15:06:24.989 INFO:teuthology.orchestra.run.smithi017.stdout:node-exporter.b smithi027 *:9100 running (6m) - 10m 18.8M - 1.5.0 0da6a335fe13 71fdd501b2ad 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 smithi017 running (110s) - 14m 59.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 17a5480512e7 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 smithi017 running (78s) - 13m 59.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 1c41de2c6f27 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 smithi017 running (46s) - 13m 49.1M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 49117382ccb9 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 smithi017 running (15s) - 13m 12.7M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 3f1a3bdcd680 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 smithi027 running (12m) - 12m 94.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 f59f399d4c7d 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 smithi027 running (12m) - 12m 98.3M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 6daba84c8455 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 smithi027 running (11m) - 11m 96.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 37890724419e 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 smithi027 running (11m) - 11m 111M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 8566abdd88d9 2023-09-23T15:06:24.990 INFO:teuthology.orchestra.run.smithi017.stdout:prometheus.a smithi027 *:9095 running (6m) - 11m 62.3M - 2.43.0 a07b618ecd1d bbda5037d082 2023-09-23T15:06:25.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T15:06:25.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:25 smithi017 ceph-mon[215646]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:25 smithi017 ceph-mon[222878]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:26.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:25 smithi027 ceph-mon[195233]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:26.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:26 smithi017 ceph-mon[215646]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:26.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:26 smithi017 ceph-mon[222878]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:26 smithi027 ceph-mon[195233]: from='client.34554 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:06:27.636 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:06:27.636 INFO:teuthology.orchestra.run.smithi017.stdout: "mon": { 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr": { 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "osd": { 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "mds": { 2023-09-23T15:06:27.637 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout: "overall": { 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:06:27.638 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:06:27.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:27 smithi017 ceph-mon[215646]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/492424606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:27 smithi017 ceph-mon[222878]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/492424606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:27 smithi027 ceph-mon[195233]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/492424606' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T15:06:28.311 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:28] "GET /metrics HTTP/1.1" 200 33880 "" "Prometheus/2.43.0" 2023-09-23T15:06:28.329 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:28.329 INFO:teuthology.run_tasks:Running task print... 2023-09-23T15:06:28.338 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T15:06:28.338 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T15:06:28.346 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T15:06:28.346 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T15:06:28.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T15:06:28.702 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[215646]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[222878]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:29 smithi027 ceph-mon[195233]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:30.416 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:06:31.041 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":36,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6800","nonce":2427611895},{"type":"v1","addr":"172.21.15.17:6801","nonce":2427611895}]},"active_addr":"172.21.15.17:6801/2427611895","active_change":"2023-09-23T15:00:28.003755+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25771,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":91,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3054333046}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":2072763487}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":3218362168}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.17:0","nonce":4049812391}]}]} 2023-09-23T15:06:31.046 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T15:06:31.046 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T15:06:31.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T15:06:31.411 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:31 smithi017 ceph-mon[215646]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2986696931' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T15:06:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:31 smithi017 ceph-mon[222878]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2986696931' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T15:06:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:31 smithi027 ceph-mon[195233]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2986696931' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:32.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:32.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:32.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:32 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:33.079 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:06:33.080 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":111,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T15:06:16.113424+0000","last_up_change":"2023-09-23T15:06:15.104616+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":110,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6803","nonce":1593815239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6805","nonce":1593815239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6809","nonce":1593815239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6807","nonce":1593815239}]},"public_addr":"172.21.15.17:6803/1593815239","cluster_addr":"172.21.15.17:6805/1593815239","heartbeat_back_addr":"172.21.15.17:6809/1593815239","heartbeat_front_addr":"172.21.15.17:6807/1593815239","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":100,"up_thru":110,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6811","nonce":2834890547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6813","nonce":2834890547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6817","nonce":2834890547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6815","nonce":2834890547}]},"public_addr":"172.21.15.17:6811/2834890547","cluster_addr":"172.21.15.17:6813/2834890547","heartbeat_back_addr":"172.21.15.17:6817/2834890547","heartbeat_front_addr":"172.21.15.17:6815/2834890547","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":110,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6819","nonce":1172679298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6821","nonce":1172679298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6825","nonce":1172679298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6823","nonce":1172679298}]},"public_addr":"172.21.15.17:6819/1172679298","cluster_addr":"172.21.15.17:6821/1172679298","heartbeat_back_addr":"172.21.15.17:6825/1172679298","heartbeat_front_addr":"172.21.15.17:6823/1172679298","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":110,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6827","nonce":3586551743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6829","nonce":3586551743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6833","nonce":3586551743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6831","nonce":3586551743}]},"public_addr":"172.21.15.17:6827/3586551743","cluster_addr":"172.21.15.17:6829/3586551743","heartbeat_back_addr":"172.21.15.17:6833/3586551743","heartbeat_front_addr":"172.21.15.17:6831/3586551743","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/79453409":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:6800/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2670521468":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2135416630":"2023-09-24T14:59:04.520407+0000","172.21.15.27:6834/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/707576150":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/244258131":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/1374246236":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.27:0/816410812":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.27:6835/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1467411231":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/201191039":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/1851970177":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/1666581637":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/774095134":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3598198038":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6800/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/2369426862":"2023-09-24T15:00:28.003544+0000","172.21.15.27:0/986573237":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/623737292":"2023-09-24T15:00:28.003544+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T15:06:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:06:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:32 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:06:33.749 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T15:06:33.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T15:06:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:33 smithi027 ceph-mon[195233]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1366485692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:34.112 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:33 smithi017 ceph-mon[215646]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1366485692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:33 smithi017 ceph-mon[222878]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1366485692' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:35.781 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:06:35.781 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":111,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T15:06:16.113424+0000","last_up_change":"2023-09-23T15:06:15.104616+0000","last_in_change":"2023-09-23T14:54:38.616944+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":110,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6803","nonce":1593815239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6805","nonce":1593815239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6809","nonce":1593815239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6807","nonce":1593815239}]},"public_addr":"172.21.15.17:6803/1593815239","cluster_addr":"172.21.15.17:6805/1593815239","heartbeat_back_addr":"172.21.15.17:6809/1593815239","heartbeat_front_addr":"172.21.15.17:6807/1593815239","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":96,"up_from":100,"up_thru":110,"down_at":97,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6811","nonce":2834890547}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6813","nonce":2834890547}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6817","nonce":2834890547}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":2834890547},{"type":"v1","addr":"172.21.15.17:6815","nonce":2834890547}]},"public_addr":"172.21.15.17:6811/2834890547","cluster_addr":"172.21.15.17:6813/2834890547","heartbeat_back_addr":"172.21.15.17:6817/2834890547","heartbeat_front_addr":"172.21.15.17:6815/2834890547","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":110,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6819","nonce":1172679298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6821","nonce":1172679298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6825","nonce":1172679298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6823","nonce":1172679298}]},"public_addr":"172.21.15.17:6819/1172679298","cluster_addr":"172.21.15.17:6821/1172679298","heartbeat_back_addr":"172.21.15.17:6825/1172679298","heartbeat_front_addr":"172.21.15.17:6823/1172679298","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":110,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6827","nonce":3586551743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6829","nonce":3586551743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6833","nonce":3586551743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6831","nonce":3586551743}]},"public_addr":"172.21.15.17:6827/3586551743","cluster_addr":"172.21.15.17:6829/3586551743","heartbeat_back_addr":"172.21.15.17:6833/3586551743","heartbeat_front_addr":"172.21.15.17:6831/3586551743","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":107,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6817","nonce":1429782010}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6819","nonce":1429782010}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6823","nonce":1429782010}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":1429782010},{"type":"v1","addr":"172.21.15.27:6821","nonce":1429782010}]},"public_addr":"172.21.15.27:6817/1429782010","cluster_addr":"172.21.15.27:6819/1429782010","heartbeat_back_addr":"172.21.15.27:6823/1429782010","heartbeat_front_addr":"172.21.15.27:6821/1429782010","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":110,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6825","nonce":2619255591}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6827","nonce":2619255591}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6831","nonce":2619255591}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":2619255591},{"type":"v1","addr":"172.21.15.27:6829","nonce":2619255591}]},"public_addr":"172.21.15.27:6825/2619255591","cluster_addr":"172.21.15.27:6827/2619255591","heartbeat_back_addr":"172.21.15.27:6831/2619255591","heartbeat_front_addr":"172.21.15.27:6829/2619255591","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":96},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]},{"pgid":"4.1d","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/79453409":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:6800/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2670521468":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2135416630":"2023-09-24T14:59:04.520407+0000","172.21.15.27:6834/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/707576150":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/244258131":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/1374246236":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.27:0/816410812":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.27:6835/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1467411231":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/201191039":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/1851970177":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/1666581637":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/774095134":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3598198038":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6800/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/2369426862":"2023-09-24T15:00:28.003544+0000","172.21.15.27:0/986573237":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/623737292":"2023-09-24T15:00:28.003544+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T15:06:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:35 smithi027 ceph-mon[195233]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:36.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:35 smithi017 ceph-mon[215646]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:35 smithi017 ceph-mon[222878]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:36.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T15:06:36.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T15:06:36.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T15:06:36.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T15:06:36.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T15:06:36.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T15:06:36.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T15:06:36.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T15:06:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2166665408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:37.188 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:37.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2166665408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:37.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2166665408' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:06:38.003 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.003 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:37 smithi017 ceph-mon[222878]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:38.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:37 smithi017 ceph-mon[215646]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:37 smithi027 ceph-mon[195233]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:38.234 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.275 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.277 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:38] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.43.0" 2023-09-23T15:06:38.300 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.368 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:38.454 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:40.039 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:39 smithi017 ceph-mon[215646]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:40.039 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:39 smithi017 ceph-mon[222878]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:39 smithi027 ceph-mon[195233]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:41 smithi027 ceph-mon[195233]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:41 smithi017 ceph-mon[215646]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:42.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:41 smithi017 ceph-mon[222878]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:42.933 INFO:teuthology.orchestra.run.smithi017.stdout:133143986332 2023-09-23T15:06:42.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T15:06:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:43 smithi027 ceph-mon[195233]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:44.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:43 smithi017 ceph-mon[215646]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:44.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:44.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:43 smithi017 ceph-mon[222878]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:44.172 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:44.981 INFO:teuthology.orchestra.run.smithi017.stdout:236223201423 2023-09-23T15:06:44.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T15:06:45.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:45 smithi017 ceph-mon[222878]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:45.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:45 smithi017 ceph-mon[215646]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:46.017 INFO:teuthology.orchestra.run.smithi017.stdout:408021893146 2023-09-23T15:06:46.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T15:06:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:45 smithi027 ceph-mon[195233]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:47.485 INFO:teuthology.orchestra.run.smithi017.stdout:429496729621 2023-09-23T15:06:47.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T15:06:47.555 INFO:teuthology.orchestra.run.smithi017.stdout:450971566094 2023-09-23T15:06:47.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T15:06:48.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:47 smithi017 ceph-mon[215646]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:47 smithi017 ceph-mon[222878]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:47 smithi027 ceph-mon[195233]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:48.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:48] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2023-09-23T15:06:48.850 INFO:teuthology.orchestra.run.smithi017.stdout:201863463060 2023-09-23T15:06:48.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T15:06:48.878 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:48.886 INFO:teuthology.orchestra.run.smithi017.stdout:472446402568 2023-09-23T15:06:48.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T15:06:48.925 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:49.760 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:49.791 INFO:teuthology.orchestra.run.smithi017.stdout:167503724696 2023-09-23T15:06:49.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T15:06:49.872 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:49.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:49 smithi017 ceph-mon[215646]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:49.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:49 smithi017 ceph-mon[222878]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:49 smithi027 ceph-mon[195233]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:50.848 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:51 smithi027 ceph-mon[195233]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:52.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:51 smithi017 ceph-mon[215646]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:52.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:51 smithi017 ceph-mon[222878]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:52.632 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:52.658 INFO:teuthology.orchestra.run.smithi017.stdout:133143986334 2023-09-23T15:06:53.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1934995279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:06:53.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1934995279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:06:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1934995279' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:06:53.396 INFO:teuthology.orchestra.run.smithi017.stdout:408021893148 2023-09-23T15:06:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:53 smithi027 ceph-mon[195233]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/894453254' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:06:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:53 smithi017 ceph-mon[215646]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:54.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/894453254' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:06:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:53 smithi017 ceph-mon[222878]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:54.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/894453254' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:06:54.923 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:54.952 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986332 got 133143986334 for osd.4 2023-09-23T15:06:54.953 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:55.167 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:06:55.409 INFO:tasks.cephadm.ceph_manager.ceph:need seq 408021893146 got 408021893148 for osd.0 2023-09-23T15:06:55.409 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:55.487 INFO:teuthology.orchestra.run.smithi017.stdout:236223201426 2023-09-23T15:06:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:55 smithi017 ceph-mon[215646]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2707795422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:06:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:55 smithi017 ceph-mon[222878]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2707795422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:06:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:55 smithi027 ceph-mon[195233]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2707795422' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:06:57.627 INFO:teuthology.orchestra.run.smithi017.stdout:450971566096 2023-09-23T15:06:57.728 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201423 got 236223201426 for osd.7 2023-09-23T15:06:57.729 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:57.842 INFO:teuthology.orchestra.run.smithi017.stdout:429496729623 2023-09-23T15:06:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:57 smithi017 ceph-mon[215646]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/921005763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:06:58.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:57 smithi017 ceph-mon[222878]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:58.021 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/921005763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:06:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:57 smithi027 ceph-mon[195233]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/921005763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:06:58.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:06:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:06:58] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2023-09-23T15:06:58.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1973700676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:06:58.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:58.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1973700676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:06:58.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:58.994 INFO:tasks.cephadm.ceph_manager.ceph:need seq 450971566094 got 450971566096 for osd.2 2023-09-23T15:06:58.995 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1973700676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:06:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:06:59.299 INFO:tasks.cephadm.ceph_manager.ceph:need seq 429496729621 got 429496729623 for osd.1 2023-09-23T15:06:59.300 DEBUG:teuthology.parallel:result is None 2023-09-23T15:06:59.664 INFO:teuthology.orchestra.run.smithi017.stdout:472446402570 2023-09-23T15:06:59.708 INFO:teuthology.orchestra.run.smithi017.stdout:167503724698 2023-09-23T15:06:59.889 INFO:teuthology.orchestra.run.smithi017.stdout:201863463062 2023-09-23T15:06:59.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[215646]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:59.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1330154673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:06:59.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/317029158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:06:59.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[222878]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:06:59.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1330154673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:06:59.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:06:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/317029158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:07:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:59 smithi027 ceph-mon[195233]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1330154673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:07:00.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:06:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/317029158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:07:00.782 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402568 got 472446402570 for osd.3 2023-09-23T15:07:00.783 DEBUG:teuthology.parallel:result is None 2023-09-23T15:07:00.839 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724696 got 167503724698 for osd.5 2023-09-23T15:07:00.840 DEBUG:teuthology.parallel:result is None 2023-09-23T15:07:01.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3639024749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:07:01.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3639024749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:07:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3639024749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:07:01.354 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463060 got 201863463062 for osd.6 2023-09-23T15:07:01.354 DEBUG:teuthology.parallel:result is None 2023-09-23T15:07:01.354 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T15:07:01.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:07:01.723 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:01 smithi027 ceph-mon[195233]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:02.109 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:01 smithi017 ceph-mon[215646]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:02.109 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:01 smithi017 ceph-mon[222878]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:03.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:02 smithi017 ceph-mon[215646]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:03.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:02 smithi017 ceph-mon[222878]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:02 smithi027 ceph-mon[195233]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:03.489 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:07:03.490 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:07:04.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:03 smithi017 ceph-mon[222878]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:04.060 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:03 smithi017 ceph-mon[215646]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:04.075 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":220,"stamp":"2023-09-23T15:07:02.119753+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164112,"kb_used_data":23120,"kb_used_omap":30,"kb_used_meta":140897,"kb_avail":749797104,"statfs":{"total":767960285184,"available":767792234496,"internally_reserved":0,"allocated":23674880,"data_stored":19356051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":144279128},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003005"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113418+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:06:15.113418+0000","last_peered":"2023-09-23T15:06:15.113418+0000","last_clean":"2023-09-23T15:06:15.113418+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:06:15.113418+0000","last_undegraded":"2023-09-23T15:06:15.113418+0000","last_fullsized":"2023-09-23T15:06:15.113418+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116663+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:06:16.116663+0000","last_peered":"2023-09-23T15:06:16.116663+0000","last_clean":"2023-09-23T15:06:16.116663+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:06:16.116663+0000","last_undegraded":"2023-09-23T15:06:16.116663+0000","last_fullsized":"2023-09-23T15:06:16.116663+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133197+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:06:16.133197+0000","last_peered":"2023-09-23T15:06:16.133197+0000","last_clean":"2023-09-23T15:06:16.133197+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:06:16.133197+0000","last_undegraded":"2023-09-23T15:06:16.133197+0000","last_fullsized":"2023-09-23T15:06:16.133197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":79,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129954+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:06:16.129954+0000","last_peered":"2023-09-23T15:06:16.129954+0000","last_clean":"2023-09-23T15:06:16.129954+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:06:16.129954+0000","last_undegraded":"2023-09-23T15:06:16.129954+0000","last_fullsized":"2023-09-23T15:06:16.129954+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131672+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:06:16.131672+0000","last_peered":"2023-09-23T15:06:16.131672+0000","last_clean":"2023-09-23T15:06:16.131672+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:06:16.131672+0000","last_undegraded":"2023-09-23T15:06:16.131672+0000","last_fullsized":"2023-09-23T15:06:16.131672+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.128107+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:06:15.128107+0000","last_peered":"2023-09-23T15:06:15.128107+0000","last_clean":"2023-09-23T15:06:15.128107+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:06:15.128107+0000","last_undegraded":"2023-09-23T15:06:15.128107+0000","last_fullsized":"2023-09-23T15:06:15.128107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138486+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:06:16.138486+0000","last_peered":"2023-09-23T15:06:16.138486+0000","last_clean":"2023-09-23T15:06:16.138486+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:06:16.138486+0000","last_undegraded":"2023-09-23T15:06:16.138486+0000","last_fullsized":"2023-09-23T15:06:16.138486+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.126405+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:06:15.126405+0000","last_peered":"2023-09-23T15:06:15.126405+0000","last_clean":"2023-09-23T15:06:15.126405+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:06:15.126405+0000","last_undegraded":"2023-09-23T15:06:15.126405+0000","last_fullsized":"2023-09-23T15:06:15.126405+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131256+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:06:16.131256+0000","last_peered":"2023-09-23T15:06:16.131256+0000","last_clean":"2023-09-23T15:06:16.131256+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:06:16.131256+0000","last_undegraded":"2023-09-23T15:06:16.131256+0000","last_fullsized":"2023-09-23T15:06:16.131256+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.123017+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:06:15.123017+0000","last_peered":"2023-09-23T15:06:15.123017+0000","last_clean":"2023-09-23T15:06:15.123017+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:06:15.123017+0000","last_undegraded":"2023-09-23T15:06:15.123017+0000","last_fullsized":"2023-09-23T15:06:15.123017+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114344+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:06:15.114344+0000","last_peered":"2023-09-23T15:06:15.114344+0000","last_clean":"2023-09-23T15:06:15.114344+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:06:15.114344+0000","last_undegraded":"2023-09-23T15:06:15.114344+0000","last_fullsized":"2023-09-23T15:06:15.114344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117161+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:06:16.117161+0000","last_peered":"2023-09-23T15:06:16.117161+0000","last_clean":"2023-09-23T15:06:16.117161+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:06:16.117161+0000","last_undegraded":"2023-09-23T15:06:16.117161+0000","last_fullsized":"2023-09-23T15:06:16.117161+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":542,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121716+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:06:15.121716+0000","last_peered":"2023-09-23T15:06:15.121716+0000","last_clean":"2023-09-23T15:06:15.121716+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:06:15.121716+0000","last_undegraded":"2023-09-23T15:06:15.121716+0000","last_fullsized":"2023-09-23T15:06:15.121716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131999+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:06:16.131999+0000","last_peered":"2023-09-23T15:06:16.131999+0000","last_clean":"2023-09-23T15:06:16.131999+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:06:16.131999+0000","last_undegraded":"2023-09-23T15:06:16.131999+0000","last_fullsized":"2023-09-23T15:06:16.131999+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114934+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:06:15.114934+0000","last_peered":"2023-09-23T15:06:15.114934+0000","last_clean":"2023-09-23T15:06:15.114934+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:06:15.114934+0000","last_undegraded":"2023-09-23T15:06:15.114934+0000","last_fullsized":"2023-09-23T15:06:15.114934+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133258+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:06:16.133258+0000","last_peered":"2023-09-23T15:06:16.133258+0000","last_clean":"2023-09-23T15:06:16.133258+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:06:16.133258+0000","last_undegraded":"2023-09-23T15:06:16.133258+0000","last_fullsized":"2023-09-23T15:06:16.133258+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116804+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:06:16.116804+0000","last_peered":"2023-09-23T15:06:16.116804+0000","last_clean":"2023-09-23T15:06:16.116804+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:06:16.116804+0000","last_undegraded":"2023-09-23T15:06:16.116804+0000","last_fullsized":"2023-09-23T15:06:16.116804+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113833+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:06:15.113833+0000","last_peered":"2023-09-23T15:06:15.113833+0000","last_clean":"2023-09-23T15:06:15.113833+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:06:15.113833+0000","last_undegraded":"2023-09-23T15:06:15.113833+0000","last_fullsized":"2023-09-23T15:06:15.113833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":781,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121969+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:06:15.121969+0000","last_peered":"2023-09-23T15:06:15.121969+0000","last_clean":"2023-09-23T15:06:15.121969+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:06:15.121969+0000","last_undegraded":"2023-09-23T15:06:15.121969+0000","last_fullsized":"2023-09-23T15:06:15.121969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117225+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:06:16.117225+0000","last_peered":"2023-09-23T15:06:16.117225+0000","last_clean":"2023-09-23T15:06:16.117225+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:06:16.117225+0000","last_undegraded":"2023-09-23T15:06:16.117225+0000","last_fullsized":"2023-09-23T15:06:16.117225+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131654+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:06:16.131654+0000","last_peered":"2023-09-23T15:06:16.131654+0000","last_clean":"2023-09-23T15:06:16.131654+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:06:16.131654+0000","last_undegraded":"2023-09-23T15:06:16.131654+0000","last_fullsized":"2023-09-23T15:06:16.131654+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":316,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119475+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:06:15.119475+0000","last_peered":"2023-09-23T15:06:15.119475+0000","last_clean":"2023-09-23T15:06:15.119475+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:06:15.119475+0000","last_undegraded":"2023-09-23T15:06:15.119475+0000","last_fullsized":"2023-09-23T15:06:15.119475+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121344+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:06:15.121344+0000","last_peered":"2023-09-23T15:06:15.121344+0000","last_clean":"2023-09-23T15:06:15.121344+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:06:15.121344+0000","last_undegraded":"2023-09-23T15:06:15.121344+0000","last_fullsized":"2023-09-23T15:06:15.121344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134611+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:06:16.134611+0000","last_peered":"2023-09-23T15:06:16.134611+0000","last_clean":"2023-09-23T15:06:16.134611+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:06:16.134611+0000","last_undegraded":"2023-09-23T15:06:16.134611+0000","last_fullsized":"2023-09-23T15:06:16.134611+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":451,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119438+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:06:15.119438+0000","last_peered":"2023-09-23T15:06:15.119438+0000","last_clean":"2023-09-23T15:06:15.119438+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:06:15.119438+0000","last_undegraded":"2023-09-23T15:06:15.119438+0000","last_fullsized":"2023-09-23T15:06:15.119438+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":356,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125051+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:06:15.125051+0000","last_peered":"2023-09-23T15:06:15.125051+0000","last_clean":"2023-09-23T15:06:15.125051+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:06:15.125051+0000","last_undegraded":"2023-09-23T15:06:15.125051+0000","last_fullsized":"2023-09-23T15:06:15.125051+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114264+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:06:15.114264+0000","last_peered":"2023-09-23T15:06:15.114264+0000","last_clean":"2023-09-23T15:06:15.114264+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:06:15.114264+0000","last_undegraded":"2023-09-23T15:06:15.114264+0000","last_fullsized":"2023-09-23T15:06:15.114264+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113478+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:06:15.113478+0000","last_peered":"2023-09-23T15:06:15.113478+0000","last_clean":"2023-09-23T15:06:15.113478+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:06:15.113478+0000","last_undegraded":"2023-09-23T15:06:15.113478+0000","last_fullsized":"2023-09-23T15:06:15.113478+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":719,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129635+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:06:16.129635+0000","last_peered":"2023-09-23T15:06:16.129635+0000","last_clean":"2023-09-23T15:06:16.129635+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:06:16.129635+0000","last_undegraded":"2023-09-23T15:06:16.129635+0000","last_fullsized":"2023-09-23T15:06:16.129635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129423+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:06:16.129423+0000","last_peered":"2023-09-23T15:06:16.129423+0000","last_clean":"2023-09-23T15:06:16.129423+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:06:16.129423+0000","last_undegraded":"2023-09-23T15:06:16.129423+0000","last_fullsized":"2023-09-23T15:06:16.129423+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113741+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:06:15.113741+0000","last_peered":"2023-09-23T15:06:15.113741+0000","last_clean":"2023-09-23T15:06:15.113741+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:06:15.113741+0000","last_undegraded":"2023-09-23T15:06:15.113741+0000","last_fullsized":"2023-09-23T15:06:15.113741+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":612,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125189+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:06:15.125189+0000","last_peered":"2023-09-23T15:06:15.125189+0000","last_clean":"2023-09-23T15:06:15.125189+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:06:15.125189+0000","last_undegraded":"2023-09-23T15:06:15.125189+0000","last_fullsized":"2023-09-23T15:06:15.125189+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.110328+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:06:15.110328+0000","last_peered":"2023-09-23T15:06:15.110328+0000","last_clean":"2023-09-23T15:06:15.110328+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:06:15.110328+0000","last_undegraded":"2023-09-23T15:06:15.110328+0000","last_fullsized":"2023-09-23T15:06:15.110328+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113738+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:06:15.113738+0000","last_peered":"2023-09-23T15:06:15.113738+0000","last_clean":"2023-09-23T15:06:15.113738+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:06:15.113738+0000","last_undegraded":"2023-09-23T15:06:15.113738+0000","last_fullsized":"2023-09-23T15:06:15.113738+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128245+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:06:16.128245+0000","last_peered":"2023-09-23T15:06:16.128245+0000","last_clean":"2023-09-23T15:06:16.128245+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:06:16.128245+0000","last_undegraded":"2023-09-23T15:06:16.128245+0000","last_fullsized":"2023-09-23T15:06:16.128245+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132588+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:06:16.132588+0000","last_peered":"2023-09-23T15:06:16.132588+0000","last_clean":"2023-09-23T15:06:16.132588+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:06:16.132588+0000","last_undegraded":"2023-09-23T15:06:16.132588+0000","last_fullsized":"2023-09-23T15:06:16.132588+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114017+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:06:15.114017+0000","last_peered":"2023-09-23T15:06:15.114017+0000","last_clean":"2023-09-23T15:06:15.114017+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:06:15.114017+0000","last_undegraded":"2023-09-23T15:06:15.114017+0000","last_fullsized":"2023-09-23T15:06:15.114017+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115143+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:06:15.115143+0000","last_peered":"2023-09-23T15:06:15.115143+0000","last_clean":"2023-09-23T15:06:15.115143+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:06:15.115143+0000","last_undegraded":"2023-09-23T15:06:15.115143+0000","last_fullsized":"2023-09-23T15:06:15.115143+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115699+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:06:15.115699+0000","last_peered":"2023-09-23T15:06:15.115699+0000","last_clean":"2023-09-23T15:06:15.115699+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:06:15.115699+0000","last_undegraded":"2023-09-23T15:06:15.115699+0000","last_fullsized":"2023-09-23T15:06:15.115699+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120271+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:06:15.120271+0000","last_peered":"2023-09-23T15:06:15.120271+0000","last_clean":"2023-09-23T15:06:15.120271+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:06:15.120271+0000","last_undegraded":"2023-09-23T15:06:15.120271+0000","last_fullsized":"2023-09-23T15:06:15.120271+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137694+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:06:16.137694+0000","last_peered":"2023-09-23T15:06:16.137694+0000","last_clean":"2023-09-23T15:06:16.137694+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:06:16.137694+0000","last_undegraded":"2023-09-23T15:06:16.137694+0000","last_fullsized":"2023-09-23T15:06:16.137694+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132545+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:06:16.132545+0000","last_peered":"2023-09-23T15:06:16.132545+0000","last_clean":"2023-09-23T15:06:16.132545+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:06:16.132545+0000","last_undegraded":"2023-09-23T15:06:16.132545+0000","last_fullsized":"2023-09-23T15:06:16.132545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116963+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:06:16.116963+0000","last_peered":"2023-09-23T15:06:16.116963+0000","last_clean":"2023-09-23T15:06:16.116963+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:06:16.116963+0000","last_undegraded":"2023-09-23T15:06:16.116963+0000","last_fullsized":"2023-09-23T15:06:16.116963+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120111+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:06:15.120111+0000","last_peered":"2023-09-23T15:06:15.120111+0000","last_clean":"2023-09-23T15:06:15.120111+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:06:15.120111+0000","last_undegraded":"2023-09-23T15:06:15.120111+0000","last_fullsized":"2023-09-23T15:06:15.120111+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.122093+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:06:15.122093+0000","last_peered":"2023-09-23T15:06:15.122093+0000","last_clean":"2023-09-23T15:06:15.122093+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:06:15.122093+0000","last_undegraded":"2023-09-23T15:06:15.122093+0000","last_fullsized":"2023-09-23T15:06:15.122093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131518+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:06:16.131518+0000","last_peered":"2023-09-23T15:06:16.131518+0000","last_clean":"2023-09-23T15:06:16.131518+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:06:16.131518+0000","last_undegraded":"2023-09-23T15:06:16.131518+0000","last_fullsized":"2023-09-23T15:06:16.131518+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118294+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:06:15.118294+0000","last_peered":"2023-09-23T15:06:15.118294+0000","last_clean":"2023-09-23T15:06:15.118294+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:06:15.118294+0000","last_undegraded":"2023-09-23T15:06:15.118294+0000","last_fullsized":"2023-09-23T15:06:15.118294+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124626+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:06:15.124626+0000","last_peered":"2023-09-23T15:06:15.124626+0000","last_clean":"2023-09-23T15:06:15.124626+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:06:15.124626+0000","last_undegraded":"2023-09-23T15:06:15.124626+0000","last_fullsized":"2023-09-23T15:06:15.124626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119275+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:06:15.119275+0000","last_peered":"2023-09-23T15:06:15.119275+0000","last_clean":"2023-09-23T15:06:15.119275+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:06:15.119275+0000","last_undegraded":"2023-09-23T15:06:15.119275+0000","last_fullsized":"2023-09-23T15:06:15.119275+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131473+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:06:16.131473+0000","last_peered":"2023-09-23T15:06:16.131473+0000","last_clean":"2023-09-23T15:06:16.131473+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:06:16.131473+0000","last_undegraded":"2023-09-23T15:06:16.131473+0000","last_fullsized":"2023-09-23T15:06:16.131473+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118992+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:06:15.118992+0000","last_peered":"2023-09-23T15:06:15.118992+0000","last_clean":"2023-09-23T15:06:15.118992+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:06:15.118992+0000","last_undegraded":"2023-09-23T15:06:15.118992+0000","last_fullsized":"2023-09-23T15:06:15.118992+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.127799+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:06:16.127799+0000","last_peered":"2023-09-23T15:06:16.127799+0000","last_clean":"2023-09-23T15:06:16.127799+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:06:16.127799+0000","last_undegraded":"2023-09-23T15:06:16.127799+0000","last_fullsized":"2023-09-23T15:06:16.127799+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119363+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:06:15.119363+0000","last_peered":"2023-09-23T15:06:15.119363+0000","last_clean":"2023-09-23T15:06:15.119363+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:06:15.119363+0000","last_undegraded":"2023-09-23T15:06:15.119363+0000","last_fullsized":"2023-09-23T15:06:15.119363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138039+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:06:16.138039+0000","last_peered":"2023-09-23T15:06:16.138039+0000","last_clean":"2023-09-23T15:06:16.138039+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:06:16.138039+0000","last_undegraded":"2023-09-23T15:06:16.138039+0000","last_fullsized":"2023-09-23T15:06:16.138039+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.127131+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:06:16.127131+0000","last_peered":"2023-09-23T15:06:16.127131+0000","last_clean":"2023-09-23T15:06:16.127131+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:06:16.127131+0000","last_undegraded":"2023-09-23T15:06:16.127131+0000","last_fullsized":"2023-09-23T15:06:16.127131+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.135551+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:06:16.135551+0000","last_peered":"2023-09-23T15:06:16.135551+0000","last_clean":"2023-09-23T15:06:16.135551+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:06:16.135551+0000","last_undegraded":"2023-09-23T15:06:16.135551+0000","last_fullsized":"2023-09-23T15:06:16.135551+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118610+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:06:15.118610+0000","last_peered":"2023-09-23T15:06:15.118610+0000","last_clean":"2023-09-23T15:06:15.118610+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:06:15.118610+0000","last_undegraded":"2023-09-23T15:06:15.118610+0000","last_fullsized":"2023-09-23T15:06:15.118610+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113896+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:06:15.113896+0000","last_peered":"2023-09-23T15:06:15.113896+0000","last_clean":"2023-09-23T15:06:15.113896+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:06:15.113896+0000","last_undegraded":"2023-09-23T15:06:15.113896+0000","last_fullsized":"2023-09-23T15:06:15.113896+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113914+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:06:15.113914+0000","last_peered":"2023-09-23T15:06:15.113914+0000","last_clean":"2023-09-23T15:06:15.113914+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:06:15.113914+0000","last_undegraded":"2023-09-23T15:06:15.113914+0000","last_fullsized":"2023-09-23T15:06:15.113914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113241+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:06:15.113241+0000","last_peered":"2023-09-23T15:06:15.113241+0000","last_clean":"2023-09-23T15:06:15.113241+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:06:15.113241+0000","last_undegraded":"2023-09-23T15:06:15.113241+0000","last_fullsized":"2023-09-23T15:06:15.113241+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137711+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:06:16.137711+0000","last_peered":"2023-09-23T15:06:16.137711+0000","last_clean":"2023-09-23T15:06:16.137711+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:06:16.137711+0000","last_undegraded":"2023-09-23T15:06:16.137711+0000","last_fullsized":"2023-09-23T15:06:16.137711+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134068+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:06:16.134068+0000","last_peered":"2023-09-23T15:06:16.134068+0000","last_clean":"2023-09-23T15:06:16.134068+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:06:16.134068+0000","last_undegraded":"2023-09-23T15:06:16.134068+0000","last_fullsized":"2023-09-23T15:06:16.134068+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124812+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:06:15.124812+0000","last_peered":"2023-09-23T15:06:15.124812+0000","last_clean":"2023-09-23T15:06:15.124812+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:06:15.124812+0000","last_undegraded":"2023-09-23T15:06:15.124812+0000","last_fullsized":"2023-09-23T15:06:15.124812+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131810+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:06:16.131810+0000","last_peered":"2023-09-23T15:06:16.131810+0000","last_clean":"2023-09-23T15:06:16.131810+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:06:16.131810+0000","last_undegraded":"2023-09-23T15:06:16.131810+0000","last_fullsized":"2023-09-23T15:06:16.131810+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":52,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.126708+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:06:16.126708+0000","last_peered":"2023-09-23T15:06:16.126708+0000","last_clean":"2023-09-23T15:06:16.126708+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:06:16.126708+0000","last_undegraded":"2023-09-23T15:06:16.126708+0000","last_fullsized":"2023-09-23T15:06:16.126708+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121746+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:06:15.121746+0000","last_peered":"2023-09-23T15:06:15.121746+0000","last_clean":"2023-09-23T15:06:15.121746+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:06:15.121746+0000","last_undegraded":"2023-09-23T15:06:15.121746+0000","last_fullsized":"2023-09-23T15:06:15.121746+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132197+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:06:16.132197+0000","last_peered":"2023-09-23T15:06:16.132197+0000","last_clean":"2023-09-23T15:06:16.132197+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:06:16.132197+0000","last_undegraded":"2023-09-23T15:06:16.132197+0000","last_fullsized":"2023-09-23T15:06:16.132197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128449+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:06:16.128449+0000","last_peered":"2023-09-23T15:06:16.128449+0000","last_clean":"2023-09-23T15:06:16.128449+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:06:16.128449+0000","last_undegraded":"2023-09-23T15:06:16.128449+0000","last_fullsized":"2023-09-23T15:06:16.128449+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129498+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:06:16.129498+0000","last_peered":"2023-09-23T15:06:16.129498+0000","last_clean":"2023-09-23T15:06:16.129498+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:06:16.129498+0000","last_undegraded":"2023-09-23T15:06:16.129498+0000","last_fullsized":"2023-09-23T15:06:16.129498+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":37,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134796+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:06:16.134796+0000","last_peered":"2023-09-23T15:06:16.134796+0000","last_clean":"2023-09-23T15:06:16.134796+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:06:16.134796+0000","last_undegraded":"2023-09-23T15:06:16.134796+0000","last_fullsized":"2023-09-23T15:06:16.134796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113716+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:06:15.113716+0000","last_peered":"2023-09-23T15:06:15.113716+0000","last_clean":"2023-09-23T15:06:15.113716+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:06:15.113716+0000","last_undegraded":"2023-09-23T15:06:15.113716+0000","last_fullsized":"2023-09-23T15:06:15.113716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201427,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16948,"kb_used_data":5804,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728204,"statfs":{"total":95995035648,"available":95977680896,"internally_reserved":0,"allocated":5943296,"data_stored":5408381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57499999999999996,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.59999999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.628},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.65000000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.49299999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":1.585,"15min":1.7529999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65900000000000003,"15min":0.624},"min":{"1min":0.48999999999999999,"5min":0.39700000000000002,"15min":0.375},"max":{"1min":0.86599999999999999,"5min":1.5289999999999999,"15min":3.4809999999999999},"last":0.70099999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54300000000000004,"15min":0.50900000000000001},"min":{"1min":0.38200000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80700000000000005,"5min":0.82899999999999996,"15min":2.226},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.53800000000000003},"min":{"1min":0.52900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.71899999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57799999999999996,"15min":0.56399999999999995},"min":{"1min":0.47799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80800000000000005,"15min":0.92700000000000005},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.622,"15min":0.58299999999999996},"min":{"1min":0.49399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.88600000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.72799999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63900000000000001,"15min":0.59399999999999997},"min":{"1min":0.55000000000000004,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":1.002,"15min":1.002},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.64600000000000002,"15min":0.60299999999999998},"min":{"1min":0.55000000000000004,"5min":0.307,"15min":0.307},"max":{"1min":0.878,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.68500000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463063,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8300,"kb_used_data":1700,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93736852,"statfs":{"total":95995035648,"available":95986536448,"internally_reserved":0,"allocated":1740800,"data_stored":1213950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71499999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.63,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.501},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.68100000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.60699999999999998,"15min":0.54100000000000004},"min":{"1min":0.42499999999999999,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":1.494,"15min":1.494},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.621,"5min":0.64400000000000002,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.372,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.626}]},{"osd":4,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.54100000000000004,"15min":0.497},"min":{"1min":0.38200000000000001,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":0.77300000000000002,"5min":1.675,"15min":1.675},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.58699999999999997,"15min":0.51300000000000001},"min":{"1min":0.376,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":0.83899999999999997,"5min":1.806,"15min":1.806},"last":0.64800000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":0.58599999999999997,"15min":0.52900000000000003},"min":{"1min":0.40400000000000003,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":0.80300000000000005,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.61499999999999999,"15min":0.55000000000000004},"min":{"1min":0.46400000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":0.89300000000000002,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.70899999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:06:18 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.58999999999999997},"min":{"1min":0.443,"5min":0.35599999999999998,"15min":0.28000000000000003},"max":{"1min":0.90300000000000002,"5min":1.843,"15min":1.843},"last":0.67100000000000004},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.69299999999999995,"15min":0.63400000000000001},"min":{"1min":0.56299999999999994,"5min":0.40999999999999998,"15min":0.373},"max":{"1min":0.88100000000000001,"5min":1.766,"15min":1.766},"last":0.61299999999999999}]}]},{"osd":1,"up_from":100,"seq":429496729624,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32844,"kb_used_data":5252,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93712308,"statfs":{"total":95995035648,"available":95961403392,"internally_reserved":0,"allocated":5378048,"data_stored":4823646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":28238923},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 15:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.26300000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30099999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.437}]}]},{"osd":0,"up_from":95,"seq":408021893150,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33440,"kb_used_data":5848,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711712,"statfs":{"total":95995035648,"available":95960793088,"internally_reserved":0,"allocated":5988352,"data_stored":5415358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":28237396},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:06:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.254},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.308}]},{"osd":2,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":0.746,"15min":0.68100000000000005},"min":{"1min":0.39500000000000002,"5min":0.375,"15min":0.375},"max":{"1min":8.1050000000000004,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.69499999999999995,"15min":0.69599999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.82399999999999995,"15min":0.76200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.71499999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68200000000000005},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.56499999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":1.175,"5min":0.80600000000000005,"15min":0.74399999999999999},"min":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.157,"5min":8.157,"15min":8.157},"last":0.57599999999999996}]}]},{"osd":2,"up_from":105,"seq":450971566097,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28680,"kb_used_data":1152,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93716472,"statfs":{"total":95995035648,"available":95965667328,"internally_reserved":0,"allocated":1179648,"data_stored":628060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":28168976},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.27000000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.496}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.47499999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.51700000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402571,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1128,"kb_used_omap":3,"kb_used_meta":27452,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1155072,"data_stored":623615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":28111017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33000000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986336,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7852,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737300,"statfs":{"total":95995035648,"available":95986995200,"internally_reserved":0,"allocated":1150976,"data_stored":623639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.378},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.54900000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.42299999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.59899999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.6619999999999999,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.42799999999999999,"5min":0.376,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.88200000000000001,"15min":0.93600000000000005},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60599999999999998,"15min":0.56499999999999995},"min":{"1min":0.436,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91400000000000003},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.51100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":0.77400000000000002,"5min":0.876,"15min":0.88400000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.52800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.26200000000000001},"max":{"1min":0.80100000000000005,"5min":0.85399999999999998,"15min":0.91700000000000004},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:05:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.61599999999999999,"15min":0.59299999999999997},"min":{"1min":0.39700000000000002,"5min":0.379,"15min":0.318},"max":{"1min":0.92500000000000004,"5min":0.92500000000000004,"15min":1.4610000000000001},"last":0.63600000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.61899999999999999,"15min":0.58099999999999996},"min":{"1min":0.47499999999999998,"5min":0.34200000000000003,"15min":0.33400000000000002},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.64600000000000002,"15min":0.59699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.28799999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69599999999999995,"15min":0.65800000000000003},"min":{"1min":0.48799999999999999,"5min":0.47699999999999998,"15min":0.38800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":1.0249999999999999},"last":0.69399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724699,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7456,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737696,"statfs":{"total":95995035648,"available":95987400704,"internally_reserved":0,"allocated":1138688,"data_stored":619402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.374,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.34300000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.35199999999999998,"15min":0.312},"max":{"1min":0.748,"5min":0.879,"15min":1.262},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.60399999999999998,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.373,"15min":0.35499999999999998},"max":{"1min":0.70899999999999996,"5min":0.96099999999999997,"15min":1.248},"last":0.505}]},{"osd":4,"last update":"Sat Sep 23 15:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.501,"15min":0.46000000000000002},"min":{"1min":0.40100000000000002,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":0.73499999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.50800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.74199999999999999,"15min":0.78400000000000003},"last":0.47699999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.58099999999999996,"15min":0.52800000000000002},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64800000000000002,"15min":0.59799999999999998},"min":{"1min":0.56999999999999995,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":0.80900000000000005,"5min":0.85499999999999998,"15min":0.91000000000000003},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.60799999999999998,"15min":0.59199999999999997},"min":{"1min":0.5,"5min":0.39400000000000002,"15min":0.28299999999999997},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":1.0109999999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.63300000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:07:04.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:07:04.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:03 smithi027 ceph-mon[195233]: from='client.44588 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:04.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:04 smithi017 ceph-mon[215646]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:04 smithi017 ceph-mon[222878]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:04 smithi027 ceph-mon[195233]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:07:06.087 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:07:06.692 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":221,"stamp":"2023-09-23T15:07:04.120105+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164112,"kb_used_data":23120,"kb_used_omap":30,"kb_used_meta":140897,"kb_avail":749797104,"statfs":{"total":767960285184,"available":767792234496,"internally_reserved":0,"allocated":23674880,"data_stored":19356051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":144279128},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002704"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113418+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:06:15.113418+0000","last_peered":"2023-09-23T15:06:15.113418+0000","last_clean":"2023-09-23T15:06:15.113418+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:06:15.113418+0000","last_undegraded":"2023-09-23T15:06:15.113418+0000","last_fullsized":"2023-09-23T15:06:15.113418+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116663+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:06:16.116663+0000","last_peered":"2023-09-23T15:06:16.116663+0000","last_clean":"2023-09-23T15:06:16.116663+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:06:16.116663+0000","last_undegraded":"2023-09-23T15:06:16.116663+0000","last_fullsized":"2023-09-23T15:06:16.116663+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133197+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:06:16.133197+0000","last_peered":"2023-09-23T15:06:16.133197+0000","last_clean":"2023-09-23T15:06:16.133197+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:06:16.133197+0000","last_undegraded":"2023-09-23T15:06:16.133197+0000","last_fullsized":"2023-09-23T15:06:16.133197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":79,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129954+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:06:16.129954+0000","last_peered":"2023-09-23T15:06:16.129954+0000","last_clean":"2023-09-23T15:06:16.129954+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:06:16.129954+0000","last_undegraded":"2023-09-23T15:06:16.129954+0000","last_fullsized":"2023-09-23T15:06:16.129954+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131672+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:06:16.131672+0000","last_peered":"2023-09-23T15:06:16.131672+0000","last_clean":"2023-09-23T15:06:16.131672+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:06:16.131672+0000","last_undegraded":"2023-09-23T15:06:16.131672+0000","last_fullsized":"2023-09-23T15:06:16.131672+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.128107+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:06:15.128107+0000","last_peered":"2023-09-23T15:06:15.128107+0000","last_clean":"2023-09-23T15:06:15.128107+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:06:15.128107+0000","last_undegraded":"2023-09-23T15:06:15.128107+0000","last_fullsized":"2023-09-23T15:06:15.128107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138486+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:06:16.138486+0000","last_peered":"2023-09-23T15:06:16.138486+0000","last_clean":"2023-09-23T15:06:16.138486+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:06:16.138486+0000","last_undegraded":"2023-09-23T15:06:16.138486+0000","last_fullsized":"2023-09-23T15:06:16.138486+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.126405+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:06:15.126405+0000","last_peered":"2023-09-23T15:06:15.126405+0000","last_clean":"2023-09-23T15:06:15.126405+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:06:15.126405+0000","last_undegraded":"2023-09-23T15:06:15.126405+0000","last_fullsized":"2023-09-23T15:06:15.126405+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131256+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:06:16.131256+0000","last_peered":"2023-09-23T15:06:16.131256+0000","last_clean":"2023-09-23T15:06:16.131256+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:06:16.131256+0000","last_undegraded":"2023-09-23T15:06:16.131256+0000","last_fullsized":"2023-09-23T15:06:16.131256+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":40,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.123017+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:06:15.123017+0000","last_peered":"2023-09-23T15:06:15.123017+0000","last_clean":"2023-09-23T15:06:15.123017+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:06:15.123017+0000","last_undegraded":"2023-09-23T15:06:15.123017+0000","last_fullsized":"2023-09-23T15:06:15.123017+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114344+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:06:15.114344+0000","last_peered":"2023-09-23T15:06:15.114344+0000","last_clean":"2023-09-23T15:06:15.114344+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:06:15.114344+0000","last_undegraded":"2023-09-23T15:06:15.114344+0000","last_fullsized":"2023-09-23T15:06:15.114344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117161+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:06:16.117161+0000","last_peered":"2023-09-23T15:06:16.117161+0000","last_clean":"2023-09-23T15:06:16.117161+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:06:16.117161+0000","last_undegraded":"2023-09-23T15:06:16.117161+0000","last_fullsized":"2023-09-23T15:06:16.117161+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":542,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121716+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:06:15.121716+0000","last_peered":"2023-09-23T15:06:15.121716+0000","last_clean":"2023-09-23T15:06:15.121716+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:06:15.121716+0000","last_undegraded":"2023-09-23T15:06:15.121716+0000","last_fullsized":"2023-09-23T15:06:15.121716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131999+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:06:16.131999+0000","last_peered":"2023-09-23T15:06:16.131999+0000","last_clean":"2023-09-23T15:06:16.131999+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:06:16.131999+0000","last_undegraded":"2023-09-23T15:06:16.131999+0000","last_fullsized":"2023-09-23T15:06:16.131999+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114934+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:06:15.114934+0000","last_peered":"2023-09-23T15:06:15.114934+0000","last_clean":"2023-09-23T15:06:15.114934+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:06:15.114934+0000","last_undegraded":"2023-09-23T15:06:15.114934+0000","last_fullsized":"2023-09-23T15:06:15.114934+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133258+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:06:16.133258+0000","last_peered":"2023-09-23T15:06:16.133258+0000","last_clean":"2023-09-23T15:06:16.133258+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:06:16.133258+0000","last_undegraded":"2023-09-23T15:06:16.133258+0000","last_fullsized":"2023-09-23T15:06:16.133258+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116804+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:06:16.116804+0000","last_peered":"2023-09-23T15:06:16.116804+0000","last_clean":"2023-09-23T15:06:16.116804+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:06:16.116804+0000","last_undegraded":"2023-09-23T15:06:16.116804+0000","last_fullsized":"2023-09-23T15:06:16.116804+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113833+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:06:15.113833+0000","last_peered":"2023-09-23T15:06:15.113833+0000","last_clean":"2023-09-23T15:06:15.113833+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:06:15.113833+0000","last_undegraded":"2023-09-23T15:06:15.113833+0000","last_fullsized":"2023-09-23T15:06:15.113833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":781,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121969+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:06:15.121969+0000","last_peered":"2023-09-23T15:06:15.121969+0000","last_clean":"2023-09-23T15:06:15.121969+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:06:15.121969+0000","last_undegraded":"2023-09-23T15:06:15.121969+0000","last_fullsized":"2023-09-23T15:06:15.121969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117225+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:06:16.117225+0000","last_peered":"2023-09-23T15:06:16.117225+0000","last_clean":"2023-09-23T15:06:16.117225+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:06:16.117225+0000","last_undegraded":"2023-09-23T15:06:16.117225+0000","last_fullsized":"2023-09-23T15:06:16.117225+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131654+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:06:16.131654+0000","last_peered":"2023-09-23T15:06:16.131654+0000","last_clean":"2023-09-23T15:06:16.131654+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:06:16.131654+0000","last_undegraded":"2023-09-23T15:06:16.131654+0000","last_fullsized":"2023-09-23T15:06:16.131654+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":316,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119475+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:06:15.119475+0000","last_peered":"2023-09-23T15:06:15.119475+0000","last_clean":"2023-09-23T15:06:15.119475+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:06:15.119475+0000","last_undegraded":"2023-09-23T15:06:15.119475+0000","last_fullsized":"2023-09-23T15:06:15.119475+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121344+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:06:15.121344+0000","last_peered":"2023-09-23T15:06:15.121344+0000","last_clean":"2023-09-23T15:06:15.121344+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:06:15.121344+0000","last_undegraded":"2023-09-23T15:06:15.121344+0000","last_fullsized":"2023-09-23T15:06:15.121344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134611+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:06:16.134611+0000","last_peered":"2023-09-23T15:06:16.134611+0000","last_clean":"2023-09-23T15:06:16.134611+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:06:16.134611+0000","last_undegraded":"2023-09-23T15:06:16.134611+0000","last_fullsized":"2023-09-23T15:06:16.134611+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":451,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119438+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:06:15.119438+0000","last_peered":"2023-09-23T15:06:15.119438+0000","last_clean":"2023-09-23T15:06:15.119438+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:06:15.119438+0000","last_undegraded":"2023-09-23T15:06:15.119438+0000","last_fullsized":"2023-09-23T15:06:15.119438+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":356,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125051+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:06:15.125051+0000","last_peered":"2023-09-23T15:06:15.125051+0000","last_clean":"2023-09-23T15:06:15.125051+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:06:15.125051+0000","last_undegraded":"2023-09-23T15:06:15.125051+0000","last_fullsized":"2023-09-23T15:06:15.125051+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114264+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:06:15.114264+0000","last_peered":"2023-09-23T15:06:15.114264+0000","last_clean":"2023-09-23T15:06:15.114264+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:06:15.114264+0000","last_undegraded":"2023-09-23T15:06:15.114264+0000","last_fullsized":"2023-09-23T15:06:15.114264+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113478+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:06:15.113478+0000","last_peered":"2023-09-23T15:06:15.113478+0000","last_clean":"2023-09-23T15:06:15.113478+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:06:15.113478+0000","last_undegraded":"2023-09-23T15:06:15.113478+0000","last_fullsized":"2023-09-23T15:06:15.113478+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":719,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129635+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:06:16.129635+0000","last_peered":"2023-09-23T15:06:16.129635+0000","last_clean":"2023-09-23T15:06:16.129635+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:06:16.129635+0000","last_undegraded":"2023-09-23T15:06:16.129635+0000","last_fullsized":"2023-09-23T15:06:16.129635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129423+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:06:16.129423+0000","last_peered":"2023-09-23T15:06:16.129423+0000","last_clean":"2023-09-23T15:06:16.129423+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:06:16.129423+0000","last_undegraded":"2023-09-23T15:06:16.129423+0000","last_fullsized":"2023-09-23T15:06:16.129423+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113741+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:06:15.113741+0000","last_peered":"2023-09-23T15:06:15.113741+0000","last_clean":"2023-09-23T15:06:15.113741+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:06:15.113741+0000","last_undegraded":"2023-09-23T15:06:15.113741+0000","last_fullsized":"2023-09-23T15:06:15.113741+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":612,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125189+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:06:15.125189+0000","last_peered":"2023-09-23T15:06:15.125189+0000","last_clean":"2023-09-23T15:06:15.125189+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:06:15.125189+0000","last_undegraded":"2023-09-23T15:06:15.125189+0000","last_fullsized":"2023-09-23T15:06:15.125189+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.110328+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:06:15.110328+0000","last_peered":"2023-09-23T15:06:15.110328+0000","last_clean":"2023-09-23T15:06:15.110328+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:06:15.110328+0000","last_undegraded":"2023-09-23T15:06:15.110328+0000","last_fullsized":"2023-09-23T15:06:15.110328+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113738+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:06:15.113738+0000","last_peered":"2023-09-23T15:06:15.113738+0000","last_clean":"2023-09-23T15:06:15.113738+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:06:15.113738+0000","last_undegraded":"2023-09-23T15:06:15.113738+0000","last_fullsized":"2023-09-23T15:06:15.113738+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128245+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:06:16.128245+0000","last_peered":"2023-09-23T15:06:16.128245+0000","last_clean":"2023-09-23T15:06:16.128245+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:06:16.128245+0000","last_undegraded":"2023-09-23T15:06:16.128245+0000","last_fullsized":"2023-09-23T15:06:16.128245+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132588+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:06:16.132588+0000","last_peered":"2023-09-23T15:06:16.132588+0000","last_clean":"2023-09-23T15:06:16.132588+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:06:16.132588+0000","last_undegraded":"2023-09-23T15:06:16.132588+0000","last_fullsized":"2023-09-23T15:06:16.132588+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114017+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:06:15.114017+0000","last_peered":"2023-09-23T15:06:15.114017+0000","last_clean":"2023-09-23T15:06:15.114017+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:06:15.114017+0000","last_undegraded":"2023-09-23T15:06:15.114017+0000","last_fullsized":"2023-09-23T15:06:15.114017+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115143+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:06:15.115143+0000","last_peered":"2023-09-23T15:06:15.115143+0000","last_clean":"2023-09-23T15:06:15.115143+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:06:15.115143+0000","last_undegraded":"2023-09-23T15:06:15.115143+0000","last_fullsized":"2023-09-23T15:06:15.115143+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115699+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:06:15.115699+0000","last_peered":"2023-09-23T15:06:15.115699+0000","last_clean":"2023-09-23T15:06:15.115699+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:06:15.115699+0000","last_undegraded":"2023-09-23T15:06:15.115699+0000","last_fullsized":"2023-09-23T15:06:15.115699+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120271+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:06:15.120271+0000","last_peered":"2023-09-23T15:06:15.120271+0000","last_clean":"2023-09-23T15:06:15.120271+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:06:15.120271+0000","last_undegraded":"2023-09-23T15:06:15.120271+0000","last_fullsized":"2023-09-23T15:06:15.120271+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137694+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:06:16.137694+0000","last_peered":"2023-09-23T15:06:16.137694+0000","last_clean":"2023-09-23T15:06:16.137694+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:06:16.137694+0000","last_undegraded":"2023-09-23T15:06:16.137694+0000","last_fullsized":"2023-09-23T15:06:16.137694+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132545+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:06:16.132545+0000","last_peered":"2023-09-23T15:06:16.132545+0000","last_clean":"2023-09-23T15:06:16.132545+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:06:16.132545+0000","last_undegraded":"2023-09-23T15:06:16.132545+0000","last_fullsized":"2023-09-23T15:06:16.132545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116963+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:06:16.116963+0000","last_peered":"2023-09-23T15:06:16.116963+0000","last_clean":"2023-09-23T15:06:16.116963+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:06:16.116963+0000","last_undegraded":"2023-09-23T15:06:16.116963+0000","last_fullsized":"2023-09-23T15:06:16.116963+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120111+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:06:15.120111+0000","last_peered":"2023-09-23T15:06:15.120111+0000","last_clean":"2023-09-23T15:06:15.120111+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:06:15.120111+0000","last_undegraded":"2023-09-23T15:06:15.120111+0000","last_fullsized":"2023-09-23T15:06:15.120111+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.122093+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:06:15.122093+0000","last_peered":"2023-09-23T15:06:15.122093+0000","last_clean":"2023-09-23T15:06:15.122093+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:06:15.122093+0000","last_undegraded":"2023-09-23T15:06:15.122093+0000","last_fullsized":"2023-09-23T15:06:15.122093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131518+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:06:16.131518+0000","last_peered":"2023-09-23T15:06:16.131518+0000","last_clean":"2023-09-23T15:06:16.131518+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:06:16.131518+0000","last_undegraded":"2023-09-23T15:06:16.131518+0000","last_fullsized":"2023-09-23T15:06:16.131518+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118294+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:06:15.118294+0000","last_peered":"2023-09-23T15:06:15.118294+0000","last_clean":"2023-09-23T15:06:15.118294+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:06:15.118294+0000","last_undegraded":"2023-09-23T15:06:15.118294+0000","last_fullsized":"2023-09-23T15:06:15.118294+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124626+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:06:15.124626+0000","last_peered":"2023-09-23T15:06:15.124626+0000","last_clean":"2023-09-23T15:06:15.124626+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:06:15.124626+0000","last_undegraded":"2023-09-23T15:06:15.124626+0000","last_fullsized":"2023-09-23T15:06:15.124626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119275+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:06:15.119275+0000","last_peered":"2023-09-23T15:06:15.119275+0000","last_clean":"2023-09-23T15:06:15.119275+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:06:15.119275+0000","last_undegraded":"2023-09-23T15:06:15.119275+0000","last_fullsized":"2023-09-23T15:06:15.119275+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131473+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:06:16.131473+0000","last_peered":"2023-09-23T15:06:16.131473+0000","last_clean":"2023-09-23T15:06:16.131473+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:06:16.131473+0000","last_undegraded":"2023-09-23T15:06:16.131473+0000","last_fullsized":"2023-09-23T15:06:16.131473+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118992+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:06:15.118992+0000","last_peered":"2023-09-23T15:06:15.118992+0000","last_clean":"2023-09-23T15:06:15.118992+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:06:15.118992+0000","last_undegraded":"2023-09-23T15:06:15.118992+0000","last_fullsized":"2023-09-23T15:06:15.118992+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128037+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:06:16.128037+0000","last_peered":"2023-09-23T15:06:16.128037+0000","last_clean":"2023-09-23T15:06:16.128037+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:06:16.128037+0000","last_undegraded":"2023-09-23T15:06:16.128037+0000","last_fullsized":"2023-09-23T15:06:16.128037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119363+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:06:15.119363+0000","last_peered":"2023-09-23T15:06:15.119363+0000","last_clean":"2023-09-23T15:06:15.119363+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:06:15.119363+0000","last_undegraded":"2023-09-23T15:06:15.119363+0000","last_fullsized":"2023-09-23T15:06:15.119363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138039+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:06:16.138039+0000","last_peered":"2023-09-23T15:06:16.138039+0000","last_clean":"2023-09-23T15:06:16.138039+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:06:16.138039+0000","last_undegraded":"2023-09-23T15:06:16.138039+0000","last_fullsized":"2023-09-23T15:06:16.138039+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.127131+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:06:16.127131+0000","last_peered":"2023-09-23T15:06:16.127131+0000","last_clean":"2023-09-23T15:06:16.127131+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:06:16.127131+0000","last_undegraded":"2023-09-23T15:06:16.127131+0000","last_fullsized":"2023-09-23T15:06:16.127131+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.135551+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:06:16.135551+0000","last_peered":"2023-09-23T15:06:16.135551+0000","last_clean":"2023-09-23T15:06:16.135551+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:06:16.135551+0000","last_undegraded":"2023-09-23T15:06:16.135551+0000","last_fullsized":"2023-09-23T15:06:16.135551+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118610+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:06:15.118610+0000","last_peered":"2023-09-23T15:06:15.118610+0000","last_clean":"2023-09-23T15:06:15.118610+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:06:15.118610+0000","last_undegraded":"2023-09-23T15:06:15.118610+0000","last_fullsized":"2023-09-23T15:06:15.118610+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113896+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:06:15.113896+0000","last_peered":"2023-09-23T15:06:15.113896+0000","last_clean":"2023-09-23T15:06:15.113896+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:06:15.113896+0000","last_undegraded":"2023-09-23T15:06:15.113896+0000","last_fullsized":"2023-09-23T15:06:15.113896+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113914+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:06:15.113914+0000","last_peered":"2023-09-23T15:06:15.113914+0000","last_clean":"2023-09-23T15:06:15.113914+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:06:15.113914+0000","last_undegraded":"2023-09-23T15:06:15.113914+0000","last_fullsized":"2023-09-23T15:06:15.113914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113241+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:06:15.113241+0000","last_peered":"2023-09-23T15:06:15.113241+0000","last_clean":"2023-09-23T15:06:15.113241+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:06:15.113241+0000","last_undegraded":"2023-09-23T15:06:15.113241+0000","last_fullsized":"2023-09-23T15:06:15.113241+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137711+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:06:16.137711+0000","last_peered":"2023-09-23T15:06:16.137711+0000","last_clean":"2023-09-23T15:06:16.137711+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:06:16.137711+0000","last_undegraded":"2023-09-23T15:06:16.137711+0000","last_fullsized":"2023-09-23T15:06:16.137711+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134068+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:06:16.134068+0000","last_peered":"2023-09-23T15:06:16.134068+0000","last_clean":"2023-09-23T15:06:16.134068+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:06:16.134068+0000","last_undegraded":"2023-09-23T15:06:16.134068+0000","last_fullsized":"2023-09-23T15:06:16.134068+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124812+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:06:15.124812+0000","last_peered":"2023-09-23T15:06:15.124812+0000","last_clean":"2023-09-23T15:06:15.124812+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:06:15.124812+0000","last_undegraded":"2023-09-23T15:06:15.124812+0000","last_fullsized":"2023-09-23T15:06:15.124812+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131810+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:06:16.131810+0000","last_peered":"2023-09-23T15:06:16.131810+0000","last_clean":"2023-09-23T15:06:16.131810+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:06:16.131810+0000","last_undegraded":"2023-09-23T15:06:16.131810+0000","last_fullsized":"2023-09-23T15:06:16.131810+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128073+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:06:16.128073+0000","last_peered":"2023-09-23T15:06:16.128073+0000","last_clean":"2023-09-23T15:06:16.128073+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:06:16.128073+0000","last_undegraded":"2023-09-23T15:06:16.128073+0000","last_fullsized":"2023-09-23T15:06:16.128073+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121746+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:06:15.121746+0000","last_peered":"2023-09-23T15:06:15.121746+0000","last_clean":"2023-09-23T15:06:15.121746+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:06:15.121746+0000","last_undegraded":"2023-09-23T15:06:15.121746+0000","last_fullsized":"2023-09-23T15:06:15.121746+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132197+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:06:16.132197+0000","last_peered":"2023-09-23T15:06:16.132197+0000","last_clean":"2023-09-23T15:06:16.132197+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:06:16.132197+0000","last_undegraded":"2023-09-23T15:06:16.132197+0000","last_fullsized":"2023-09-23T15:06:16.132197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128449+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:06:16.128449+0000","last_peered":"2023-09-23T15:06:16.128449+0000","last_clean":"2023-09-23T15:06:16.128449+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:06:16.128449+0000","last_undegraded":"2023-09-23T15:06:16.128449+0000","last_fullsized":"2023-09-23T15:06:16.128449+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129498+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:06:16.129498+0000","last_peered":"2023-09-23T15:06:16.129498+0000","last_clean":"2023-09-23T15:06:16.129498+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:06:16.129498+0000","last_undegraded":"2023-09-23T15:06:16.129498+0000","last_fullsized":"2023-09-23T15:06:16.129498+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":37,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134796+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:06:16.134796+0000","last_peered":"2023-09-23T15:06:16.134796+0000","last_clean":"2023-09-23T15:06:16.134796+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:06:16.134796+0000","last_undegraded":"2023-09-23T15:06:16.134796+0000","last_fullsized":"2023-09-23T15:06:16.134796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113716+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:06:15.113716+0000","last_peered":"2023-09-23T15:06:15.113716+0000","last_clean":"2023-09-23T15:06:15.113716+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:06:15.113716+0000","last_undegraded":"2023-09-23T15:06:15.113716+0000","last_fullsized":"2023-09-23T15:06:15.113716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201428,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16948,"kb_used_data":5804,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728204,"statfs":{"total":95995035648,"available":95977680896,"internally_reserved":0,"allocated":5943296,"data_stored":5408381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57499999999999996,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.71199999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.76000000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":1.585,"15min":1.7529999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65900000000000003,"15min":0.624},"min":{"1min":0.48999999999999999,"5min":0.39700000000000002,"15min":0.375},"max":{"1min":0.86599999999999999,"5min":1.5289999999999999,"15min":3.4809999999999999},"last":0.81000000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54300000000000004,"15min":0.50900000000000001},"min":{"1min":0.38200000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80700000000000005,"5min":0.82899999999999996,"15min":2.226},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.53800000000000003},"min":{"1min":0.52900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57799999999999996,"15min":0.56399999999999995},"min":{"1min":0.47799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80800000000000005,"15min":0.92700000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.622,"15min":0.58299999999999996},"min":{"1min":0.49399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.88600000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.83399999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63900000000000001,"15min":0.59399999999999997},"min":{"1min":0.55000000000000004,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":1.002,"15min":1.002},"last":0.627},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.64600000000000002,"15min":0.60299999999999998},"min":{"1min":0.55000000000000004,"5min":0.307,"15min":0.307},"max":{"1min":0.878,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.66100000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463064,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8300,"kb_used_data":1700,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93736852,"statfs":{"total":95995035648,"available":95986536448,"internally_reserved":0,"allocated":1740800,"data_stored":1213950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71499999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.63},{"interface":"front","average":{"1min":0.63,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.60699999999999998,"15min":0.54100000000000004},"min":{"1min":0.42499999999999999,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":1.494,"15min":1.494},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.621,"5min":0.64400000000000002,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.372,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.54200000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54100000000000004,"15min":0.503},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":0.74399999999999999,"5min":1.675,"15min":1.675},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.59799999999999998,"15min":0.52600000000000002},"min":{"1min":0.40500000000000003,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":0.81299999999999994,"5min":1.806,"15min":1.806},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58299999999999996,"15min":0.53500000000000003},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":0.82099999999999995,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.63200000000000001,"15min":0.56299999999999994},"min":{"1min":0.46300000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":0.84099999999999997,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:18 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.58999999999999997},"min":{"1min":0.443,"5min":0.35599999999999998,"15min":0.28000000000000003},"max":{"1min":0.90300000000000002,"5min":1.843,"15min":1.843},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.69299999999999995,"15min":0.63400000000000001},"min":{"1min":0.56299999999999994,"5min":0.40999999999999998,"15min":0.373},"max":{"1min":0.88100000000000001,"5min":1.766,"15min":1.766},"last":0.71599999999999997}]}]},{"osd":1,"up_from":100,"seq":429496729624,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32844,"kb_used_data":5252,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93712308,"statfs":{"total":95995035648,"available":95961403392,"internally_reserved":0,"allocated":5378048,"data_stored":4823646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":28238923},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 15:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.26300000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.30099999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.41699999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.40300000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.51100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.437}]}]},{"osd":0,"up_from":95,"seq":408021893151,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33440,"kb_used_data":5848,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711712,"statfs":{"total":95995035648,"available":95960793088,"internally_reserved":0,"allocated":5988352,"data_stored":5415358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":28237396},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:06:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.36599999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.41599999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":0.746,"15min":0.68100000000000005},"min":{"1min":0.39500000000000002,"5min":0.375,"15min":0.375},"max":{"1min":8.1050000000000004,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.69499999999999995,"15min":0.69599999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.82399999999999995,"15min":0.76200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68200000000000005},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.55700000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.175,"5min":0.80600000000000005,"15min":0.74399999999999999},"min":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.157,"5min":8.157,"15min":8.157},"last":0.622}]}]},{"osd":2,"up_from":105,"seq":450971566098,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28680,"kb_used_data":1152,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93716472,"statfs":{"total":95995035648,"available":95965667328,"internally_reserved":0,"allocated":1179648,"data_stored":628060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":28168976},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.38900000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45700000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.63},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.70199999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.64900000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.72899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.60599999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402571,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1128,"kb_used_omap":3,"kb_used_meta":27452,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1155072,"data_stored":623615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":28111017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.33000000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50700000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66500000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.70499999999999996}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.71399999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986337,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7852,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737300,"statfs":{"total":95995035648,"available":95986995200,"internally_reserved":0,"allocated":1150976,"data_stored":623639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.54900000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.52000000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.35699999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.44},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.6619999999999999,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.42799999999999999,"5min":0.376,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.88200000000000001,"15min":0.93600000000000005},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60599999999999998,"15min":0.56499999999999995},"min":{"1min":0.436,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91400000000000003},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.51100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":0.77400000000000002,"5min":0.876,"15min":0.88400000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.52800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.26200000000000001},"max":{"1min":0.80100000000000005,"5min":0.85399999999999998,"15min":0.91700000000000004},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:06:59 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.622,"15min":0.59599999999999997},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.318},"max":{"1min":0.83899999999999997,"5min":0.92500000000000004,"15min":1.4610000000000001},"last":0.505},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.63500000000000001,"15min":0.58899999999999997},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":0.83499999999999996,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.66200000000000003,"5min":0.64600000000000002,"15min":0.59699999999999998},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.28799999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.621},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69599999999999995,"15min":0.65800000000000003},"min":{"1min":0.48799999999999999,"5min":0.47699999999999998,"15min":0.38800000000000001},"max":{"1min":0.95099999999999996,"5min":0.95099999999999996,"15min":1.0249999999999999},"last":0.56200000000000006}]}]},{"osd":5,"up_from":39,"seq":167503724700,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7456,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737696,"statfs":{"total":95995035648,"available":95987400704,"internally_reserved":0,"allocated":1138688,"data_stored":619402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64300000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.374,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.498}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.35199999999999998,"15min":0.312},"max":{"1min":0.748,"5min":0.879,"15min":1.262},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.60399999999999998,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.373,"15min":0.35499999999999998},"max":{"1min":0.70899999999999996,"5min":0.96099999999999997,"15min":1.248},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.501,"15min":0.46000000000000002},"min":{"1min":0.40100000000000002,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":0.73499999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.76900000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.50800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.74199999999999999,"15min":0.78400000000000003},"last":0.749}]},{"osd":6,"last update":"Sat Sep 23 15:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.58099999999999996,"15min":0.52800000000000002},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64800000000000002,"15min":0.59799999999999998},"min":{"1min":0.56999999999999995,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":0.80900000000000005,"5min":0.85499999999999998,"15min":0.91000000000000003},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.60799999999999998,"15min":0.59199999999999997},"min":{"1min":0.5,"5min":0.39400000000000002,"15min":0.28299999999999997},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":1.0109999999999999},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.84099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:07:06.698 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T15:07:06.698 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T15:07:06.698 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T15:07:06.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T15:07:07.061 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:07 smithi017 ceph-mon[215646]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:07 smithi017 ceph-mon[215646]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:07 smithi017 ceph-mon[222878]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:07 smithi017 ceph-mon[222878]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:07 smithi027 ceph-mon[195233]: from='client.34680 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:07 smithi027 ceph-mon[195233]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:08.293 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:08] "GET /metrics HTTP/1.1" 200 33874 "" "Prometheus/2.43.0" 2023-09-23T15:07:08.758 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:07:08.758 INFO:teuthology.orchestra.run.smithi017.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T15:07:09.425 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T15:07:09.425 INFO:teuthology.run_tasks:Running task print... 2023-09-23T15:07:09.435 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T15:07:09.435 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T15:07:09.470 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T15:07:09.470 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T15:07:09.470 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T15:07:09.472 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T15:07:09.472 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T15:07:09.472 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T15:07:09.472 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T15:07:09.472 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T15:07:09.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:07:09.832 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:10.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:09 smithi017 ceph-mon[222878]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:10.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2111198556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T15:07:10.067 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:09 smithi017 ceph-mon[215646]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:10.067 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2111198556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T15:07:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:09 smithi027 ceph-mon[195233]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2111198556' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T15:07:11.495 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:07:11.496 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:07:11.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:11 smithi017 ceph-mon[215646]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:11.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:11 smithi017 ceph-mon[222878]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:11 smithi027 ceph-mon[195233]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:12.134 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":224,"stamp":"2023-09-23T15:07:10.121454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164112,"kb_used_data":23120,"kb_used_omap":30,"kb_used_meta":140897,"kb_avail":749797104,"statfs":{"total":767960285184,"available":767792234496,"internally_reserved":0,"allocated":23674880,"data_stored":19356051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":144279128},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002564"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113418+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:06:15.113418+0000","last_peered":"2023-09-23T15:06:15.113418+0000","last_clean":"2023-09-23T15:06:15.113418+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:06:15.113418+0000","last_undegraded":"2023-09-23T15:06:15.113418+0000","last_fullsized":"2023-09-23T15:06:15.113418+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116663+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:06:16.116663+0000","last_peered":"2023-09-23T15:06:16.116663+0000","last_clean":"2023-09-23T15:06:16.116663+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:06:16.116663+0000","last_undegraded":"2023-09-23T15:06:16.116663+0000","last_fullsized":"2023-09-23T15:06:16.116663+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133197+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:06:16.133197+0000","last_peered":"2023-09-23T15:06:16.133197+0000","last_clean":"2023-09-23T15:06:16.133197+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:06:16.133197+0000","last_undegraded":"2023-09-23T15:06:16.133197+0000","last_fullsized":"2023-09-23T15:06:16.133197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":79,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129954+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:06:16.129954+0000","last_peered":"2023-09-23T15:06:16.129954+0000","last_clean":"2023-09-23T15:06:16.129954+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:06:16.129954+0000","last_undegraded":"2023-09-23T15:06:16.129954+0000","last_fullsized":"2023-09-23T15:06:16.129954+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131672+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:06:16.131672+0000","last_peered":"2023-09-23T15:06:16.131672+0000","last_clean":"2023-09-23T15:06:16.131672+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:06:16.131672+0000","last_undegraded":"2023-09-23T15:06:16.131672+0000","last_fullsized":"2023-09-23T15:06:16.131672+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.128107+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:06:15.128107+0000","last_peered":"2023-09-23T15:06:15.128107+0000","last_clean":"2023-09-23T15:06:15.128107+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:06:15.128107+0000","last_undegraded":"2023-09-23T15:06:15.128107+0000","last_fullsized":"2023-09-23T15:06:15.128107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138486+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:06:16.138486+0000","last_peered":"2023-09-23T15:06:16.138486+0000","last_clean":"2023-09-23T15:06:16.138486+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:06:16.138486+0000","last_undegraded":"2023-09-23T15:06:16.138486+0000","last_fullsized":"2023-09-23T15:06:16.138486+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.126405+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:06:15.126405+0000","last_peered":"2023-09-23T15:06:15.126405+0000","last_clean":"2023-09-23T15:06:15.126405+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:06:15.126405+0000","last_undegraded":"2023-09-23T15:06:15.126405+0000","last_fullsized":"2023-09-23T15:06:15.126405+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131256+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:06:16.131256+0000","last_peered":"2023-09-23T15:06:16.131256+0000","last_clean":"2023-09-23T15:06:16.131256+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:06:16.131256+0000","last_undegraded":"2023-09-23T15:06:16.131256+0000","last_fullsized":"2023-09-23T15:06:16.131256+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117044+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:06:16.117044+0000","last_peered":"2023-09-23T15:06:16.117044+0000","last_clean":"2023-09-23T15:06:16.117044+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:06:16.117044+0000","last_undegraded":"2023-09-23T15:06:16.117044+0000","last_fullsized":"2023-09-23T15:06:16.117044+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114344+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:06:15.114344+0000","last_peered":"2023-09-23T15:06:15.114344+0000","last_clean":"2023-09-23T15:06:15.114344+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:06:15.114344+0000","last_undegraded":"2023-09-23T15:06:15.114344+0000","last_fullsized":"2023-09-23T15:06:15.114344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117161+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:06:16.117161+0000","last_peered":"2023-09-23T15:06:16.117161+0000","last_clean":"2023-09-23T15:06:16.117161+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:06:16.117161+0000","last_undegraded":"2023-09-23T15:06:16.117161+0000","last_fullsized":"2023-09-23T15:06:16.117161+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":542,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121716+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:06:15.121716+0000","last_peered":"2023-09-23T15:06:15.121716+0000","last_clean":"2023-09-23T15:06:15.121716+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:06:15.121716+0000","last_undegraded":"2023-09-23T15:06:15.121716+0000","last_fullsized":"2023-09-23T15:06:15.121716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131999+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:06:16.131999+0000","last_peered":"2023-09-23T15:06:16.131999+0000","last_clean":"2023-09-23T15:06:16.131999+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:06:16.131999+0000","last_undegraded":"2023-09-23T15:06:16.131999+0000","last_fullsized":"2023-09-23T15:06:16.131999+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114934+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:06:15.114934+0000","last_peered":"2023-09-23T15:06:15.114934+0000","last_clean":"2023-09-23T15:06:15.114934+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:06:15.114934+0000","last_undegraded":"2023-09-23T15:06:15.114934+0000","last_fullsized":"2023-09-23T15:06:15.114934+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133258+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:06:16.133258+0000","last_peered":"2023-09-23T15:06:16.133258+0000","last_clean":"2023-09-23T15:06:16.133258+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:06:16.133258+0000","last_undegraded":"2023-09-23T15:06:16.133258+0000","last_fullsized":"2023-09-23T15:06:16.133258+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116804+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:06:16.116804+0000","last_peered":"2023-09-23T15:06:16.116804+0000","last_clean":"2023-09-23T15:06:16.116804+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:06:16.116804+0000","last_undegraded":"2023-09-23T15:06:16.116804+0000","last_fullsized":"2023-09-23T15:06:16.116804+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113833+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:06:15.113833+0000","last_peered":"2023-09-23T15:06:15.113833+0000","last_clean":"2023-09-23T15:06:15.113833+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:06:15.113833+0000","last_undegraded":"2023-09-23T15:06:15.113833+0000","last_fullsized":"2023-09-23T15:06:15.113833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":781,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121969+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:06:15.121969+0000","last_peered":"2023-09-23T15:06:15.121969+0000","last_clean":"2023-09-23T15:06:15.121969+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:06:15.121969+0000","last_undegraded":"2023-09-23T15:06:15.121969+0000","last_fullsized":"2023-09-23T15:06:15.121969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117225+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:06:16.117225+0000","last_peered":"2023-09-23T15:06:16.117225+0000","last_clean":"2023-09-23T15:06:16.117225+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:06:16.117225+0000","last_undegraded":"2023-09-23T15:06:16.117225+0000","last_fullsized":"2023-09-23T15:06:16.117225+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131654+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:06:16.131654+0000","last_peered":"2023-09-23T15:06:16.131654+0000","last_clean":"2023-09-23T15:06:16.131654+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:06:16.131654+0000","last_undegraded":"2023-09-23T15:06:16.131654+0000","last_fullsized":"2023-09-23T15:06:16.131654+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":316,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119475+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:06:15.119475+0000","last_peered":"2023-09-23T15:06:15.119475+0000","last_clean":"2023-09-23T15:06:15.119475+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:06:15.119475+0000","last_undegraded":"2023-09-23T15:06:15.119475+0000","last_fullsized":"2023-09-23T15:06:15.119475+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121344+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:06:15.121344+0000","last_peered":"2023-09-23T15:06:15.121344+0000","last_clean":"2023-09-23T15:06:15.121344+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:06:15.121344+0000","last_undegraded":"2023-09-23T15:06:15.121344+0000","last_fullsized":"2023-09-23T15:06:15.121344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134611+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:06:16.134611+0000","last_peered":"2023-09-23T15:06:16.134611+0000","last_clean":"2023-09-23T15:06:16.134611+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:06:16.134611+0000","last_undegraded":"2023-09-23T15:06:16.134611+0000","last_fullsized":"2023-09-23T15:06:16.134611+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":451,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119438+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:06:15.119438+0000","last_peered":"2023-09-23T15:06:15.119438+0000","last_clean":"2023-09-23T15:06:15.119438+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:06:15.119438+0000","last_undegraded":"2023-09-23T15:06:15.119438+0000","last_fullsized":"2023-09-23T15:06:15.119438+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":356,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125051+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:06:15.125051+0000","last_peered":"2023-09-23T15:06:15.125051+0000","last_clean":"2023-09-23T15:06:15.125051+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:06:15.125051+0000","last_undegraded":"2023-09-23T15:06:15.125051+0000","last_fullsized":"2023-09-23T15:06:15.125051+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114264+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:06:15.114264+0000","last_peered":"2023-09-23T15:06:15.114264+0000","last_clean":"2023-09-23T15:06:15.114264+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:06:15.114264+0000","last_undegraded":"2023-09-23T15:06:15.114264+0000","last_fullsized":"2023-09-23T15:06:15.114264+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113478+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:06:15.113478+0000","last_peered":"2023-09-23T15:06:15.113478+0000","last_clean":"2023-09-23T15:06:15.113478+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:06:15.113478+0000","last_undegraded":"2023-09-23T15:06:15.113478+0000","last_fullsized":"2023-09-23T15:06:15.113478+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":719,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129635+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:06:16.129635+0000","last_peered":"2023-09-23T15:06:16.129635+0000","last_clean":"2023-09-23T15:06:16.129635+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:06:16.129635+0000","last_undegraded":"2023-09-23T15:06:16.129635+0000","last_fullsized":"2023-09-23T15:06:16.129635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129423+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:06:16.129423+0000","last_peered":"2023-09-23T15:06:16.129423+0000","last_clean":"2023-09-23T15:06:16.129423+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:06:16.129423+0000","last_undegraded":"2023-09-23T15:06:16.129423+0000","last_fullsized":"2023-09-23T15:06:16.129423+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113741+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:06:15.113741+0000","last_peered":"2023-09-23T15:06:15.113741+0000","last_clean":"2023-09-23T15:06:15.113741+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:06:15.113741+0000","last_undegraded":"2023-09-23T15:06:15.113741+0000","last_fullsized":"2023-09-23T15:06:15.113741+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":612,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125189+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:06:15.125189+0000","last_peered":"2023-09-23T15:06:15.125189+0000","last_clean":"2023-09-23T15:06:15.125189+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:06:15.125189+0000","last_undegraded":"2023-09-23T15:06:15.125189+0000","last_fullsized":"2023-09-23T15:06:15.125189+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.110328+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:06:15.110328+0000","last_peered":"2023-09-23T15:06:15.110328+0000","last_clean":"2023-09-23T15:06:15.110328+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:06:15.110328+0000","last_undegraded":"2023-09-23T15:06:15.110328+0000","last_fullsized":"2023-09-23T15:06:15.110328+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113738+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:06:15.113738+0000","last_peered":"2023-09-23T15:06:15.113738+0000","last_clean":"2023-09-23T15:06:15.113738+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:06:15.113738+0000","last_undegraded":"2023-09-23T15:06:15.113738+0000","last_fullsized":"2023-09-23T15:06:15.113738+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128245+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:06:16.128245+0000","last_peered":"2023-09-23T15:06:16.128245+0000","last_clean":"2023-09-23T15:06:16.128245+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:06:16.128245+0000","last_undegraded":"2023-09-23T15:06:16.128245+0000","last_fullsized":"2023-09-23T15:06:16.128245+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132588+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:06:16.132588+0000","last_peered":"2023-09-23T15:06:16.132588+0000","last_clean":"2023-09-23T15:06:16.132588+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:06:16.132588+0000","last_undegraded":"2023-09-23T15:06:16.132588+0000","last_fullsized":"2023-09-23T15:06:16.132588+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114017+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:06:15.114017+0000","last_peered":"2023-09-23T15:06:15.114017+0000","last_clean":"2023-09-23T15:06:15.114017+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:06:15.114017+0000","last_undegraded":"2023-09-23T15:06:15.114017+0000","last_fullsized":"2023-09-23T15:06:15.114017+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115143+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:06:15.115143+0000","last_peered":"2023-09-23T15:06:15.115143+0000","last_clean":"2023-09-23T15:06:15.115143+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:06:15.115143+0000","last_undegraded":"2023-09-23T15:06:15.115143+0000","last_fullsized":"2023-09-23T15:06:15.115143+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115699+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:06:15.115699+0000","last_peered":"2023-09-23T15:06:15.115699+0000","last_clean":"2023-09-23T15:06:15.115699+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:06:15.115699+0000","last_undegraded":"2023-09-23T15:06:15.115699+0000","last_fullsized":"2023-09-23T15:06:15.115699+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120271+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:06:15.120271+0000","last_peered":"2023-09-23T15:06:15.120271+0000","last_clean":"2023-09-23T15:06:15.120271+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:06:15.120271+0000","last_undegraded":"2023-09-23T15:06:15.120271+0000","last_fullsized":"2023-09-23T15:06:15.120271+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137694+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:06:16.137694+0000","last_peered":"2023-09-23T15:06:16.137694+0000","last_clean":"2023-09-23T15:06:16.137694+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:06:16.137694+0000","last_undegraded":"2023-09-23T15:06:16.137694+0000","last_fullsized":"2023-09-23T15:06:16.137694+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132545+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:06:16.132545+0000","last_peered":"2023-09-23T15:06:16.132545+0000","last_clean":"2023-09-23T15:06:16.132545+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:06:16.132545+0000","last_undegraded":"2023-09-23T15:06:16.132545+0000","last_fullsized":"2023-09-23T15:06:16.132545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116963+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:06:16.116963+0000","last_peered":"2023-09-23T15:06:16.116963+0000","last_clean":"2023-09-23T15:06:16.116963+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:06:16.116963+0000","last_undegraded":"2023-09-23T15:06:16.116963+0000","last_fullsized":"2023-09-23T15:06:16.116963+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120111+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:06:15.120111+0000","last_peered":"2023-09-23T15:06:15.120111+0000","last_clean":"2023-09-23T15:06:15.120111+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:06:15.120111+0000","last_undegraded":"2023-09-23T15:06:15.120111+0000","last_fullsized":"2023-09-23T15:06:15.120111+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.122093+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:06:15.122093+0000","last_peered":"2023-09-23T15:06:15.122093+0000","last_clean":"2023-09-23T15:06:15.122093+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:06:15.122093+0000","last_undegraded":"2023-09-23T15:06:15.122093+0000","last_fullsized":"2023-09-23T15:06:15.122093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131518+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:06:16.131518+0000","last_peered":"2023-09-23T15:06:16.131518+0000","last_clean":"2023-09-23T15:06:16.131518+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:06:16.131518+0000","last_undegraded":"2023-09-23T15:06:16.131518+0000","last_fullsized":"2023-09-23T15:06:16.131518+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118294+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:06:15.118294+0000","last_peered":"2023-09-23T15:06:15.118294+0000","last_clean":"2023-09-23T15:06:15.118294+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:06:15.118294+0000","last_undegraded":"2023-09-23T15:06:15.118294+0000","last_fullsized":"2023-09-23T15:06:15.118294+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124626+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:06:15.124626+0000","last_peered":"2023-09-23T15:06:15.124626+0000","last_clean":"2023-09-23T15:06:15.124626+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:06:15.124626+0000","last_undegraded":"2023-09-23T15:06:15.124626+0000","last_fullsized":"2023-09-23T15:06:15.124626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119275+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:06:15.119275+0000","last_peered":"2023-09-23T15:06:15.119275+0000","last_clean":"2023-09-23T15:06:15.119275+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:06:15.119275+0000","last_undegraded":"2023-09-23T15:06:15.119275+0000","last_fullsized":"2023-09-23T15:06:15.119275+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131473+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:06:16.131473+0000","last_peered":"2023-09-23T15:06:16.131473+0000","last_clean":"2023-09-23T15:06:16.131473+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:06:16.131473+0000","last_undegraded":"2023-09-23T15:06:16.131473+0000","last_fullsized":"2023-09-23T15:06:16.131473+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118992+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:06:15.118992+0000","last_peered":"2023-09-23T15:06:15.118992+0000","last_clean":"2023-09-23T15:06:15.118992+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:06:15.118992+0000","last_undegraded":"2023-09-23T15:06:15.118992+0000","last_fullsized":"2023-09-23T15:06:15.118992+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128037+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:06:16.128037+0000","last_peered":"2023-09-23T15:06:16.128037+0000","last_clean":"2023-09-23T15:06:16.128037+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:06:16.128037+0000","last_undegraded":"2023-09-23T15:06:16.128037+0000","last_fullsized":"2023-09-23T15:06:16.128037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119363+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:06:15.119363+0000","last_peered":"2023-09-23T15:06:15.119363+0000","last_clean":"2023-09-23T15:06:15.119363+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:06:15.119363+0000","last_undegraded":"2023-09-23T15:06:15.119363+0000","last_fullsized":"2023-09-23T15:06:15.119363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138039+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:06:16.138039+0000","last_peered":"2023-09-23T15:06:16.138039+0000","last_clean":"2023-09-23T15:06:16.138039+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:06:16.138039+0000","last_undegraded":"2023-09-23T15:06:16.138039+0000","last_fullsized":"2023-09-23T15:06:16.138039+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.127131+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:06:16.127131+0000","last_peered":"2023-09-23T15:06:16.127131+0000","last_clean":"2023-09-23T15:06:16.127131+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:06:16.127131+0000","last_undegraded":"2023-09-23T15:06:16.127131+0000","last_fullsized":"2023-09-23T15:06:16.127131+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.135551+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:06:16.135551+0000","last_peered":"2023-09-23T15:06:16.135551+0000","last_clean":"2023-09-23T15:06:16.135551+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:06:16.135551+0000","last_undegraded":"2023-09-23T15:06:16.135551+0000","last_fullsized":"2023-09-23T15:06:16.135551+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118610+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:06:15.118610+0000","last_peered":"2023-09-23T15:06:15.118610+0000","last_clean":"2023-09-23T15:06:15.118610+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:06:15.118610+0000","last_undegraded":"2023-09-23T15:06:15.118610+0000","last_fullsized":"2023-09-23T15:06:15.118610+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113896+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:06:15.113896+0000","last_peered":"2023-09-23T15:06:15.113896+0000","last_clean":"2023-09-23T15:06:15.113896+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:06:15.113896+0000","last_undegraded":"2023-09-23T15:06:15.113896+0000","last_fullsized":"2023-09-23T15:06:15.113896+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113914+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:06:15.113914+0000","last_peered":"2023-09-23T15:06:15.113914+0000","last_clean":"2023-09-23T15:06:15.113914+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:06:15.113914+0000","last_undegraded":"2023-09-23T15:06:15.113914+0000","last_fullsized":"2023-09-23T15:06:15.113914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113241+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:06:15.113241+0000","last_peered":"2023-09-23T15:06:15.113241+0000","last_clean":"2023-09-23T15:06:15.113241+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:06:15.113241+0000","last_undegraded":"2023-09-23T15:06:15.113241+0000","last_fullsized":"2023-09-23T15:06:15.113241+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137711+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:06:16.137711+0000","last_peered":"2023-09-23T15:06:16.137711+0000","last_clean":"2023-09-23T15:06:16.137711+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:06:16.137711+0000","last_undegraded":"2023-09-23T15:06:16.137711+0000","last_fullsized":"2023-09-23T15:06:16.137711+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134068+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:06:16.134068+0000","last_peered":"2023-09-23T15:06:16.134068+0000","last_clean":"2023-09-23T15:06:16.134068+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:06:16.134068+0000","last_undegraded":"2023-09-23T15:06:16.134068+0000","last_fullsized":"2023-09-23T15:06:16.134068+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124812+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:06:15.124812+0000","last_peered":"2023-09-23T15:06:15.124812+0000","last_clean":"2023-09-23T15:06:15.124812+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:06:15.124812+0000","last_undegraded":"2023-09-23T15:06:15.124812+0000","last_fullsized":"2023-09-23T15:06:15.124812+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131810+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:06:16.131810+0000","last_peered":"2023-09-23T15:06:16.131810+0000","last_clean":"2023-09-23T15:06:16.131810+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:06:16.131810+0000","last_undegraded":"2023-09-23T15:06:16.131810+0000","last_fullsized":"2023-09-23T15:06:16.131810+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128073+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:06:16.128073+0000","last_peered":"2023-09-23T15:06:16.128073+0000","last_clean":"2023-09-23T15:06:16.128073+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:06:16.128073+0000","last_undegraded":"2023-09-23T15:06:16.128073+0000","last_fullsized":"2023-09-23T15:06:16.128073+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121746+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:06:15.121746+0000","last_peered":"2023-09-23T15:06:15.121746+0000","last_clean":"2023-09-23T15:06:15.121746+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:06:15.121746+0000","last_undegraded":"2023-09-23T15:06:15.121746+0000","last_fullsized":"2023-09-23T15:06:15.121746+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132197+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:06:16.132197+0000","last_peered":"2023-09-23T15:06:16.132197+0000","last_clean":"2023-09-23T15:06:16.132197+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:06:16.132197+0000","last_undegraded":"2023-09-23T15:06:16.132197+0000","last_fullsized":"2023-09-23T15:06:16.132197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128449+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:06:16.128449+0000","last_peered":"2023-09-23T15:06:16.128449+0000","last_clean":"2023-09-23T15:06:16.128449+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:06:16.128449+0000","last_undegraded":"2023-09-23T15:06:16.128449+0000","last_fullsized":"2023-09-23T15:06:16.128449+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129498+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:06:16.129498+0000","last_peered":"2023-09-23T15:06:16.129498+0000","last_clean":"2023-09-23T15:06:16.129498+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:06:16.129498+0000","last_undegraded":"2023-09-23T15:06:16.129498+0000","last_fullsized":"2023-09-23T15:06:16.129498+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":37,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134796+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:06:16.134796+0000","last_peered":"2023-09-23T15:06:16.134796+0000","last_clean":"2023-09-23T15:06:16.134796+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:06:16.134796+0000","last_undegraded":"2023-09-23T15:06:16.134796+0000","last_fullsized":"2023-09-23T15:06:16.134796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113716+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:06:15.113716+0000","last_peered":"2023-09-23T15:06:15.113716+0000","last_clean":"2023-09-23T15:06:15.113716+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:06:15.113716+0000","last_undegraded":"2023-09-23T15:06:15.113716+0000","last_fullsized":"2023-09-23T15:06:15.113716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201429,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16948,"kb_used_data":5804,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728204,"statfs":{"total":95995035648,"available":95977680896,"internally_reserved":0,"allocated":5943296,"data_stored":5408381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57499999999999996,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.745},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":1.585,"15min":1.7529999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65900000000000003,"15min":0.624},"min":{"1min":0.48999999999999999,"5min":0.39700000000000002,"15min":0.375},"max":{"1min":0.86599999999999999,"5min":1.5289999999999999,"15min":3.4809999999999999},"last":0.78600000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54300000000000004,"15min":0.50900000000000001},"min":{"1min":0.38200000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80700000000000005,"5min":0.82899999999999996,"15min":2.226},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.53800000000000003},"min":{"1min":0.52900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57799999999999996,"15min":0.56399999999999995},"min":{"1min":0.47799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80800000000000005,"15min":0.92700000000000005},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.622,"15min":0.58299999999999996},"min":{"1min":0.49399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.88600000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.69399999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63900000000000001,"15min":0.59399999999999997},"min":{"1min":0.55000000000000004,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":1.002,"15min":1.002},"last":0.62},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.64600000000000002,"15min":0.60299999999999998},"min":{"1min":0.55000000000000004,"5min":0.307,"15min":0.307},"max":{"1min":0.878,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.67600000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463065,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8300,"kb_used_data":1700,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93736852,"statfs":{"total":95995035648,"available":95986536448,"internally_reserved":0,"allocated":1740800,"data_stored":1213950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71499999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.69599999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.503}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.627},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.60699999999999998,"15min":0.54100000000000004},"min":{"1min":0.42499999999999999,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":1.494,"15min":1.494},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.64400000000000002,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.372,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54100000000000004,"15min":0.503},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":0.74399999999999999,"5min":1.675,"15min":1.675},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.59799999999999998,"15min":0.52600000000000002},"min":{"1min":0.40500000000000003,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":0.81299999999999994,"5min":1.806,"15min":1.806},"last":0.73499999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58299999999999996,"15min":0.53500000000000003},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":0.82099999999999995,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.63200000000000001,"15min":0.56299999999999994},"min":{"1min":0.46300000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":0.84099999999999997,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:06:18 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.58999999999999997},"min":{"1min":0.443,"5min":0.35599999999999998,"15min":0.28000000000000003},"max":{"1min":0.90300000000000002,"5min":1.843,"15min":1.843},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.69299999999999995,"15min":0.63400000000000001},"min":{"1min":0.56299999999999994,"5min":0.40999999999999998,"15min":0.373},"max":{"1min":0.88100000000000001,"5min":1.766,"15min":1.766},"last":0.72099999999999997}]}]},{"osd":1,"up_from":100,"seq":429496729625,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32844,"kb_used_data":5252,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93712308,"statfs":{"total":95995035648,"available":95961403392,"internally_reserved":0,"allocated":5378048,"data_stored":4823646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":28238923},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.56999999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.495},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.52100000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.83899999999999997},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.98599999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.96099999999999997},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.871}]},{"osd":6,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.90700000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":1.022},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":1.0009999999999999}]}]},{"osd":0,"up_from":95,"seq":408021893152,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33440,"kb_used_data":5848,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711712,"statfs":{"total":95995035648,"available":95960793088,"internally_reserved":0,"allocated":5988352,"data_stored":5415358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":28237396},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:06:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.255}]},{"osd":2,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":0.746,"15min":0.68100000000000005},"min":{"1min":0.39500000000000002,"5min":0.375,"15min":0.375},"max":{"1min":8.1050000000000004,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.69499999999999995,"15min":0.69599999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.82399999999999995,"15min":0.76200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68200000000000005},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.67500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.175,"5min":0.80600000000000005,"15min":0.74399999999999999},"min":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.157,"5min":8.157,"15min":8.157},"last":0.86599999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566099,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28680,"kb_used_data":1152,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93716472,"statfs":{"total":95995035648,"available":95965667328,"internally_reserved":0,"allocated":1179648,"data_stored":628060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":28168976},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.31}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.62},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.49399999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402573,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1128,"kb_used_omap":3,"kb_used_meta":27452,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1155072,"data_stored":623615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":28111017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.317}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986338,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7852,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737300,"statfs":{"total":95995035648,"available":95986995200,"internally_reserved":0,"allocated":1150976,"data_stored":623639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.54900000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.6619999999999999,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.42799999999999999,"5min":0.376,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.88200000000000001,"15min":0.93600000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60599999999999998,"15min":0.56499999999999995},"min":{"1min":0.436,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91400000000000003},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.51100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":0.77400000000000002,"5min":0.876,"15min":0.88400000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.52800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.26200000000000001},"max":{"1min":0.80100000000000005,"5min":0.85399999999999998,"15min":0.91700000000000004},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:06:59 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.622,"15min":0.59599999999999997},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.318},"max":{"1min":0.83899999999999997,"5min":0.92500000000000004,"15min":1.4610000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.63500000000000001,"15min":0.58899999999999997},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":0.83499999999999996,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.66000000000000003,"15min":0.60599999999999998},"min":{"1min":0.47799999999999998,"5min":0.38600000000000001,"15min":0.28799999999999998},"max":{"1min":0.89500000000000002,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69899999999999995,"15min":0.66300000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.38800000000000001},"max":{"1min":0.876,"5min":0.95099999999999996,"15min":1.0249999999999999},"last":0.63600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724701,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7456,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737696,"statfs":{"total":95995035648,"available":95987400704,"internally_reserved":0,"allocated":1138688,"data_stored":619402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.374,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 15:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.35199999999999998,"15min":0.312},"max":{"1min":0.748,"5min":0.879,"15min":1.262},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.60399999999999998,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.373,"15min":0.35499999999999998},"max":{"1min":0.70899999999999996,"5min":0.96099999999999997,"15min":1.248},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.501,"15min":0.46000000000000002},"min":{"1min":0.40100000000000002,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":0.73499999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.626},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.50800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.74199999999999999,"15min":0.78400000000000003},"last":0.376}]},{"osd":6,"last update":"Sat Sep 23 15:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.58099999999999996,"15min":0.52800000000000002},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64800000000000002,"15min":0.59799999999999998},"min":{"1min":0.56999999999999995,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":0.80900000000000005,"5min":0.85499999999999998,"15min":0.91000000000000003},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.60799999999999998,"15min":0.59199999999999997},"min":{"1min":0.5,"5min":0.39400000000000002,"15min":0.28299999999999997},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":1.0109999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.67000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:07:12.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:07:12.497 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:12.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:12 smithi017 ceph-mon[215646]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:12 smithi017 ceph-mon[222878]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:12 smithi027 ceph-mon[195233]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:07:14.085 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:07:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:13 smithi027 ceph-mon[195233]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:14.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:13 smithi017 ceph-mon[215646]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:14.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:14.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:13 smithi017 ceph-mon[222878]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:14.111 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:14.689 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":225,"stamp":"2023-09-23T15:07:12.121959+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":164112,"kb_used_data":23120,"kb_used_omap":30,"kb_used_meta":140897,"kb_avail":749797104,"statfs":{"total":767960285184,"available":767792234496,"internally_reserved":0,"allocated":23674880,"data_stored":19356051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":144279128},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002718"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":79,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113418+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:06:15.113418+0000","last_peered":"2023-09-23T15:06:15.113418+0000","last_clean":"2023-09-23T15:06:15.113418+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:06:15.113418+0000","last_undegraded":"2023-09-23T15:06:15.113418+0000","last_fullsized":"2023-09-23T15:06:15.113418+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116663+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:06:16.116663+0000","last_peered":"2023-09-23T15:06:16.116663+0000","last_clean":"2023-09-23T15:06:16.116663+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:06:16.116663+0000","last_undegraded":"2023-09-23T15:06:16.116663+0000","last_fullsized":"2023-09-23T15:06:16.116663+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133197+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:06:16.133197+0000","last_peered":"2023-09-23T15:06:16.133197+0000","last_clean":"2023-09-23T15:06:16.133197+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:06:16.133197+0000","last_undegraded":"2023-09-23T15:06:16.133197+0000","last_fullsized":"2023-09-23T15:06:16.133197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":79,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129954+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:06:16.129954+0000","last_peered":"2023-09-23T15:06:16.129954+0000","last_clean":"2023-09-23T15:06:16.129954+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:06:16.129954+0000","last_undegraded":"2023-09-23T15:06:16.129954+0000","last_fullsized":"2023-09-23T15:06:16.129954+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131672+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:06:16.131672+0000","last_peered":"2023-09-23T15:06:16.131672+0000","last_clean":"2023-09-23T15:06:16.131672+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:06:16.131672+0000","last_undegraded":"2023-09-23T15:06:16.131672+0000","last_fullsized":"2023-09-23T15:06:16.131672+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":55,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.128107+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:06:15.128107+0000","last_peered":"2023-09-23T15:06:15.128107+0000","last_clean":"2023-09-23T15:06:15.128107+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:06:15.128107+0000","last_undegraded":"2023-09-23T15:06:15.128107+0000","last_fullsized":"2023-09-23T15:06:15.128107+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138486+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:06:16.138486+0000","last_peered":"2023-09-23T15:06:16.138486+0000","last_clean":"2023-09-23T15:06:16.138486+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:06:16.138486+0000","last_undegraded":"2023-09-23T15:06:16.138486+0000","last_fullsized":"2023-09-23T15:06:16.138486+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.126405+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:06:15.126405+0000","last_peered":"2023-09-23T15:06:15.126405+0000","last_clean":"2023-09-23T15:06:15.126405+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:06:15.126405+0000","last_undegraded":"2023-09-23T15:06:15.126405+0000","last_fullsized":"2023-09-23T15:06:15.126405+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131256+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:06:16.131256+0000","last_peered":"2023-09-23T15:06:16.131256+0000","last_clean":"2023-09-23T15:06:16.131256+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:06:16.131256+0000","last_undegraded":"2023-09-23T15:06:16.131256+0000","last_fullsized":"2023-09-23T15:06:16.131256+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":41,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117044+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:06:16.117044+0000","last_peered":"2023-09-23T15:06:16.117044+0000","last_clean":"2023-09-23T15:06:16.117044+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:06:16.117044+0000","last_undegraded":"2023-09-23T15:06:16.117044+0000","last_fullsized":"2023-09-23T15:06:16.117044+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114344+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:06:15.114344+0000","last_peered":"2023-09-23T15:06:15.114344+0000","last_clean":"2023-09-23T15:06:15.114344+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:06:15.114344+0000","last_undegraded":"2023-09-23T15:06:15.114344+0000","last_fullsized":"2023-09-23T15:06:15.114344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":56,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117161+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:06:16.117161+0000","last_peered":"2023-09-23T15:06:16.117161+0000","last_clean":"2023-09-23T15:06:16.117161+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:06:16.117161+0000","last_undegraded":"2023-09-23T15:06:16.117161+0000","last_fullsized":"2023-09-23T15:06:16.117161+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":542,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121716+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:06:15.121716+0000","last_peered":"2023-09-23T15:06:15.121716+0000","last_clean":"2023-09-23T15:06:15.121716+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:06:15.121716+0000","last_undegraded":"2023-09-23T15:06:15.121716+0000","last_fullsized":"2023-09-23T15:06:15.121716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131999+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:06:16.131999+0000","last_peered":"2023-09-23T15:06:16.131999+0000","last_clean":"2023-09-23T15:06:16.131999+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:06:16.131999+0000","last_undegraded":"2023-09-23T15:06:16.131999+0000","last_fullsized":"2023-09-23T15:06:16.131999+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":640,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114934+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:06:15.114934+0000","last_peered":"2023-09-23T15:06:15.114934+0000","last_clean":"2023-09-23T15:06:15.114934+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:06:15.114934+0000","last_undegraded":"2023-09-23T15:06:15.114934+0000","last_fullsized":"2023-09-23T15:06:15.114934+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.133258+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:06:16.133258+0000","last_peered":"2023-09-23T15:06:16.133258+0000","last_clean":"2023-09-23T15:06:16.133258+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:06:16.133258+0000","last_undegraded":"2023-09-23T15:06:16.133258+0000","last_fullsized":"2023-09-23T15:06:16.133258+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":38,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116804+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:06:16.116804+0000","last_peered":"2023-09-23T15:06:16.116804+0000","last_clean":"2023-09-23T15:06:16.116804+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:06:16.116804+0000","last_undegraded":"2023-09-23T15:06:16.116804+0000","last_fullsized":"2023-09-23T15:06:16.116804+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113833+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:06:15.113833+0000","last_peered":"2023-09-23T15:06:15.113833+0000","last_clean":"2023-09-23T15:06:15.113833+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:06:15.113833+0000","last_undegraded":"2023-09-23T15:06:15.113833+0000","last_fullsized":"2023-09-23T15:06:15.113833+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":781,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121969+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:06:15.121969+0000","last_peered":"2023-09-23T15:06:15.121969+0000","last_clean":"2023-09-23T15:06:15.121969+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:06:15.121969+0000","last_undegraded":"2023-09-23T15:06:15.121969+0000","last_fullsized":"2023-09-23T15:06:15.121969+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":68,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.117225+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:06:16.117225+0000","last_peered":"2023-09-23T15:06:16.117225+0000","last_clean":"2023-09-23T15:06:16.117225+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:06:16.117225+0000","last_undegraded":"2023-09-23T15:06:16.117225+0000","last_fullsized":"2023-09-23T15:06:16.117225+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131654+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:06:16.131654+0000","last_peered":"2023-09-23T15:06:16.131654+0000","last_clean":"2023-09-23T15:06:16.131654+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:06:16.131654+0000","last_undegraded":"2023-09-23T15:06:16.131654+0000","last_fullsized":"2023-09-23T15:06:16.131654+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":316,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119475+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:06:15.119475+0000","last_peered":"2023-09-23T15:06:15.119475+0000","last_clean":"2023-09-23T15:06:15.119475+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:06:15.119475+0000","last_undegraded":"2023-09-23T15:06:15.119475+0000","last_fullsized":"2023-09-23T15:06:15.119475+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121344+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:06:15.121344+0000","last_peered":"2023-09-23T15:06:15.121344+0000","last_clean":"2023-09-23T15:06:15.121344+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:06:15.121344+0000","last_undegraded":"2023-09-23T15:06:15.121344+0000","last_fullsized":"2023-09-23T15:06:15.121344+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134611+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:06:16.134611+0000","last_peered":"2023-09-23T15:06:16.134611+0000","last_clean":"2023-09-23T15:06:16.134611+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:06:16.134611+0000","last_undegraded":"2023-09-23T15:06:16.134611+0000","last_fullsized":"2023-09-23T15:06:16.134611+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":451,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119438+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:06:15.119438+0000","last_peered":"2023-09-23T15:06:15.119438+0000","last_clean":"2023-09-23T15:06:15.119438+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:06:15.119438+0000","last_undegraded":"2023-09-23T15:06:15.119438+0000","last_fullsized":"2023-09-23T15:06:15.119438+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":356,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125051+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:06:15.125051+0000","last_peered":"2023-09-23T15:06:15.125051+0000","last_clean":"2023-09-23T15:06:15.125051+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:06:15.125051+0000","last_undegraded":"2023-09-23T15:06:15.125051+0000","last_fullsized":"2023-09-23T15:06:15.125051+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":76,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114264+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:06:15.114264+0000","last_peered":"2023-09-23T15:06:15.114264+0000","last_clean":"2023-09-23T15:06:15.114264+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:06:15.114264+0000","last_undegraded":"2023-09-23T15:06:15.114264+0000","last_fullsized":"2023-09-23T15:06:15.114264+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":48,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.120254+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:06:16.120254+0000","last_peered":"2023-09-23T15:06:16.120254+0000","last_clean":"2023-09-23T15:06:16.120254+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:06:16.120254+0000","last_undegraded":"2023-09-23T15:06:16.120254+0000","last_fullsized":"2023-09-23T15:06:16.120254+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":719,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129635+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:06:16.129635+0000","last_peered":"2023-09-23T15:06:16.129635+0000","last_clean":"2023-09-23T15:06:16.129635+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:06:16.129635+0000","last_undegraded":"2023-09-23T15:06:16.129635+0000","last_fullsized":"2023-09-23T15:06:16.129635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129423+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:06:16.129423+0000","last_peered":"2023-09-23T15:06:16.129423+0000","last_clean":"2023-09-23T15:06:16.129423+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:06:16.129423+0000","last_undegraded":"2023-09-23T15:06:16.129423+0000","last_fullsized":"2023-09-23T15:06:16.129423+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113741+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:06:15.113741+0000","last_peered":"2023-09-23T15:06:15.113741+0000","last_clean":"2023-09-23T15:06:15.113741+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:06:15.113741+0000","last_undegraded":"2023-09-23T15:06:15.113741+0000","last_fullsized":"2023-09-23T15:06:15.113741+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":612,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.125189+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:06:15.125189+0000","last_peered":"2023-09-23T15:06:15.125189+0000","last_clean":"2023-09-23T15:06:15.125189+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:06:15.125189+0000","last_undegraded":"2023-09-23T15:06:15.125189+0000","last_fullsized":"2023-09-23T15:06:15.125189+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":28,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.110328+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:06:15.110328+0000","last_peered":"2023-09-23T15:06:15.110328+0000","last_clean":"2023-09-23T15:06:15.110328+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:06:15.110328+0000","last_undegraded":"2023-09-23T15:06:15.110328+0000","last_fullsized":"2023-09-23T15:06:15.110328+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":87,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":358,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113738+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:06:15.113738+0000","last_peered":"2023-09-23T15:06:15.113738+0000","last_clean":"2023-09-23T15:06:15.113738+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:06:15.113738+0000","last_undegraded":"2023-09-23T15:06:15.113738+0000","last_fullsized":"2023-09-23T15:06:15.113738+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128245+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:06:16.128245+0000","last_peered":"2023-09-23T15:06:16.128245+0000","last_clean":"2023-09-23T15:06:16.128245+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:06:16.128245+0000","last_undegraded":"2023-09-23T15:06:16.128245+0000","last_fullsized":"2023-09-23T15:06:16.128245+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132588+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:06:16.132588+0000","last_peered":"2023-09-23T15:06:16.132588+0000","last_clean":"2023-09-23T15:06:16.132588+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:06:16.132588+0000","last_undegraded":"2023-09-23T15:06:16.132588+0000","last_fullsized":"2023-09-23T15:06:16.132588+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.114017+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:06:15.114017+0000","last_peered":"2023-09-23T15:06:15.114017+0000","last_clean":"2023-09-23T15:06:15.114017+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:06:15.114017+0000","last_undegraded":"2023-09-23T15:06:15.114017+0000","last_fullsized":"2023-09-23T15:06:15.114017+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115143+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:06:15.115143+0000","last_peered":"2023-09-23T15:06:15.115143+0000","last_clean":"2023-09-23T15:06:15.115143+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:06:15.115143+0000","last_undegraded":"2023-09-23T15:06:15.115143+0000","last_fullsized":"2023-09-23T15:06:15.115143+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":86,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.115699+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:06:15.115699+0000","last_peered":"2023-09-23T15:06:15.115699+0000","last_clean":"2023-09-23T15:06:15.115699+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:06:15.115699+0000","last_undegraded":"2023-09-23T15:06:15.115699+0000","last_fullsized":"2023-09-23T15:06:15.115699+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120271+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:06:15.120271+0000","last_peered":"2023-09-23T15:06:15.120271+0000","last_clean":"2023-09-23T15:06:15.120271+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:06:15.120271+0000","last_undegraded":"2023-09-23T15:06:15.120271+0000","last_fullsized":"2023-09-23T15:06:15.120271+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137694+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:06:16.137694+0000","last_peered":"2023-09-23T15:06:16.137694+0000","last_clean":"2023-09-23T15:06:16.137694+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:06:16.137694+0000","last_undegraded":"2023-09-23T15:06:16.137694+0000","last_fullsized":"2023-09-23T15:06:16.137694+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":71,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132545+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:06:16.132545+0000","last_peered":"2023-09-23T15:06:16.132545+0000","last_clean":"2023-09-23T15:06:16.132545+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:06:16.132545+0000","last_undegraded":"2023-09-23T15:06:16.132545+0000","last_fullsized":"2023-09-23T15:06:16.132545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.116963+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:06:16.116963+0000","last_peered":"2023-09-23T15:06:16.116963+0000","last_clean":"2023-09-23T15:06:16.116963+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:06:16.116963+0000","last_undegraded":"2023-09-23T15:06:16.116963+0000","last_fullsized":"2023-09-23T15:06:16.116963+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":70,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137756+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:06:16.137756+0000","last_peered":"2023-09-23T15:06:16.137756+0000","last_clean":"2023-09-23T15:06:16.137756+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:06:16.137756+0000","last_undegraded":"2023-09-23T15:06:16.137756+0000","last_fullsized":"2023-09-23T15:06:16.137756+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.120111+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:06:15.120111+0000","last_peered":"2023-09-23T15:06:15.120111+0000","last_clean":"2023-09-23T15:06:15.120111+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:06:15.120111+0000","last_undegraded":"2023-09-23T15:06:15.120111+0000","last_fullsized":"2023-09-23T15:06:15.120111+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":54,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.122093+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:06:15.122093+0000","last_peered":"2023-09-23T15:06:15.122093+0000","last_clean":"2023-09-23T15:06:15.122093+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:06:15.122093+0000","last_undegraded":"2023-09-23T15:06:15.122093+0000","last_fullsized":"2023-09-23T15:06:15.122093+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131518+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:06:16.131518+0000","last_peered":"2023-09-23T15:06:16.131518+0000","last_clean":"2023-09-23T15:06:16.131518+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:06:16.131518+0000","last_undegraded":"2023-09-23T15:06:16.131518+0000","last_fullsized":"2023-09-23T15:06:16.131518+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118294+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:06:15.118294+0000","last_peered":"2023-09-23T15:06:15.118294+0000","last_clean":"2023-09-23T15:06:15.118294+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:06:15.118294+0000","last_undegraded":"2023-09-23T15:06:15.118294+0000","last_fullsized":"2023-09-23T15:06:15.118294+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":52,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124626+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:06:15.124626+0000","last_peered":"2023-09-23T15:06:15.124626+0000","last_clean":"2023-09-23T15:06:15.124626+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:06:15.124626+0000","last_undegraded":"2023-09-23T15:06:15.124626+0000","last_fullsized":"2023-09-23T15:06:15.124626+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119275+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:06:15.119275+0000","last_peered":"2023-09-23T15:06:15.119275+0000","last_clean":"2023-09-23T15:06:15.119275+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:06:15.119275+0000","last_undegraded":"2023-09-23T15:06:15.119275+0000","last_fullsized":"2023-09-23T15:06:15.119275+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131473+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:06:16.131473+0000","last_peered":"2023-09-23T15:06:16.131473+0000","last_clean":"2023-09-23T15:06:16.131473+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:06:16.131473+0000","last_undegraded":"2023-09-23T15:06:16.131473+0000","last_fullsized":"2023-09-23T15:06:16.131473+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":78,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118992+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:06:15.118992+0000","last_peered":"2023-09-23T15:06:15.118992+0000","last_clean":"2023-09-23T15:06:15.118992+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:06:15.118992+0000","last_undegraded":"2023-09-23T15:06:15.118992+0000","last_fullsized":"2023-09-23T15:06:15.118992+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128037+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:06:16.128037+0000","last_peered":"2023-09-23T15:06:16.128037+0000","last_clean":"2023-09-23T15:06:16.128037+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:06:16.128037+0000","last_undegraded":"2023-09-23T15:06:16.128037+0000","last_fullsized":"2023-09-23T15:06:16.128037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":53,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.119363+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:06:15.119363+0000","last_peered":"2023-09-23T15:06:15.119363+0000","last_clean":"2023-09-23T15:06:15.119363+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:06:15.119363+0000","last_undegraded":"2023-09-23T15:06:15.119363+0000","last_fullsized":"2023-09-23T15:06:15.119363+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":69,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.138039+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:06:16.138039+0000","last_peered":"2023-09-23T15:06:16.138039+0000","last_clean":"2023-09-23T15:06:16.138039+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:06:16.138039+0000","last_undegraded":"2023-09-23T15:06:16.138039+0000","last_fullsized":"2023-09-23T15:06:16.138039+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.127131+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:06:16.127131+0000","last_peered":"2023-09-23T15:06:16.127131+0000","last_clean":"2023-09-23T15:06:16.127131+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:06:16.127131+0000","last_undegraded":"2023-09-23T15:06:16.127131+0000","last_fullsized":"2023-09-23T15:06:16.127131+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.135551+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:06:16.135551+0000","last_peered":"2023-09-23T15:06:16.135551+0000","last_clean":"2023-09-23T15:06:16.135551+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:06:16.135551+0000","last_undegraded":"2023-09-23T15:06:16.135551+0000","last_fullsized":"2023-09-23T15:06:16.135551+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.118610+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:06:15.118610+0000","last_peered":"2023-09-23T15:06:15.118610+0000","last_clean":"2023-09-23T15:06:15.118610+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:06:15.118610+0000","last_undegraded":"2023-09-23T15:06:15.118610+0000","last_fullsized":"2023-09-23T15:06:15.118610+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113896+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:06:15.113896+0000","last_peered":"2023-09-23T15:06:15.113896+0000","last_clean":"2023-09-23T15:06:15.113896+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:06:15.113896+0000","last_undegraded":"2023-09-23T15:06:15.113896+0000","last_fullsized":"2023-09-23T15:06:15.113896+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":75,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113914+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:06:15.113914+0000","last_peered":"2023-09-23T15:06:15.113914+0000","last_clean":"2023-09-23T15:06:15.113914+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:06:15.113914+0000","last_undegraded":"2023-09-23T15:06:15.113914+0000","last_fullsized":"2023-09-23T15:06:15.113914+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113241+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:06:15.113241+0000","last_peered":"2023-09-23T15:06:15.113241+0000","last_clean":"2023-09-23T15:06:15.113241+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:06:15.113241+0000","last_undegraded":"2023-09-23T15:06:15.113241+0000","last_fullsized":"2023-09-23T15:06:15.113241+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.137711+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:06:16.137711+0000","last_peered":"2023-09-23T15:06:16.137711+0000","last_clean":"2023-09-23T15:06:16.137711+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:06:16.137711+0000","last_undegraded":"2023-09-23T15:06:16.137711+0000","last_fullsized":"2023-09-23T15:06:16.137711+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":22,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134068+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:06:16.134068+0000","last_peered":"2023-09-23T15:06:16.134068+0000","last_clean":"2023-09-23T15:06:16.134068+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:06:16.134068+0000","last_undegraded":"2023-09-23T15:06:16.134068+0000","last_fullsized":"2023-09-23T15:06:16.134068+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":70,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.124812+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:06:15.124812+0000","last_peered":"2023-09-23T15:06:15.124812+0000","last_clean":"2023-09-23T15:06:15.124812+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:06:15.124812+0000","last_undegraded":"2023-09-23T15:06:15.124812+0000","last_fullsized":"2023-09-23T15:06:15.124812+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":73,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.131810+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:06:16.131810+0000","last_peered":"2023-09-23T15:06:16.131810+0000","last_clean":"2023-09-23T15:06:16.131810+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:06:16.131810+0000","last_undegraded":"2023-09-23T15:06:16.131810+0000","last_fullsized":"2023-09-23T15:06:16.131810+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128073+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:06:16.128073+0000","last_peered":"2023-09-23T15:06:16.128073+0000","last_clean":"2023-09-23T15:06:16.128073+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:06:16.128073+0000","last_undegraded":"2023-09-23T15:06:16.128073+0000","last_fullsized":"2023-09-23T15:06:16.128073+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":69,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.121746+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:06:15.121746+0000","last_peered":"2023-09-23T15:06:15.121746+0000","last_clean":"2023-09-23T15:06:15.121746+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:06:15.121746+0000","last_undegraded":"2023-09-23T15:06:15.121746+0000","last_fullsized":"2023-09-23T15:06:15.121746+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.132197+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:06:16.132197+0000","last_peered":"2023-09-23T15:06:16.132197+0000","last_clean":"2023-09-23T15:06:16.132197+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:06:16.132197+0000","last_undegraded":"2023-09-23T15:06:16.132197+0000","last_fullsized":"2023-09-23T15:06:16.132197+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":53,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.128449+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:06:16.128449+0000","last_peered":"2023-09-23T15:06:16.128449+0000","last_clean":"2023-09-23T15:06:16.128449+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:06:16.128449+0000","last_undegraded":"2023-09-23T15:06:16.128449+0000","last_fullsized":"2023-09-23T15:06:16.128449+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":47,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.129498+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:06:16.129498+0000","last_peered":"2023-09-23T15:06:16.129498+0000","last_clean":"2023-09-23T15:06:16.129498+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:06:16.129498+0000","last_undegraded":"2023-09-23T15:06:16.129498+0000","last_fullsized":"2023-09-23T15:06:16.129498+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":37,"reported_epoch":111,"state":"active+clean","last_fresh":"2023-09-23T15:06:16.134796+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:06:16.134796+0000","last_peered":"2023-09-23T15:06:16.134796+0000","last_clean":"2023-09-23T15:06:16.134796+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:06:16.134796+0000","last_undegraded":"2023-09-23T15:06:16.134796+0000","last_fullsized":"2023-09-23T15:06:16.134796+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":88,"reported_epoch":110,"state":"active+clean","last_fresh":"2023-09-23T15:06:15.113716+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:06:15.113716+0000","last_peered":"2023-09-23T15:06:15.113716+0000","last_clean":"2023-09-23T15:06:15.113716+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:06:15.113716+0000","last_undegraded":"2023-09-23T15:06:15.113716+0000","last_fullsized":"2023-09-23T15:06:15.113716+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201429,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":16948,"kb_used_data":5804,"kb_used_omap":0,"kb_used_meta":11136,"kb_avail":93728204,"statfs":{"total":95995035648,"available":95977680896,"internally_reserved":0,"allocated":5943296,"data_stored":5408381,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11403264},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.53900000000000003,"5min":0.57899999999999996,"15min":0.58499999999999996},"min":{"1min":0.36399999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57499999999999996,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.76000000000000001,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.56399999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.745},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.58899999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:05:12 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.60099999999999998,"15min":0.59199999999999997},"min":{"1min":0.40799999999999997,"5min":0.35599999999999998,"15min":0.33400000000000002},"max":{"1min":0.78000000000000003,"5min":1.585,"15min":1.7529999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.63500000000000001,"5min":0.65900000000000003,"15min":0.624},"min":{"1min":0.48999999999999999,"5min":0.39700000000000002,"15min":0.375},"max":{"1min":0.86599999999999999,"5min":1.5289999999999999,"15min":3.4809999999999999},"last":0.78600000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54300000000000004,"15min":0.50900000000000001},"min":{"1min":0.38200000000000001,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.80700000000000005,"5min":0.82899999999999996,"15min":2.226},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.57299999999999995,"15min":0.53800000000000003},"min":{"1min":0.52900000000000003,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.57799999999999996,"15min":0.56399999999999995},"min":{"1min":0.47799999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":0.80700000000000005,"5min":0.80800000000000005,"15min":0.92700000000000005},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.66700000000000004,"5min":0.622,"15min":0.58299999999999996},"min":{"1min":0.49399999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.88600000000000001,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.69399999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:06:13 2023","interfaces":[{"interface":"back","average":{"1min":0.68000000000000005,"5min":0.63900000000000001,"15min":0.59399999999999997},"min":{"1min":0.55000000000000004,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.85599999999999998,"5min":1.002,"15min":1.002},"last":0.62},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.64600000000000002,"15min":0.60299999999999998},"min":{"1min":0.55000000000000004,"5min":0.307,"15min":0.307},"max":{"1min":0.878,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.67600000000000005}]}]},{"osd":6,"up_from":47,"seq":201863463065,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8300,"kb_used_data":1700,"kb_used_omap":0,"kb_used_meta":6592,"kb_avail":93736852,"statfs":{"total":95995035648,"available":95986536448,"internally_reserved":0,"allocated":1740800,"data_stored":1213950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6750208},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":0.53800000000000003,"15min":0.54200000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.71499999999999997,"5min":0.73599999999999999,"15min":0.73599999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.58899999999999997,"15min":0.58199999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.69599999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.53200000000000003},{"interface":"front","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.503}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.81100000000000005,"5min":0.81100000000000005,"15min":0.81100000000000005},"last":0.627},{"interface":"front","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.0409999999999999,"5min":1.0409999999999999,"15min":1.0409999999999999},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:05:59 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.60699999999999998,"15min":0.54100000000000004},"min":{"1min":0.42499999999999999,"5min":0.32400000000000001,"15min":0.29399999999999998},"max":{"1min":0.85499999999999998,"5min":1.494,"15min":1.494},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.621,"5min":0.64400000000000002,"15min":0.58599999999999997},"min":{"1min":0.39800000000000002,"5min":0.372,"15min":0.35899999999999999},"max":{"1min":0.76500000000000001,"5min":1.7430000000000001,"15min":1.7430000000000001},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.54100000000000004,"15min":0.503},"min":{"1min":0.34000000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":0.74399999999999999,"5min":1.675,"15min":1.675},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.59799999999999998,"15min":0.52600000000000002},"min":{"1min":0.40500000000000003,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":0.81299999999999994,"5min":1.806,"15min":1.806},"last":0.73499999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:07:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.58299999999999996,"15min":0.53500000000000003},"min":{"1min":0.38200000000000001,"5min":0.32800000000000001,"15min":0.30199999999999999},"max":{"1min":0.82099999999999995,"5min":1.8959999999999999,"15min":1.8959999999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.63200000000000001,"15min":0.56299999999999994},"min":{"1min":0.46300000000000002,"5min":0.39200000000000002,"15min":0.31900000000000001},"max":{"1min":0.84099999999999997,"5min":1.8819999999999999,"15min":1.8819999999999999},"last":0.65200000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:06:18 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":0.64000000000000001,"15min":0.58999999999999997},"min":{"1min":0.443,"5min":0.35599999999999998,"15min":0.28000000000000003},"max":{"1min":0.90300000000000002,"5min":1.843,"15min":1.843},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.72299999999999998,"5min":0.69299999999999995,"15min":0.63400000000000001},"min":{"1min":0.56299999999999994,"5min":0.40999999999999998,"15min":0.373},"max":{"1min":0.88100000000000001,"5min":1.766,"15min":1.766},"last":0.72099999999999997}]}]},{"osd":1,"up_from":100,"seq":429496729626,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32844,"kb_used_data":5252,"kb_used_omap":6,"kb_used_meta":27577,"kb_avail":93712308,"statfs":{"total":95995035648,"available":95961403392,"internally_reserved":0,"allocated":5378048,"data_stored":4823646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":28238923},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":1.3560000000000001,"5min":1.3560000000000001,"15min":1.3560000000000001},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:06:48 2023","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.621,"5min":1.621,"15min":1.621},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.315}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39800000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.92200000000000004,"5min":0.92200000000000004,"15min":0.92200000000000004},"last":0.432}]},{"osd":5,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":1.024,"5min":1.024,"15min":1.024},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"max":{"1min":0.97299999999999998,"5min":0.97299999999999998,"15min":0.97299999999999998},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 15:06:16 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"min":{"1min":0.504,"5min":0.504,"15min":0.504},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.48299999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893152,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33440,"kb_used_data":5848,"kb_used_omap":8,"kb_used_meta":27575,"kb_avail":93711712,"statfs":{"total":95995035648,"available":95960793088,"internally_reserved":0,"allocated":5988352,"data_stored":5415358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":28237396},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:06:12 2023","interfaces":[{"interface":"back","average":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.255}]},{"osd":2,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.538,"5min":1.538,"15min":1.538},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.419,"5min":1.419,"15min":1.419},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":0.746,"15min":0.68100000000000005},"min":{"1min":0.39500000000000002,"5min":0.375,"15min":0.375},"max":{"1min":8.1050000000000004,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.79200000000000004},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.64000000000000001,"15min":0.63800000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":1.579,"5min":1.579,"15min":1.579},"last":0.622}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.69499999999999995,"15min":0.69599999999999995},"min":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"max":{"1min":1.294,"5min":1.294,"15min":1.294},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.1919999999999999,"5min":0.82399999999999995,"15min":0.76200000000000001},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":8.1859999999999999,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.54800000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.68899999999999995,"15min":0.69499999999999995},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.6040000000000001,"5min":1.6040000000000001,"15min":1.6040000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.67700000000000005,"15min":0.68200000000000005},"min":{"1min":0.437,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":1.3919999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.67500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.67300000000000004,"15min":0.67500000000000004},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.3320000000000001,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.78400000000000003},{"interface":"front","average":{"1min":1.175,"5min":0.80600000000000005,"15min":0.74399999999999999},"min":{"1min":0.48499999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":8.157,"5min":8.157,"15min":8.157},"last":0.86599999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566099,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28680,"kb_used_data":1152,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93716472,"statfs":{"total":95995035648,"available":95965667328,"internally_reserved":0,"allocated":1179648,"data_stored":628060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":28168976},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.27500000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.33900000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"last":0.31}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38800000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.62},{"interface":"front","average":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":0.63500000000000001,"15min":0.63500000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.45900000000000002,"5min":0.45900000000000002,"15min":0.45900000000000002},"max":{"1min":0.93400000000000005,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.57699999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.9870000000000001,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"min":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"max":{"1min":0.95499999999999996,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 15:06:44 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.92900000000000005,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.49399999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402573,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28592,"kb_used_data":1128,"kb_used_omap":3,"kb_used_meta":27452,"kb_avail":93716560,"statfs":{"total":95995035648,"available":95965757440,"internally_reserved":0,"allocated":1155072,"data_stored":623615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":28111017},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.317}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35599999999999998}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45800000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.436}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55500000000000005}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56699999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986338,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7852,"kb_used_data":1124,"kb_used_omap":0,"kb_used_meta":6720,"kb_avail":93737300,"statfs":{"total":95995035648,"available":95986995200,"internally_reserved":0,"allocated":1150976,"data_stored":623639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6881280},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.53800000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.54900000000000004,"15min":0.54000000000000004},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.438}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48699999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.376}]},{"osd":2,"last update":"Sat Sep 23 15:06:45 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"min":{"1min":0.45500000000000002,"5min":0.45500000000000002,"15min":0.45500000000000002},"max":{"1min":2.6619999999999999,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":0.59799999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:06:04 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.42799999999999999,"5min":0.376,"15min":0.34399999999999997},"max":{"1min":0.83899999999999997,"5min":0.88200000000000001,"15min":0.93600000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.59799999999999998,"5min":0.60599999999999998,"15min":0.56499999999999995},"min":{"1min":0.436,"5min":0.372,"15min":0.34100000000000003},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91400000000000003},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:06:28 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55400000000000005,"15min":0.51100000000000001},"min":{"1min":0.311,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":0.77400000000000002,"5min":0.876,"15min":0.88400000000000001},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58499999999999996,"15min":0.52800000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.26200000000000001},"max":{"1min":0.80100000000000005,"5min":0.85399999999999998,"15min":0.91700000000000004},"last":0.61599999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:06:59 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":0.622,"15min":0.59599999999999997},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.318},"max":{"1min":0.83899999999999997,"5min":0.92500000000000004,"15min":1.4610000000000001},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.63500000000000001,"15min":0.58899999999999997},"min":{"1min":0.40300000000000002,"5min":0.38900000000000001,"15min":0.33400000000000002},"max":{"1min":0.83499999999999996,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:07:06 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.66000000000000003,"15min":0.60599999999999998},"min":{"1min":0.47799999999999998,"5min":0.38600000000000001,"15min":0.28799999999999998},"max":{"1min":0.89500000000000002,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.69999999999999996},{"interface":"front","average":{"1min":0.69499999999999995,"5min":0.69899999999999995,"15min":0.66300000000000003},"min":{"1min":0.432,"5min":0.432,"15min":0.38800000000000001},"max":{"1min":0.876,"5min":0.95099999999999996,"15min":1.0249999999999999},"last":0.63600000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724701,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7456,"kb_used_data":1112,"kb_used_omap":0,"kb_used_meta":6336,"kb_avail":93737696,"statfs":{"total":95995035648,"available":95987400704,"internally_reserved":0,"allocated":1138688,"data_stored":619402,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6488064},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:06:42 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.36599999999999999,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.374,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.83699999999999997,"5min":0.83699999999999997,"15min":0.83699999999999997},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:06:15 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:06:46 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.78100000000000003,"5min":0.78100000000000003,"15min":0.78100000000000003},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 15:05:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.59499999999999997,"15min":0.55200000000000005},"min":{"1min":0.40400000000000003,"5min":0.35199999999999998,"15min":0.312},"max":{"1min":0.748,"5min":0.879,"15min":1.262},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.60399999999999998,"15min":0.58299999999999996},"min":{"1min":0.377,"5min":0.373,"15min":0.35499999999999998},"max":{"1min":0.70899999999999996,"5min":0.96099999999999997,"15min":1.248},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:06:29 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.501,"15min":0.46000000000000002},"min":{"1min":0.40100000000000002,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":0.73499999999999999,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.626},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.54900000000000004,"15min":0.50800000000000001},"min":{"1min":0.33800000000000002,"5min":0.33000000000000002,"15min":0.27400000000000002},"max":{"1min":0.72799999999999998,"5min":0.74199999999999999,"15min":0.78400000000000003},"last":0.376}]},{"osd":6,"last update":"Sat Sep 23 15:06:51 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.58099999999999996,"15min":0.52800000000000002},"min":{"1min":0.439,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.68000000000000005,"5min":0.64800000000000002,"15min":0.59799999999999998},"min":{"1min":0.56999999999999995,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":0.80900000000000005,"5min":0.85499999999999998,"15min":0.91000000000000003},"last":0.57199999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:06:17 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.60799999999999998,"15min":0.59199999999999997},"min":{"1min":0.5,"5min":0.39400000000000002,"15min":0.28299999999999997},"max":{"1min":0.79900000000000004,"5min":0.86899999999999999,"15min":1.0109999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.63800000000000001,"15min":0.58599999999999997},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.33300000000000002},"max":{"1min":0.81200000000000006,"5min":0.92500000000000004,"15min":0.92500000000000004},"last":0.67000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:07:14.694 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T15:07:14.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T15:07:15.090 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:15.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:15 smithi017 ceph-mon[215646]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:15.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:15 smithi017 ceph-mon[215646]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:15 smithi017 ceph-mon[222878]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:15 smithi017 ceph-mon[222878]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:15 smithi027 ceph-mon[195233]: from='client.34698 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:07:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:15 smithi027 ceph-mon[195233]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:epoch 111 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:created 2023-09-23T14:50:02.735782+0000 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:modified 2023-09-23T15:06:16.113424+0000 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:crush_version 22 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:full_ratio 0.95 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:backfillfull_ratio 0.9 2023-09-23T15:07:16.744 INFO:teuthology.orchestra.run.smithi017.stdout:nearfull_ratio 0.85 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:require_min_compat_client luminous 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:min_compat_client luminous 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:require_osd_release quincy 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:stretch_mode_enabled false 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 85 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 68 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:max_osd 8 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 up in weight 1 up_from 95 up_thru 110 down_at 92 last_clean_interval [8,91) [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239] [v2:172.21.15.17:6804/1593815239,v1:172.21.15.17:6805/1593815239] exists,up 0ab537e0-788a-404a-aee3-04d5c85f3ea8 2023-09-23T15:07:16.745 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 up in weight 1 up_from 100 up_thru 110 down_at 97 last_clean_interval [13,96) [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547] [v2:172.21.15.17:6812/2834890547,v1:172.21.15.17:6813/2834890547] exists,up 2d26e6ec-8564-4b09-a9e8-f7fe77e36a65 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 up in weight 1 up_from 105 up_thru 110 down_at 102 last_clean_interval [18,101) [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298] [v2:172.21.15.17:6820/1172679298,v1:172.21.15.17:6821/1172679298] exists,up 509065ed-f964-449d-8ed3-bb7c0b529dd8 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 up in weight 1 up_from 110 up_thru 110 down_at 107 last_clean_interval [26,106) [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743] [v2:172.21.15.17:6828/3586551743,v1:172.21.15.17:6829/3586551743] exists,up 1430d849-5f40-43a2-9ebd-c82ddaaa262b 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 up in weight 1 up_from 31 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574] [v2:172.21.15.27:6802/2107894574,v1:172.21.15.27:6803/2107894574] exists,up 5b9137a6-fdb7-4150-813f-c72707df3509 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 up in weight 1 up_from 39 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946] [v2:172.21.15.27:6810/634894946,v1:172.21.15.27:6811/634894946] exists,up 06d868f4-810f-45ef-be6a-0c6edcea5ad7 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6 up in weight 1 up_from 47 up_thru 107 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010] [v2:172.21.15.27:6818/1429782010,v1:172.21.15.27:6819/1429782010] exists,up 06f02713-faf7-456c-bd0d-52eb13680742 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7 up in weight 1 up_from 55 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591] [v2:172.21.15.27:6826/2619255591,v1:172.21.15.27:6827/2619255591] exists,up d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.2 [1,2] 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.18 [1,2] 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:pg_upmap_items 4.1d [1,5] 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/79453409 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.746 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1503872993 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1791820912 expires 2023-09-24T14:50:31.828392+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/625940860 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2670521468 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2135416630 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:6834/1216874840 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/707576150 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/244258131 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/1374246236 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/1159417105 expires 2023-09-24T14:50:31.828392+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/816410812 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/814296392 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/4217628619 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.747 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:6835/1216874840 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/1159417105 expires 2023-09-24T14:50:31.828392+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1467411231 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/625940860 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/3796759750 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/3040364158 expires 2023-09-24T14:50:58.900461+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2463078697 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1605426391 expires 2023-09-24T14:50:31.828392+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2823536185 expires 2023-09-24T14:50:58.900461+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/3040364158 expires 2023-09-24T14:50:58.900461+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/201191039 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1851970177 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1666581637 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.748 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/774095134 expires 2023-09-24T14:59:04.520407+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/3796759750 expires 2023-09-24T14:55:28.581618+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3598198038 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6800/1926022044 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:6801/1926022044 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/2369426862 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.27:0/986573237 expires 2023-09-24T14:59:45.132154+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/1446163903 expires 2023-09-24T14:50:58.900461+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/577227578 expires 2023-09-24T14:50:31.828392+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/3013219017 expires 2023-09-24T14:50:58.900461+0000 2023-09-23T15:07:16.749 INFO:teuthology.orchestra.run.smithi017.stdout:blocklist 172.21.15.17:0/623737292 expires 2023-09-24T15:00:28.003544+0000 2023-09-23T15:07:17.356 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 95 up_thru 110 down_at 92 last_clean_interval [8,91) [v2:172.21.15.17:6802/1593815239,v1:172.21.15.17:6803/1593815239] [v2:172.21.15.17:6804/1593815239,v1:172.21.15.17:6805/1593815239] exists,up 0ab537e0-788a-404a-aee3-04d5c85f3ea8', 'osd.1 up in weight 1 up_from 100 up_thru 110 down_at 97 last_clean_interval [13,96) [v2:172.21.15.17:6810/2834890547,v1:172.21.15.17:6811/2834890547] [v2:172.21.15.17:6812/2834890547,v1:172.21.15.17:6813/2834890547] exists,up 2d26e6ec-8564-4b09-a9e8-f7fe77e36a65', 'osd.2 up in weight 1 up_from 105 up_thru 110 down_at 102 last_clean_interval [18,101) [v2:172.21.15.17:6818/1172679298,v1:172.21.15.17:6819/1172679298] [v2:172.21.15.17:6820/1172679298,v1:172.21.15.17:6821/1172679298] exists,up 509065ed-f964-449d-8ed3-bb7c0b529dd8', 'osd.3 up in weight 1 up_from 110 up_thru 110 down_at 107 last_clean_interval [26,106) [v2:172.21.15.17:6826/3586551743,v1:172.21.15.17:6827/3586551743] [v2:172.21.15.17:6828/3586551743,v1:172.21.15.17:6829/3586551743] exists,up 1430d849-5f40-43a2-9ebd-c82ddaaa262b', 'osd.4 up in weight 1 up_from 31 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6800/2107894574,v1:172.21.15.27:6801/2107894574] [v2:172.21.15.27:6802/2107894574,v1:172.21.15.27:6803/2107894574] exists,up 5b9137a6-fdb7-4150-813f-c72707df3509', 'osd.5 up in weight 1 up_from 39 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6808/634894946,v1:172.21.15.27:6809/634894946] [v2:172.21.15.27:6810/634894946,v1:172.21.15.27:6811/634894946] exists,up 06d868f4-810f-45ef-be6a-0c6edcea5ad7', 'osd.6 up in weight 1 up_from 47 up_thru 107 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6816/1429782010,v1:172.21.15.27:6817/1429782010] [v2:172.21.15.27:6818/1429782010,v1:172.21.15.27:6819/1429782010] exists,up 06f02713-faf7-456c-bd0d-52eb13680742', 'osd.7 up in weight 1 up_from 55 up_thru 110 down_at 0 last_clean_interval [0,0) [v2:172.21.15.27:6824/2619255591,v1:172.21.15.27:6825/2619255591] [v2:172.21.15.27:6826/2619255591,v1:172.21.15.27:6827/2619255591] exists,up d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4'] 2023-09-23T15:07:17.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T15:07:17.716 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:18.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:17 smithi017 ceph-mon[215646]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:18.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/223896721' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:07:18.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:17 smithi017 ceph-mon[222878]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:18.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/223896721' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:07:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:17 smithi027 ceph-mon[195233]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/223896721' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:07:18.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:18] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.43.0" 2023-09-23T15:07:19.112 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:07:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "name": "mon.a", 2023-09-23T15:07:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster": "ceph", 2023-09-23T15:07:19.112 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "admin_socket_mode": "", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_client_required": "cephx, none", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_cluster_required": "cephx", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_debug": "true", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_required": "cephx", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T15:07:19.113 INFO:teuthology.orchestra.run.smithi017.stdout: "auth_supported": "", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio": "true", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_aio_reap_max": "16", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_async_discard": "false", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_block_size": "4096", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio": "false", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T15:07:19.114 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_enable_discard": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry": "3", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash": "0", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_hipri": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T15:07:19.115 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bdev_type": "", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_allocator": "hybrid", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_buffered_io": "true", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T15:07:19.116 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery": "false", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluefs_sync_write": "false", 2023-09-23T15:07:19.117 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_allocator": "hybrid", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T15:07:19.118 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_create": "true", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_create": "false", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_path": "", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_db_size": "0", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_path": "", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_size": "107374182400", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_create": "false", 2023-09-23T15:07:19.119 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_path": "", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs": "true", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T15:07:19.120 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune": "true", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T15:07:19.121 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size": "0", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cache_type": "2q", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T15:07:19.122 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_clone_cow": "true", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_mode": "none", 2023-09-23T15:07:19.123 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_freelist": "false", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T15:07:19.124 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T15:07:19.125 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T15:07:19.126 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T15:07:19.127 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fail_eio": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T15:07:19.128 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T15:07:19.129 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size": "0", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T15:07:19.130 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T15:07:19.131 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_spdk_mem": "512", 2023-09-23T15:07:19.132 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T15:07:19.133 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_tracing": "false", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T15:07:19.134 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T15:07:19.135 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T15:07:19.136 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_cluster_require_version": "2", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_signatures": "false", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_require_version": "2", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_signatures": "false", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_service_require_version": "2", 2023-09-23T15:07:19.137 INFO:teuthology.orchestra.run.smithi017.stdout: "cephx_sign_messages": "true", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "chdir": "", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_acl_type": "", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_asio_thread_count": "2", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_mid": "0.750000", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_cache_size": "16384", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_caps_release_delay": "5", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_check_pool_perm": "true", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_force_sync_read": "false", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_getattr_caps": "false", 2023-09-23T15:07:19.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_die_on_failed_remount": "false", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_dirsize_rbytes": "true", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_force_lazyio": "false", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_fs": "", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_inject_release_failure": "false", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_inline_size": "4096", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mds_namespace": "", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_metadata": "", 2023-09-23T15:07:19.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_gid": "-1", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_timeout": "300", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mount_uid": "-1", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_mountpoint": "/", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_notify_timeout": "10", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc": "true", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_max_objects": "1000", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_size": "209715200", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_permissions": "true", 2023-09-23T15:07:19.140 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota": "true", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_quota_df": "true", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_bytes": "0", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_max_periods": "4", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_readahead_min": "131072", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_reconnect_stale": "false", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_shutdown_timeout": "30", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_snapdir": ".snap", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_tick_interval": "1", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_trace": "", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_faked_inos": "false", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "client_use_random_mds": "false", 2023-09-23T15:07:19.141 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog": "false", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_graylog_port": "12201", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_monitors": "default=true", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog": "false", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "clog_to_syslog_level": "info", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_addr": "-", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network": "", 2023-09-23T15:07:19.142 INFO:teuthology.orchestra.run.smithi017.stdout: "cluster_network_interface": "", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_isal": "false", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_level": "5", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "compressor_zstd_level": "1", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T15:07:19.143 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "crimson_seastar_smp": "1", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location": "", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook": "", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "crush_location_hook_timeout": "10", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "daemonize": "false", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "daos_pool": "tank", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_alienstore": "0/5", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok": "1/5", 2023-09-23T15:07:19.144 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asok_assert_abort": "false", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_auth": "1/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bdev": "1/3", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluefs": "1/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_bluestore": "1/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_buffer": "0/1", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cephsqlite": "0/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_client": "0/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_compressor": "1/5", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_context": "0/1", 2023-09-23T15:07:19.145 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crush": "1/1", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_crypto": "1/5", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_cyanstore": "0/5", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_dpdk": "1/5", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_eventtrace": "1/5", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filer": "0/1", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_filestore": "1/3", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_finisher": "1/1", 2023-09-23T15:07:19.146 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_fuse": "1/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_javaclient": "1/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journal": "1/3", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_journaler": "0/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_kstore": "1/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_leveldb": "4/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_lockdep": "0/1", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mclock": "1/5", 2023-09-23T15:07:19.147 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_balancer": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_locker": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mds_migrator": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_memstore": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgr": "2/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mgrc": "1/5", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_mon": "20/20", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_monc": "0/10", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_ms": "1/1", 2023-09-23T15:07:19.148 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_none": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objclass": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objectcacher": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_objecter": "0/1", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_optracker": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_osd": "1/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_paxos": "20/20", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_perfcounter": "1/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_prioritycache": "1/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rados": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T15:07:19.149 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rbd_replay": "0/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_refs": "0/0", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_reserver": "1/1", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_access": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_flight": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rgw_sync": "1/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_rocksdb": "4/5", 2023-09-23T15:07:19.150 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_backref": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cache": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_device": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_epm": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_journal": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_lba": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_odata": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_omap": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_onode": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_t": "0/5", 2023-09-23T15:07:19.151 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_seastore_tm": "0/5", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_striper": "0/1", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_test": "0/5", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_throttle": "1/1", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_timer": "0/1", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "debug_tp": "0/5", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "device_failure_prediction_mode": "none", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_graylog": "false", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_journald": "false", 2023-09-23T15:07:19.152 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_stderr": "true", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "err_to_syslog": "false", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event_tracing": "false", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_http_port": "9926", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_prio_limit": "5", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_sort_metrics": "true", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "exporter_stats_period": "5", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "fake_statfs_for_testing": "0", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "fatal_signal_handlers": "true", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_purge_ops": "10", 2023-09-23T15:07:19.153 INFO:teuthology.orchestra.run.smithi017.stdout: "filer_max_truncate_ops": "128", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_blackhole": "false", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_btrfs_snap": "true", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_caller_concurrency": "10", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_omap_check": "false", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_debug_verify_split": "false", 2023-09-23T15:07:19.154 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_dump_file": "", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fadvise": "true", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fail_eio": "true", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fd_cache_size": "128", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap": "false", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_inject_stall": "0", 2023-09-23T15:07:19.155 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_parallel": "false", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_trailing": "false", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_journal_writeahead": "false", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_kill_at": "0", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T15:07:19.156 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_merge_threshold": "-10", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_odsync_write": "false", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_backend_path": "", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T15:07:19.157 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_op_threads": "2", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_punch_hole": "false", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T15:07:19.158 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_queue_max_ops": "50", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_seek_data_hole": "false", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc": "false", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_splice": "false", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_multiple": "2", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_split_rand_factor": "20", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_update_to": "1000", 2023-09-23T15:07:19.159 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T15:07:19.160 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_xfs_extsize": "false", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "filestore_zfs_snap": "false", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fio_dir": "/tmp/fio", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fsid": "546d6668-5a20-11ee-8db4-212e2dc638e7", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_allow_other": "true", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_big_writes": "true", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_debug": "false", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_default_permissions": "false", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_disable_pagecache": "false", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_max_write": "0", 2023-09-23T15:07:19.161 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_multithreaded": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_require_active_mds": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_set_user_groups": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_move": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_read": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_splice_write": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "gss_target_name": "ceph", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_file": "", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_inject_failure": "0", 2023-09-23T15:07:19.162 INFO:teuthology.orchestra.run.smithi017.stdout: "heartbeat_interval": "5", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "host": "smithi017", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T15:07:19.163 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "inject_early_sigterm": "false", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_agent_port": "6799", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "jaeger_tracing_enable": "false", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_aio": "true", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_align_min_size": "65536", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_align": "true", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_block_size": "4096", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_dio": "true", 2023-09-23T15:07:19.164 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_discard": "false", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_force_aio": "false", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_ignore_corruption": "false", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_max_write_entries": "100", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_replay_from": "0", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_write_header_frequency": "0", 2023-09-23T15:07:19.165 INFO:teuthology.orchestra.run.smithi017.stdout: "journal_zero_on_create": "false", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prefetch_periods": "10", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_prezero_periods": "5", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "journaler_write_head_interval": "15", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "key": "", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "keyfile": "", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_backend": "rocksdb", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T15:07:19.166 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_bytes": "67108864", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_max_ops": "512", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_onode_map_size": "1024", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "kstore_sync_transaction": "false", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_block_size": "65536", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_bloom_size": "0", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_cache_size": "536870912", 2023-09-23T15:07:19.167 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_compression": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_log": "", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_max_open_files": "0", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_paranoid": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "librados_thread_count": "2", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "lockdep_force_backtrace": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "log_coarse_timestamps": "true", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "log_flush_on_exit": "false", 2023-09-23T15:07:19.168 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_graylog_port": "12201", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_new": "1000", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_max_recent": "10000", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stderr_prefix": "", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_file": "true", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_graylog": "false", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_journald": "false", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_stderr": "false", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "log_to_syslog": "false", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T15:07:19.169 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_action_on_write_error": "1", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_alternate_name_max": "8192", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_asio_thread_count": "2", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_export_pin": "true", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_interval": "10", 2023-09-23T15:07:19.170 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max": "-1", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_max_until": "-1", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_merge_size": "50", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_mode": "0", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_bits": "3", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T15:07:19.171 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_size": "10000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_mid": "0.700000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_interval": "1", 2023-09-23T15:07:19.172 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_auth_pins": "false", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_frag": "false", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_scatterstat": "false", 2023-09-23T15:07:19.173 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_debug_subtrees": "false", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_default_dir_hash": "2", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_defer_session_stale": "true", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_max_entries": "0", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dir_prefetch": "true", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T15:07:19.174 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_early_reply": "true", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enable_op_tracker": "true", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_enforce_unique_name": "true", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T15:07:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_health_dummy": "false", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T15:07:19.176 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_join_fs": "", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_journal_format": "1", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_create_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_export_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_import_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_link_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_openc_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_rename_at": "0", 2023-09-23T15:07:19.177 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_events": "-1", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_max_segments": "128", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_pause": "false", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_segment_size": "0", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_completed_requests": "100000", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_export_size": "20971520", 2023-09-23T15:07:19.178 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_file_recover": "32", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_files": "64", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops": "8192", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_metrics_update_interval": "2", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_per_client": "100", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_numa_node": "-1", 2023-09-23T15:07:19.179 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_duration": "600", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_history_size": "20", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_op_log_threshold": "5", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_grace": "15", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_ping_interval": "5", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_caps": "30000", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T15:07:19.180 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_interval": "1.000000", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_gid": "0", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_root_ino_uid": "0", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T15:07:19.181 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_shutdown_check": "0", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_skip_ino": "0", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_min_uid": "0", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_snap_rstat": "false", 2023-09-23T15:07:19.182 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_symlink_recovery": "true", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_exports": "0", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_thrash_fragments": "0", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_tick_interval": "5.000000", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_valgrind_exit": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_backtrace": "1", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_verify_scatter": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mds_wipe_sessions": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "mempool_debug": "false", 2023-09-23T15:07:19.183 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_set": "false", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "memstore_page_size": "65536", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_bytes": "134217728", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_messages": "512", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_disabled_modules": "", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T15:07:19.184 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mds_messages": "128", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_mon_messages": "128", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_osd_messages": "8192", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_pool": "true", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_standby_modules": "true", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_period": "5", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_stats_threshold": "5", 2023-09-23T15:07:19.185 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_tick_period": "2", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_delete": "true", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_bytes": "104857600", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T15:07:19.186 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T15:07:19.187 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_start": "false", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_compact_on_trim": "true", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_cpu_threads": "4", 2023-09-23T15:07:19.188 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_crit": "5", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_avail_warn": "5", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_json": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_extra_checks": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T15:07:19.189 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_default_strategy": "1", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_election_timeout": "5.000000", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T15:07:19.190 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_enable_op_tracker": "true", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_fake_pool_delete": "false", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_force_quorum_join": "false", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_log_update_period": "5", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_max_detail": "50", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog": "true", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host": "[v2:172.21.15.17:3300/0,v1:172.21.15.17:6789/0] [v2:172.21.15.27:3300/0,v1:172.21.15.27:6789/0] [v2:172.21.15.17:3301/0,v1:172.21.15.17:6790/0]", 2023-09-23T15:07:19.191 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_host_override": "", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_initial_members": "", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease": "5.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_full_interval": "50", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max": "10000", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_log_max_summary": "50", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T15:07:19.192 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_log_epochs": "500", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_osd": "10000", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_autotune": "true", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_memory_target": "2147483648", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T15:07:19.193 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_digest_period": "5", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_complaint_time": "30", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_duration": "600", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_size": "20", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_op_log_threshold": "5", 2023-09-23T15:07:19.194 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size": "500", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T15:07:19.195 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T15:07:19.196 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_report_timeout": "900", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T15:07:19.197 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T15:07:19.198 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_max_osds": "4", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_interval": "86400", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_max_keys": "100", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_scrub_timeout": "300", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_session_timeout": "300", 2023-09-23T15:07:19.199 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_smart_report_timeout": "5", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_pool_size": "4", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_debug": "false", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T15:07:19.200 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_tick_interval": "5", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T15:07:19.201 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_older_version": "true", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T15:07:19.202 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "monmap": "", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "motr_tracing_enabled": "false", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_op_threads": "3", 2023-09-23T15:07:19.203 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_cm": "false", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_device_name": "", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T15:07:19.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_sl": "3", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_rdma_type": "ib", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_async_reap_threshold": "5", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_before_connect": "false", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv4": "true", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_ipv6": "false", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr1": "true", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_msgr2": "true", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_max": "7568", 2023-09-23T15:07:19.205 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_port_min": "6800", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_count": "3", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_bind_retry_delay": "5", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_client": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mds": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mgr": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_mon": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_blackhole_osd": "false", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_client_mode": "crc secure", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_cluster_type": "", 2023-09-23T15:07:19.206 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_compress_secure": "false", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_data": "true", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_crc_header": "true", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_bug": "true", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_old_message": "true", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T15:07:19.207 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hugepages": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_lro": "true", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_pmd": "", 2023-09-23T15:07:19.208 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_port_id": "0", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_dump_on_send": "false", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_delay_type": "", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_network_congestion": "0", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_inject_socket_failures": "0", 2023-09-23T15:07:19.209 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_accept_failures": "4", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_max_backoff": "15.000000", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compress_mode": "none", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_pq_min_cost": "65536", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_public_type": "", 2023-09-23T15:07:19.210 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_service_mode": "crc secure", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_nodelay": "true", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "ms_type": "async+posix", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "no_config_file": "false", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inflight_ops": "1024", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T15:07:19.211 INFO:teuthology.orchestra.run.smithi017.stdout: "objecter_timeout": "10.000000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "objectstore_blackhole": "false", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "openssl_engine_opts": "", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_max_ops": "4", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_agent_slop": "0.020000", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T15:07:19.212 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_asio_thread_count": "2", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_max": "512", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backfill_scan_min": "64", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_peering": "false", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_beacon_report_interval": "300", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_duration": "30", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T15:07:19.213 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blkin_trace_all": "false", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T15:07:19.214 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_class_update_on_start": "true", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_cap": "256", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_op_priority": "63", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_client_watch_timeout": "30", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_command_max_records": "256", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_compact_on_start": "false", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_on_start": "true", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T15:07:19.215 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_op_order": "false", 2023-09-23T15:07:19.216 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_shutdown": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T15:07:19.217 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_default_notify_timeout": "30", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T15:07:19.218 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_enable_op_tracker": "true", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_info": "true", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown": "true", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_function_tracing": "false", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_grace": "20", 2023-09-23T15:07:19.219 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_interval": "6", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_stale": "600", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T15:07:19.220 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_journal_size": "5120", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_kill_backfill_at": "0", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_cache_size": "50", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_dedup": "true", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max": "40", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_name_len": "100", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_attr_size": "0", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_backfills": "1", 2023-09-23T15:07:19.221 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_count": "5", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_markdown_period": "600", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_name_len": "2048", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_object_size": "134217728", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_pgls": "1024", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_cost": "8388608", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_push_objects": "10", 2023-09-23T15:07:19.222 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_scrubs": "1", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_max_write_size": "90", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T15:07:19.223 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_profile": "balanced", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T15:07:19.224 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_base": "805306368", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target": "4294967296", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_autotune": "false", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_min_recovery_priority": "0", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T15:07:19.225 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_interval": "5", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_cache_shards": "32", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_node": "-1", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objecter_finishers": "1", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore": "bluestore", 2023-09-23T15:07:19.226 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_fuse": "false", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_objectstore_tracing": "false", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_duration": "600", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_size": "20", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_log_threshold": "5", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards": "0", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T15:07:19.227 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_op_thread_timeout": "15", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_open_classes_on_start": "true", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_os_flags": "0", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_peering_op_priority": "255", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_delete_priority": "5", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T15:07:19.228 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T15:07:19.229 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crimson": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_flags": "0", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_min_size": "0", 2023-09-23T15:07:19.230 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_size": "3", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_default_type": "replicated", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T15:07:19.231 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_cost": "20971520", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active": "0", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_priority": "3", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_priority": "5", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T15:07:19.232 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_repair_during_recovery": "false", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T15:07:19.233 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_cost": "52428800", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_hour": "0", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T15:07:19.234 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_priority": "5", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_skip_data_digest": "false", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_smart_report_timeout": "5", 2023-09-23T15:07:19.235 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_priority": "5", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_target_transaction_size": "30", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T15:07:19.236 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_tracing": "false", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_use_stale_snap": "false", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T15:07:19.237 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_kill_at": "0", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_max_join_drift": "10", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min": "500", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_min_wait": "0.050000", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max": "500", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_service_trim_min": "250", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_stash_full_interval": "25", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_max": "500", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "paxos_trim_min": "250", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "perf": "true", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "pid_file": "", 2023-09-23T15:07:19.238 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addr": "-", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "public_addrv": "", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "public_bind_addr": "-", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network": "172.21.0.0/20", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "public_network_interface": "", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_enabled": "false", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_mon_op_timeout": "0", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_osd_op_timeout": "0", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "rados_tracing": "false", 2023-09-23T15:07:19.239 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_atime_update_interval": "60", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache": "true", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T15:07:19.240 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_policy": "writearound", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_size": "33554432", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_compression_hint": "none", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_clone_format": "auto", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_data_pool": "", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_features": "61", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_format": "2", 2023-09-23T15:07:19.241 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_map_options": "", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_order": "22", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_pool": "rbd", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_count": "0", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler": "simple", 2023-09-23T15:07:19.242 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_order": "24", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_pool": "", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_journal_splay_width": "4", 2023-09-23T15:07:19.243 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T15:07:19.244 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T15:07:19.245 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T15:07:19.246 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_op_threads": "1", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_plugins": "", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T15:07:19.247 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T15:07:19.248 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T15:07:19.249 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_tracing": "false", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_names": "true", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rbd_validate_pool": "true", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_base_url": "", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "restapi_log_level": "", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_admin_entry": "admin", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_backend_store": "rados", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_barbican_url": "", 2023-09-23T15:07:19.250 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_beast_enable_async": "true", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_enabled": "true", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_config_store": "rados", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_content_length_compat": "false", 2023-09-23T15:07:19.251 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_cross_domain_policy": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T15:07:19.252 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T15:07:19.253 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T15:07:19.254 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T15:07:19.255 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_log_window": "30", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T15:07:19.256 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T15:07:19.257 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_name": "", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dns_s3website_name": "", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_ops_log": "false", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_static_website": "false", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enable_usage_log": "false", 2023-09-23T15:07:19.258 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_expose_bucket": "false", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_extended_http_attrs": "", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_filter": "none", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T15:07:19.259 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_objs": "32", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_init_timeout": "300", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T15:07:19.260 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_project": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_admin_user": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_api_version": "2", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T15:07:19.261 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_url": "", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T15:07:19.262 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_objs": "32", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_worker": "3", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T15:07:19.263 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_http_headers": "", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attr_size": "0", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T15:07:19.264 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_listing_results": "1000", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_notify_retries": "10", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T15:07:19.265 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T15:07:19.266 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_num_control_oids": "8", 2023-09-23T15:07:19.267 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_numa_node": "-1", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T15:07:19.268 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_op_tracing": "false", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_token": "", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_url": "", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T15:07:19.269 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_rados": "false", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T15:07:19.270 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_continue": "true", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T15:07:19.271 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rados_tracing": "false", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm": "", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_id": "", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region": "", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T15:07:19.272 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_request_uri": "", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T15:07:19.273 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_resolve_cname": "false", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_run_sync_thread": "true", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T15:07:19.274 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_script_uri": "", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_service_provider_name": "", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_id": "", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_client_secret": "", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_entry": "sts", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_key": "sts", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T15:07:19.275 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_auth_url": "", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_custom_header": "", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_need_stats": "true", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_tenant_name": "", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url": "", 2023-09-23T15:07:19.276 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_lease_period": "120", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T15:07:19.277 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_thread_pool_size": "512", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_comment": "", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_createby": "", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_encoding": "", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_flag": "false", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_origin": "", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_torrent_tracker": "", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T15:07:19.278 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_shards": "32", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_use_opa_authz": "false", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_user_unique_email": "true", 2023-09-23T15:07:19.279 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_verify_ssl": "true", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone": "", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_id": "", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup": "", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_id": "", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T15:07:19.280 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_block_size": "4096", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T15:07:19.281 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_partition_filters": "false", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_perf": "false", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "run_dir": "/var/run/ceph", 2023-09-23T15:07:19.282 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_block_create": "true", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_device_size": "53687091200", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_main_device_type": "SSD", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T15:07:19.283 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "seastore_segment_size": "67108864", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "set_keepcaps": "false", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "setgroup": "ceph", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser": "ceph", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "setuser_match_path": "", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "thp": "false", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_default_timeout": "60", 2023-09-23T15:07:19.284 INFO:teuthology.orchestra.run.smithi017.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T15:07:19.285 INFO:teuthology.orchestra.run.smithi017.stdout: "throttler_perf_counter": "true" 2023-09-23T15:07:19.285 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:07:19.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T15:07:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:19 smithi027 ceph-mon[195233]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:20.151 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:20.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:19 smithi017 ceph-mon[215646]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:19 smithi017 ceph-mon[222878]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:21.568 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2023-09-23T15:07:21.577 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2023-09-23T15:07:21.587 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2023-09-23T15:07:21.589 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T15:07:21.589 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T15:07:21.589 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T15:07:21.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:21 smithi017 ceph-mon[215646]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:21.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2423030794' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:07:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:21 smithi017 ceph-mon[222878]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:21.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2423030794' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:07:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:21 smithi027 ceph-mon[195233]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2423030794' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:07:22.190 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T15:07:22.191 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T15:07:22.191 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:07:22.193 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:07:22.193 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T15:07:22.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T15:07:22.193 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T15:07:22.194 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T15:07:22.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:07:22.194 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T15:07:22.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:22.195 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T15:07:22.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:07:22.195 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T15:07:22.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:07:22.196 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T15:07:22.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:07:22.248 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:22.300 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:22.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:22.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:22.449 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:22.526 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:22.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:22.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:22.631 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:22.652 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:22.677 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:22.689 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:22.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:22.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:22.750 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:22.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:22.888 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:22.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:22.940 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:22.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:22.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:23.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:23.073 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:23.325 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:23.328 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:23.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:23.460 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:23.544 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:24.010 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:24.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:23 smithi027 ceph-mon[195233]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:24.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:23 smithi017 ceph-mon[215646]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:23 smithi017 ceph-mon[222878]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:24.420 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:24.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:24.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:24.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:24.558 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:24.609 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:24.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:24.613 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:24.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:24.744 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:25.545 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:25.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:25.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:25.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:25.678 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:25 smithi017 ceph-mon[215646]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:25.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:25 smithi017 ceph-mon[222878]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:25.984 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:25.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:25.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:26.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:07:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:25 smithi027 ceph-mon[195233]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:26.122 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:26.173 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:07:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:26.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:07:26.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:26.310 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:26.357 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:26.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:26.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:26.491 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:26.535 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:26.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:26.539 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:26.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:26.666 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:26.709 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:26.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:26.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:26.758 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:07:26.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:26.846 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:26.997 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3670279347' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:26.997 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:26.997 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3670279347' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:26.997 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3670279347' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:07:27.649 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:27.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:27.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:27.706 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:07:27.728 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:07:27.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:07:27.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:27.757 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:07:27.776 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:07:27.788 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:27.799 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:07:27.821 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:07:27.843 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:07:27.871 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.873 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:07:27.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:27.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: failed to encode map e112 with expected crc 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2864292551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3989029278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: failed to encode map e112 with expected crc 2023-09-23T15:07:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:07:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T15:07:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2864292551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3989029278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: failed to encode map e112 with expected crc 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2864292551' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:07:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3989029278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:28.433 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:28] "GET /metrics HTTP/1.1" 200 33876 "" "Prometheus/2.43.0" 2023-09-23T15:07:28.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:07:28.939 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:28.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.029 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4217216229' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[215646]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:07:29.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4217216229' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.031 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:28 smithi017 ceph-mon[222878]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:29.071 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4217216229' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:28 smithi027 ceph-mon[195233]: failed to encode map e113 with expected crc 2023-09-23T15:07:29.101 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:07:29.101 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:07:29.128 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' created 2023-09-23T15:07:29.179 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:29.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:29.184 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:29.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:07:29.393 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:29.445 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:07:29.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:29.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:07:29.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:29.579 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:29.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:07:29.756 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:29.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:29.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[215646]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[215646]: failed to encode map e114 with expected crc 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[215646]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[222878]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:29.840 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[222878]: failed to encode map e114 with expected crc 2023-09-23T15:07:29.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T15:07:29.841 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:29 smithi017 ceph-mon[222878]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T15:07:29.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:29.887 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:29.932 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:29.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:29.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:30.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:30.062 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:29 smithi027 ceph-mon[195233]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:29 smithi027 ceph-mon[195233]: failed to encode map e114 with expected crc 2023-09-23T15:07:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:29 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T15:07:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:29 smithi027 ceph-mon[195233]: osdmap e114: 8 total, 8 up, 8 in 2023-09-23T15:07:30.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T15:07:30.292 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:30.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:30.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:30.345 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:30.388 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:30.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:30.429 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:30.481 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:30.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:30.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:30.616 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:30.766 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:30.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:30.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: failed to encode map e114 with expected crc 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[215646]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: failed to encode map e114 with expected crc 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.826 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:30 smithi017 ceph-mon[222878]: failed to encode map e115 with expected crc 2023-09-23T15:07:30.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:30.903 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:30.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:07:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: failed to encode map e114 with expected crc 2023-09-23T15:07:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: osdmap e115: 8 total, 8 up, 8 in 2023-09-23T15:07:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:30 smithi027 ceph-mon[195233]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:07:31.353 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:31.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:31.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:31.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:31.490 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:31.541 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:31.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:31.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:31.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:31.678 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:31.726 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:31.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:31.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:31.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:31.859 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:31.966 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[222878]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[222878]: pgmap v238: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[222878]: failed to encode map e116 with expected crc 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[222878]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[215646]: failed to encode map e115 with expected crc 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[215646]: pgmap v238: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[215646]: failed to encode map e116 with expected crc 2023-09-23T15:07:31.967 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:31 smithi017 ceph-mon[215646]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T15:07:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:31 smithi027 ceph-mon[195233]: failed to encode map e115 with expected crc 2023-09-23T15:07:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:31 smithi027 ceph-mon[195233]: pgmap v238: 89 pgs: 16 unknown, 73 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:31 smithi027 ceph-mon[195233]: failed to encode map e116 with expected crc 2023-09-23T15:07:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:31 smithi027 ceph-mon[195233]: osdmap e116: 8 total, 8 up, 8 in 2023-09-23T15:07:32.118 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:32.212 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:32.215 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:32.271 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:32.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:32.349 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:32.399 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:32.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:32.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:32.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:32.531 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:32.535 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:32.543 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:07:32.543 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:07:32.584 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:32.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:32.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:32.691 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:32.720 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:32.765 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:32.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:32.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:32.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:32.905 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:33.538 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:33.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:33.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:33.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:33.674 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:33.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:07:34.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:33 smithi017 ceph-mon[215646]: pgmap v240: 89 pgs: 15 unknown, 74 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:34.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:33 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:07:34.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:33 smithi017 ceph-mon[222878]: pgmap v240: 89 pgs: 15 unknown, 74 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:34.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:33 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:07:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:33 smithi027 ceph-mon[195233]: pgmap v240: 89 pgs: 15 unknown, 74 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:34.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:33 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:07:34.233 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:34.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:34.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:34.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:34.370 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:34.416 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:34.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:34.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:34.552 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:34.599 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:34.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:34.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:34.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:07:34.732 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:34.767 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:07:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:07:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:07:34.951 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:07:34.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:34.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:07:35.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:35.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:07:35.087 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:35.183 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:35.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:35.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:35.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:35.317 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:35.735 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:35.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:35.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:35.771 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T15:07:35.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:35.876 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:35.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: pgmap v241: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: failed to encode map e117 with expected crc 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: failed to encode map e117 with expected crc 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/493337987' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: pgmap v241: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: failed to encode map e117 with expected crc 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: failed to encode map e117 with expected crc 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/493337987' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:35.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:35 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: pgmap v241: 89 pgs: 1 unknown, 88 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3130083063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:07:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: osdmap e117: 8 total, 8 up, 8 in 2023-09-23T15:07:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/493337987' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:36.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:35 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:07:36.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:07:36.577 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:36.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:36.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:36.709 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:07:36.795 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.796 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:07:36.963 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: failed to encode map e118 with expected crc 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: failed to encode map e118 with expected crc 2023-09-23T15:07:36.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3604551087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: failed to encode map e118 with expected crc 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: failed to encode map e117 with expected crc 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: failed to encode map e118 with expected crc 2023-09-23T15:07:36.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3604551087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:36.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:36.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:37.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: failed to encode map e117 with expected crc 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: failed to encode map e118 with expected crc 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: osdmap e118: 8 total, 8 up, 8 in 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: failed to encode map e117 with expected crc 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: failed to encode map e118 with expected crc 2023-09-23T15:07:37.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3604551087' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:37.101 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:37.147 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:37.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:37.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:37.250 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:37.286 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:37.338 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:37.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:37.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:37.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:37.475 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:37.522 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:37.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:37.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:37.661 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:37.742 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:37.818 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:37.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:37.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:37.829 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T15:07:37.864 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:07:37.864 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:07:37.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:37.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:07:37.924 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:07:37.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_288048 2023-09-23T15:07:37.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:07:37.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2023-09-23T15:07:37.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:37 smithi017 ceph-mon[222878]: pgmap v244: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:37.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:37 smithi017 ceph-mon[215646]: pgmap v244: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:37.982 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:38.035 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:38.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:38.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:37 smithi027 ceph-mon[195233]: pgmap v244: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:38.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:38.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:38.267 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:38] "GET /metrics HTTP/1.1" 200 33944 "" "Prometheus/2.43.0" 2023-09-23T15:07:38.275 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:38.800 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:07:38.925 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3386 3370 210.604 210.625 0.00268075 0.00473414 2023-09-23T15:07:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:07:39.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:07:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:07:39.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:07:39.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6200 6184 192.992 175.875 0.00260905 0.00516163 2023-09-23T15:07:40.054 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: pgmap v245: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:07:40.060 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[222878]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: pgmap v245: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:07:40.061 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:39 smithi017 ceph-mon[215646]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T15:07:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:40.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: pgmap v245: 89 pgs: 89 active+clean; 4.6 MiB data, 165 MiB used, 715 GiB / 715 GiB avail 2023-09-23T15:07:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: failed to encode map e119 with expected crc 2023-09-23T15:07:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4012397330' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:07:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:39 smithi027 ceph-mon[195233]: osdmap e119: 8 total, 8 up, 8 in 2023-09-23T15:07:40.163 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:40.208 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:40.306 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:40.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:40.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:40.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:40.453 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:40.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 8122 8106 168.707 120.125 0.00433264 0.00591964 2023-09-23T15:07:41.532 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:41.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:41.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:41.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:41.668 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:41.791 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:41 smithi017 ceph-mon[222878]: pgmap v247: 89 pgs: 89 active+clean; 149 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 385 op/s 2023-09-23T15:07:41.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 10813 10797 168.573 168.188 0.00242175 0.00592597 2023-09-23T15:07:42.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:41 smithi027 ceph-mon[195233]: pgmap v247: 89 pgs: 89 active+clean; 149 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 385 op/s 2023-09-23T15:07:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:41 smithi017 ceph-mon[215646]: pgmap v247: 89 pgs: 89 active+clean; 149 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 385 op/s 2023-09-23T15:07:42.409 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:42.421 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:42.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:42.534 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:42.569 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:07:42.569 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:07:42.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:07:42.569 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:07:42.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:4349 5.5 5:a171530c:::benchmark_data_smithi017_288048_object4348:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:39.165083+0000", 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.39572188, 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.080001842000000004, 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 4349 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.570 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165083+0000", 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165083+0000", 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165082+0000", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165105+0000", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2544e-05 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165117+0000", 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.222e-05 2023-09-23T15:07:42.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165125+0000", 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1470000000000001e-06 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165171+0000", 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6480999999999998e-05 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165286+0000", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114931 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.165396+0000", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010977800000000001 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.166036+0000", 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00064002999999999998 2023-09-23T15:07:42.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.166223+0000", 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018662299999999999 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.244996+0000", 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078772949999999994 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.245048+0000", 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2021000000000002e-05 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:39.245085+0000", 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7598000000000002e-05 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7534 5.5 5:ad190f28:::benchmark_data_smithi017_288048_object7533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.576 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.421463+0000", 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1393425270000002, 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.180003884, 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7534 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421463+0000", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421463+0000", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421461+0000", 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:07:42.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421488+0000", 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7353999999999999e-05 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421498+0000", 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0046e-05 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421505+0000", 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1110000000000002e-06 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421585+0000", 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9581999999999998e-05 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421738+0000", 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153015 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.421898+0000", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016063300000000001 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.422952+0000", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010538889999999999 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423313+0000", 2023-09-23T15:07:42.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036106499999999999 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425071+0000", 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001757783 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601418+0000", 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17634649799999999 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601467+0000", 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8977999999999998e-05 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7539 5.3 5:c9b17a4d:::benchmark_data_smithi017_288048_object7538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.423005+0000", 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.137800028, 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17862285999999999, 2023-09-23T15:07:42.583 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7539 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423005+0000", 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423005+0000", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423003+0000", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423032+0000", 2023-09-23T15:07:42.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8242000000000002e-05 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423046+0000", 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4600000000000001e-05 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423054+0000", 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6299999999999998e-06 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423138+0000", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4468999999999994e-05 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423302+0000", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016315700000000001 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.423477+0000", 2023-09-23T15:07:42.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175897 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424097+0000", 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00061980300000000004 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601512+0000", 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177414289 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601548+0000", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6554000000000003e-05 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601591+0000", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3084000000000003e-05 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601628+0000", 2023-09-23T15:07:42.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6934999999999998e-05 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:7541 5.e e119/114 5:7a7390bc:::benchmark_data_smithi017_288048_object7540:head v 119'459, mlcod=119'456)", 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.424207+0000", 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1365986600000002, 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177219087, 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.590 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424207+0000", 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424207+0000", 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424206+0000", 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424268+0000", 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2430999999999999e-05 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424279+0000", 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1018e-05 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424287+0000", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6569999999999997e-06 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424326+0000", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9663000000000003e-05 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424345+0000", 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8838000000000002e-05 2023-09-23T15:07:42.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425128+0000", 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078266499999999999 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601426+0000", 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17629777199999999 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.594 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:7534 5.5 e119/114 ondisk, result = 0)", 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.424807+0000", 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1359984029999999, 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17666841999999999, 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424807+0000", 2023-09-23T15:07:42.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424807+0000", 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424806+0000", 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424814+0000", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0839999999999992e-06 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424825+0000", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1197e-05 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424833+0000", 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2790000000000001e-06 2023-09-23T15:07:42.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425058+0000", 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022501899999999999 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425065+0000", 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3170000000000004e-06 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601475+0000", 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.176410226 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:7547 5.c e119/114 5:32932e1e:::benchmark_data_smithi017_288048_object7546:head v 119'487, mlcod=119'485)", 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.424863+0000", 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.135942268, 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.178111245, 2023-09-23T15:07:42.599 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424863+0000", 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424863+0000", 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424863+0000", 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424893+0000", 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0031000000000001e-05 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424902+0000", 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7890000000000006e-06 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424909+0000", 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.173e-06 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425062+0000", 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153449 2023-09-23T15:07:42.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425081+0000", 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9425000000000001e-05 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602935+0000", 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17785326500000001 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602974+0000", 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9560000000000001e-05 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:7537 5.b e119/114 5:d4acac37:::benchmark_data_smithi017_288048_object7536:head v 119'471, mlcod=119'469)", 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.424966+0000", 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1358391769999998, 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17771527600000001, 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.604 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424966+0000", 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424966+0000", 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424966+0000", 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424989+0000", 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3028e-05 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424995+0000", 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6579999999999996e-06 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.424999+0000", 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8349999999999997e-06 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425036+0000", 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7159000000000001e-05 2023-09-23T15:07:42.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425056+0000", 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9746999999999999e-05 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602643+0000", 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177586788 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602681+0000", 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.862e-05 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:7540 5.7 e119/114 5:e55b1646:::benchmark_data_smithi017_288048_object7539:head v 119'466, mlcod=119'464)", 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.425392+0000", 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1354130310000001, 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17715596, 2023-09-23T15:07:42.609 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425392+0000", 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425392+0000", 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425392+0000", 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425435+0000", 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3302e-05 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601372+0000", 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17593688099999999 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601382+0000", 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9369999999999999e-06 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601441+0000", 2023-09-23T15:07:42.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9079000000000001e-05 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601462+0000", 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1016999999999999e-05 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602505+0000", 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001043277 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602548+0000", 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3115000000000003e-05 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:7550 5.8 e119/114 5:1db29e59:::benchmark_data_smithi017_288048_object7549:head v 119'461, mlcod=119'459)", 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.425730+0000", 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1350752719999999, 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.176793858, 2023-09-23T15:07:42.614 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425730+0000", 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425730+0000", 2023-09-23T15:07:42.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425729+0000", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.425760+0000", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0720999999999999e-05 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601350+0000", 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17558976400000001 2023-09-23T15:07:42.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601368+0000", 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7941e-05 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601424+0000", 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6529e-05 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601454+0000", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.974e-05 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602493+0000", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001038913 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.602524+0000", 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0969000000000001e-05 2023-09-23T15:07:42.619 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7554 5.5 5:a067f1b8:::benchmark_data_smithi017_288048_object7553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.427701+0000", 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 2.1331043689999998, 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177987489, 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.620 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7554 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.427701+0000", 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.427701+0000", 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.427699+0000", 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.427726+0000", 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7492e-05 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601352+0000", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17362581999999999 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601372+0000", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9530000000000001e-05 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601502+0000", 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012988499999999999 2023-09-23T15:07:42.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601662+0000", 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016034299999999999 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.601814+0000", 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015207299999999999 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.603050+0000", 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012363579999999999 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.603061+0000", 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0341e-05 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.605599+0000", 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002538621 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.605653+0000", 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3285000000000002e-05 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.605688+0000", 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5914000000000002e-05 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7789 5.5 5:a68cf7be:::benchmark_data_smithi017_288048_object7788:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.663639+0000", 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.897166575, 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.118838977, 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7789 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663639+0000", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663639+0000", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663638+0000", 2023-09-23T15:07:42.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663659+0000", 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1219000000000001e-05 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663669+0000", 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0057e-05 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663676+0000", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3590000000000002e-06 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663713+0000", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6363999999999999e-05 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.663859+0000", 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014632800000000001 2023-09-23T15:07:42.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.664012+0000", 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015262699999999999 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.665479+0000", 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001467248 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.665733+0000", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025402600000000002 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782371+0000", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11663805200000001 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782437+0000", 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5681999999999999e-05 2023-09-23T15:07:42.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782478+0000", 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0920999999999997e-05 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7821 5.5 5:a0b6e0e9:::benchmark_data_smithi017_288048_object7820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.673843+0000", 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.886961925, 2023-09-23T15:07:42.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10913365999999999, 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7821 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673843+0000", 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673843+0000", 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673842+0000", 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673866+0000", 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3589000000000001e-05 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673886+0000", 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0530000000000002e-05 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673895+0000", 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9430000000000006e-06 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.673946+0000", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1325000000000001e-05 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674063+0000", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116928 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674162+0000", 2023-09-23T15:07:42.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9059000000000001e-05 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.675064+0000", 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00090135599999999997 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.675110+0000", 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.634e-05 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782888+0000", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10777811 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782939+0000", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0977999999999999e-05 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.782977+0000", 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7789e-05 2023-09-23T15:07:42.639 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:7822 5.5 5:a295a874:::benchmark_data_smithi017_288048_object7821:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:40.674270+0000", 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.886535758, 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10881571600000001, 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.640 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 7822 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674270+0000", 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674270+0000", 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674269+0000", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674305+0000", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5831999999999998e-05 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674319+0000", 2023-09-23T15:07:42.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4129e-05 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674327+0000", 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7980000000000006e-06 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674579+0000", 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025221100000000002 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674705+0000", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012627499999999999 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.674809+0000", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103589 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.675192+0000", 2023-09-23T15:07:42.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038339899999999998 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.675718+0000", 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052644499999999995 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.783013+0000", 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10729488299999999 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.783053+0000", 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9635000000000003e-05 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:40.783085+0000", 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2326999999999997e-05 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:10060 5.3 5:c4456337:::benchmark_data_smithi017_288048_object10059:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.546435+0000", 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.014369874, 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.159774526, 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 10060 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.647 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546435+0000", 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546435+0000", 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546434+0000", 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546459+0000", 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4358999999999998e-05 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546471+0000", 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2019e-05 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546479+0000", 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1100000000000003e-06 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546533+0000", 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3832000000000002e-05 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546669+0000", 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013621 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546845+0000", 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175568 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548194+0000", 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001349774 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706061+0000", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15786656600000001 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706117+0000", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6310000000000001e-05 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706170+0000", 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2500000000000002e-05 2023-09-23T15:07:42.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706210+0000", 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0234000000000001e-05 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:10057 5.2 e119/114 5:49e47c4a:::benchmark_data_smithi017_288048_object10056:head v 119'682, mlcod=119'679)", 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.546668+0000", 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.014137152, 2023-09-23T15:07:42.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15935723399999999, 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546668+0000", 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546668+0000", 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546668+0000", 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546761+0000", 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3552000000000001e-05 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546769+0000", 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8609999999999998e-06 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546775+0000", 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5740000000000003e-06 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546816+0000", 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1975999999999999e-05 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546837+0000", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0806999999999999e-05 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548638+0000", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001801064 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706025+0000", 2023-09-23T15:07:42.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.157387053 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:10063 5.5 5:a0f0443c:::benchmark_data_smithi017_288048_object10062:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.546987+0000", 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.013818085, 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.160149608, 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.658 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 10063 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546987+0000", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546987+0000", 2023-09-23T15:07:42.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.546986+0000", 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547006+0000", 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0017000000000002e-05 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547018+0000", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1158000000000001e-05 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547025+0000", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4479999999999997e-06 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547072+0000", 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6904999999999999e-05 2023-09-23T15:07:42.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547183+0000", 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011100800000000001 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547316+0000", 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013343900000000001 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548331+0000", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001014332 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706071+0000", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15774016900000001 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.707032+0000", 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00096139100000000003 2023-09-23T15:07:42.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.707099+0000", 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6785000000000005e-05 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.707137+0000", 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7784000000000003e-05 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:10066 5.b e119/114 5:deea5dc2:::benchmark_data_smithi017_288048_object10065:head v 119'619, mlcod=119'617)", 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.547912+0000", 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.0128936049999999, 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15814087700000001, 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547912+0000", 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547912+0000", 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547911+0000", 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547933+0000", 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1696e-05 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547942+0000", 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5170000000000005e-06 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547948+0000", 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1029999999999998e-06 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.547990+0000", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2200999999999998e-05 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548009+0000", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8383999999999999e-05 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706012+0000", 2023-09-23T15:07:42.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15800287299999999 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706053+0000", 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0951999999999997e-05 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:10060 5.3 e119/114 ondisk, result = 0)", 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.548684+0000", 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.012121641, 2023-09-23T15:07:42.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15738054900000001, 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548684+0000", 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548684+0000", 2023-09-23T15:07:42.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548683+0000", 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548688+0000", 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6489999999999999e-06 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.705993+0000", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.157304949 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706010+0000", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6963e-05 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706046+0000", 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6055e-05 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706054+0000", 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3599999999999996e-06 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706064+0000", 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0159999999999999e-05 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:10063 5.5 e119/114 ondisk, result = 0)", 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.548971+0000", 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.0118340429999999, 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.157102926, 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548971+0000", 2023-09-23T15:07:42.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548971+0000", 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548971+0000", 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.548974+0000", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.659e-06 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.705993+0000", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15701833600000001 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706009+0000", 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5903e-05 2023-09-23T15:07:42.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706055+0000", 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6675000000000003e-05 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706064+0000", 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4759999999999998e-06 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706074+0000", 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0424999999999999e-05 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:10071 5.5 5:ab9f8d98:::benchmark_data_smithi017_288048_object10070:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:41.549232+0000", 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.011573072, 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.159054785, 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:07:42.678 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 10071 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.549232+0000", 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:07:42.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.549232+0000", 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.549231+0000", 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.549256+0000", 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5452e-05 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.705995+0000", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15673883799999999 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706010+0000", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5486e-05 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706108+0000", 2023-09-23T15:07:42.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7898000000000004e-05 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706284+0000", 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017537499999999999 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.706464+0000", 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000180638 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.707592+0000", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011273089999999999 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.707836+0000", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000244143 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.708203+0000", 2023-09-23T15:07:42.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00036710600000000002 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.708252+0000", 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9274000000000003e-05 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:41.708287+0000", 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4851000000000002e-05 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:07:42.685 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:07:42.685 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:07:42.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:42.763 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:42.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:42.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:42.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:42.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 13505 13489 168.505 168.25 0.00702811 0.00592805 2023-09-23T15:07:42.931 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:43.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:07:43.458 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:43.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:43.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:43.638 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:43.929 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 15918 15902 165.554 150.812 0.00163667 0.00598053 2023-09-23T15:07:43.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[215646]: pgmap v248: 89 pgs: 89 active+clean; 180 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2023-09-23T15:07:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:43.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[222878]: pgmap v248: 89 pgs: 89 active+clean; 180 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2023-09-23T15:07:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:44.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:43 smithi027 ceph-mon[195233]: pgmap v248: 89 pgs: 89 active+clean; 180 MiB data, 629 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s wr, 381 op/s 2023-09-23T15:07:44.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:44.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:44.408 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:44.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:44.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:44.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:07:44.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:44.551 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:44.625 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:44.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:44.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:44.766 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:44.832 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:44.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:44.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:44.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 18035 18019 160.806 132.312 0.00458462 0.00621217 2023-09-23T15:07:44.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:44.996 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:45.054 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:45.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:45.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:45.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:45.189 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:45.745 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:45.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:45.749 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:45.820 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:07:45.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:45.882 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:45.893 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:45.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 19826 19810 154.697 111.938 0.00221415 0.00645688 2023-09-23T15:07:45.961 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:45.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:45.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:46.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[222878]: pgmap v249: 89 pgs: 89 active+clean; 862 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T15:07:46.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1816132045' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[215646]: pgmap v249: 89 pgs: 89 active+clean; 862 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T15:07:46.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1816132045' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:45 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:45 smithi027 ceph-mon[195233]: pgmap v249: 89 pgs: 89 active+clean; 862 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 103 MiB/s wr, 1.64k op/s 2023-09-23T15:07:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1816132045' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:45 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:07:46.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:46.886 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:46.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:46.918 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:46.928 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 22006 21990 152.646 136.25 0.00864613 0.00654461 2023-09-23T15:07:46.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:07:46.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:47.072 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:47.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:07:47.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T15:07:47.075 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[215646]: pgmap v251: 89 pgs: 89 active+clean; 952 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.076 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:46 smithi017 ceph-mon[222878]: pgmap v251: 89 pgs: 89 active+clean; 952 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T15:07:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:07:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: osdmap e120: 8 total, 8 up, 8 in 2023-09-23T15:07:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: failed to encode map e120 with expected crc 2023-09-23T15:07:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:46 smithi027 ceph-mon[195233]: pgmap v251: 89 pgs: 89 active+clean; 952 MiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2023-09-23T15:07:47.259 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:47.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:47.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:47.445 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:47.570 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:07:47.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:07:47.655 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:47.657 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:07:47.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:47.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:47.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:07:47.734 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:07:47.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:47.774 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:07:47.811 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:07:47.815 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:47.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:07:47.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 23967 23951 149.616 122.562 0.00323791 0.00667926 2023-09-23T15:07:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3274676585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:48.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:48] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2023-09-23T15:07:48.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3274676585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:48.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3274676585' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:48.866 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:48.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:48.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:48.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 25526 25510 144.873 97.4375 0.00524735 0.00689646 2023-09-23T15:07:48.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:48 smithi017 ceph-mon[222878]: pgmap v252: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:48.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:48 smithi017 ceph-mon[215646]: pgmap v252: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:48.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:49.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:49.061 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:49.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:48 smithi027 ceph-mon[195233]: pgmap v252: 89 pgs: 89 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:49.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:49.228 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:49.553 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:49.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:49.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:49.714 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:49.782 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:49.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:49.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:49.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 27277 27261 141.921 109.438 0.0119382 0.00704048 2023-09-23T15:07:49.953 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:50.048 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:50.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:50.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:07:50.236 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:50.371 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:07:50.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:50.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:07:50.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:50.557 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:50.626 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:50.670 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:50.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:50.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:50.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:50.860 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:50.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 29208 29192 140.287 120.688 0.0196765 0.00712163 2023-09-23T15:07:51.068 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:51.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:51.077 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:51.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:51.262 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:51.373 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:07:51.373 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:07:51.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:51 smithi017 ceph-mon[215646]: pgmap v253: 89 pgs: 89 active+clean; 1.5 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:07:51.487 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:51 smithi017 ceph-mon[222878]: pgmap v253: 89 pgs: 89 active+clean; 1.5 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:07:51.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:51 smithi027 ceph-mon[195233]: pgmap v253: 89 pgs: 89 active+clean; 1.5 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:07:51.660 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:51.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:51.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:51.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:51.821 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:51.930 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 31149 31133 138.931 121.312 0.00514859 0.00719295 2023-09-23T15:07:52.229 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:07:52.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:07:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:07:52.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:07:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:07:52.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 33139 33123 137.961 124.375 0.00168569 0.00724054 2023-09-23T15:07:53.282 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:07:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:53.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:07:53.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:53.447 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:53.490 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: pgmap v254: 89 pgs: 89 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:07:53.495 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:53 smithi027 ceph-mon[195233]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T15:07:53.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:53.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:53.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: pgmap v254: 89 pgs: 89 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:07:53.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[215646]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: pgmap v254: 89 pgs: 89 active+clean; 1.5 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: failed to encode map e121 with expected crc 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1299604459' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:07:53.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:53 smithi017 ceph-mon[222878]: osdmap e121: 8 total, 8 up, 8 in 2023-09-23T15:07:53.650 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:53.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 34740 34724 135.592 100.062 0.0065552 0.00737016 2023-09-23T15:07:54.331 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:54.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:54.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:54.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:07:54.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:54.468 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:54.485 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:54.554 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:54.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:54.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:54.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:07:54.712 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:54.799 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:07:54.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:54.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:07:54.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:54.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 36526 36510 134.181 111.625 0.00633571 0.007446 2023-09-23T15:07:54.997 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:55.278 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:55.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:55.285 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:55.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:07:55.426 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:55.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:55 smithi017 ceph-mon[215646]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2023-09-23T15:07:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:55 smithi017 ceph-mon[222878]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2023-09-23T15:07:55.501 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:07:55.506 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:55 smithi027 ceph-mon[195233]: pgmap v256: 89 pgs: 89 active+clean; 2.1 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 140 MiB/s wr, 2.25k op/s 2023-09-23T15:07:55.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:55.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:07:55.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:07:55.677 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:55.775 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:07:55.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:55.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:07:55.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:07:55.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 38142 38126 132.338 101 0.00833357 0.00755136 2023-09-23T15:07:55.940 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:56.013 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:07:56.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:56.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:07:56.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:56.146 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:56.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:07:56.346 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:07:56.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3550382441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3550382441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:56 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.527 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:56.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:56 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:56 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3550382441' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:56 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:07:56.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:56.694 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:56.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 39192 39176 128.827 65.625 0.00195154 0.00772351 2023-09-23T15:07:57.296 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:57.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:57.364 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:57.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: pgmap v257: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T15:07:57.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:07:57.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:07:57.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:57.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:07:57.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: pgmap v257: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:57.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:07:57.416 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:57.486 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: pgmap v257: 89 pgs: 89 active+clean; 2.1 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 121 MiB/s wr, 1.93k op/s 2023-09-23T15:07:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: failed to encode map e122 with expected crc 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: osdmap e122: 8 total, 8 up, 8 in 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:07:57.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:07:57.714 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:07:57.739 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:57.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:57.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:57.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:07:57.884 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:07:57.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:07:57.931920+0000 min lat: 0.00107351 max lat: 0.239852 avg lat: 0.00786141 2023-09-23T15:07:57.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:07:57.932 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 40706 40690 127.117 94.625 0.00421908 0.00786141 2023-09-23T15:07:57.949 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:07:57.955 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:57.991 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:07:58.048 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:07:58.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:07:58.116 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:07:58.156 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:07:58.196 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:07:58.337 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:07:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:07:58] "GET /metrics HTTP/1.1" 200 36292 "" "Prometheus/2.43.0" 2023-09-23T15:07:58.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2377622391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:58.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:58.525 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:07:58.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:58.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:07:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2377622391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:58.631 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:07:58.656 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2377622391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:07:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:07:58.707 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:07:58.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:07:58.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:07:58.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:07:58.841 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:07:58.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 41937 41921 124.727 76.9375 0.00340475 0.00792474 2023-09-23T15:07:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:07:59 smithi027 ceph-mon[195233]: pgmap v259: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:07:59.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:07:59 smithi017 ceph-mon[215646]: pgmap v259: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:07:59.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:07:59 smithi017 ceph-mon[222878]: pgmap v259: 89 pgs: 89 active+clean; 2.3 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:07:59.931 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 43558 43542 123.663 101.312 0.0083907 0.00808116 2023-09-23T15:08:00.038 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:00.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:00.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:00.222 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:00.414 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:00.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:00.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:00.587 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:00.691 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:00.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:00.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:00.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:00.860 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:00.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 45253 45237 122.87 105.938 0.00343723 0.00813229 2023-09-23T15:08:01.231 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:01.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:01.323 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:08:01.323 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:08:01.323 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:08:01.323 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:08:01.323 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:08:01.333 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:01.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:01.409 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:01 smithi017 ceph-mon[222878]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T15:08:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:01 smithi017 ceph-mon[215646]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T15:08:01.817 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:01.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:01.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:01.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:01 smithi027 ceph-mon[195233]: pgmap v260: 89 pgs: 89 active+clean; 2.5 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 126 MiB/s wr, 2.02k op/s 2023-09-23T15:08:01.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:01.935 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 46779 46763 121.724 95.375 0.00202648 0.00819206 2023-09-23T15:08:01.948 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:01.991 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:01.996 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:02.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:02.168 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:02.403 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:02.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:02.410 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:02.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:08:02.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:02.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:08:02.627 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:02.805 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:02.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:02.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:02.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:02.936 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 47950 47934 119.779 73.1875 0.014347 0.00834329 2023-09-23T15:08:02.961 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:03.039 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:03.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:03.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:03.191 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:03.281 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:03.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:03.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:03.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:03.416 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:03.775 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:03.782 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:08:03.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:03.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:03.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:03.927 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:03.937 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 49375 49359 118.598 89.0625 0.0147881 0.00842703 2023-09-23T15:08:04.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[222878]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T15:08:04.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2607770147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[215646]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T15:08:04.018 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2607770147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.018 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.024 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:04.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:04.026 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:03 smithi027 ceph-mon[195233]: pgmap v261: 89 pgs: 89 active+clean; 2.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2023-09-23T15:08:04.026 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2607770147' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.026 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:04.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:04.174 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:04.629 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:04.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:04.720 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:04.736 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:04.783 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:04.874 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:04.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:04.937 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 50742 50726 117.37 85.4375 0.011645 0.00851457 2023-09-23T15:08:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:08:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[222878]: failed to encode map e123 with expected crc 2023-09-23T15:08:04.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:04.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[222878]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T15:08:04.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:04.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:08:04.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[215646]: failed to encode map e123 with expected crc 2023-09-23T15:08:04.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:04.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:04 smithi017 ceph-mon[215646]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T15:08:04.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:05.045 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:05.052 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:04 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:04 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:08:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:04 smithi027 ceph-mon[195233]: failed to encode map e123 with expected crc 2023-09-23T15:08:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:04 smithi027 ceph-mon[195233]: osdmap e123: 8 total, 8 up, 8 in 2023-09-23T15:08:05.198 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:05.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:05.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:05.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:05.350 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:05.430 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:05.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:05.434 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:05.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:05.629 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:05.796 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:08:05.938 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 52134 52118 116.281 87 0.0474499 0.00858798 2023-09-23T15:08:06.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:05 smithi027 ceph-mon[195233]: pgmap v263: 89 pgs: 89 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T15:08:06.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4101982003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[222878]: pgmap v263: 89 pgs: 89 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T15:08:06.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4101982003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[215646]: pgmap v263: 89 pgs: 89 active+clean; 3.0 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s 2023-09-23T15:08:06.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4101982003' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.107 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:08:06.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:08:06.870 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:06.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:06.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:06.938 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 53581 53565 115.39 90.4375 0.0056701 0.00866139 2023-09-23T15:08:06.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:07.032 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:07.098 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:06 smithi027 ceph-mon[195233]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:06 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:08:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:06 smithi027 ceph-mon[195233]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T15:08:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:06 smithi027 ceph-mon[195233]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:07.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[215646]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[215646]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[215646]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[222878]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:08:07.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[222878]: osdmap e124: 8 total, 8 up, 8 in 2023-09-23T15:08:07.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:06 smithi017 ceph-mon[222878]: failed to encode map e124 with expected crc 2023-09-23T15:08:07.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:07.242 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:07.322 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:07.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:07.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:07.498 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:07.597 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:07.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:07.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:07.751 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:07.938 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 54653 54637 113.778 67 0.00874274 0.00878383 2023-09-23T15:08:07.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:08:08.017 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:08.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:08.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:07 smithi017 ceph-mon[215646]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:08:08.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:07 smithi017 ceph-mon[222878]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:08:08.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:07 smithi027 ceph-mon[195233]: pgmap v265: 89 pgs: 89 active+clean; 3.0 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2023-09-23T15:08:08.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:08.172 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:08.249 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:08.256 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:08.324 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:08] "GET /metrics HTTP/1.1" 200 36297 "" "Prometheus/2.43.0" 2023-09-23T15:08:08.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:08.401 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:08.478 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:08.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:08.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:08.530 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:08.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:08.682 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:08.690 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:08.728 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:08:08.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:08.798 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:08.834 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:08.880 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:08.910 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1096766611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:08.911 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1096766611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:08.921 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:08.938 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 55878 55862 112.575 76.5625 0.0129813 0.00887697 2023-09-23T15:08:08.960 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.981 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:08:08.990 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:08.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:08.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:09.010 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:08:09.010 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:08:09.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:09.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1096766611' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:09.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:09.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 57161 57145 111.563 80.1875 0.0132125 0.00895887 2023-09-23T15:08:09.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:09 smithi017 ceph-mon[215646]: pgmap v266: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:08:09.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:09 smithi017 ceph-mon[222878]: pgmap v266: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:08:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:09 smithi027 ceph-mon[195233]: pgmap v266: 89 pgs: 89 active+clean; 3.1 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:08:10.539 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:10.637 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:10.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:10.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:10.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:08:10.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:10.808 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:10.902 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:10.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:10.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:10.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 58356 58340 110.446 74.6875 0.00917662 0.00904968 2023-09-23T15:08:11.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:11.072 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:11.185 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:11.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:11.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:11.349 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:11.469 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:11.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:11.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:11.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:11.637 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:11.770 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:11.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:11.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:11.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:11.919 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:11.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 59608 59592 109.499 78.25 0.0110983 0.00912702 2023-09-23T15:08:11.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:11 smithi017 ceph-mon[215646]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:08:11.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:11 smithi017 ceph-mon[222878]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:08:12.017 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:12.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:12.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:12.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:11 smithi027 ceph-mon[195233]: pgmap v267: 89 pgs: 89 active+clean; 3.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:08:12.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:12.183 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:12.303 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:12.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:12.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:12.433 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:12.511 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:12.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:12.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:12.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:12.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:12.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:08:12.908 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:12.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:12.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:12 smithi017 ceph-mon[222878]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:12.930 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:12 smithi017 ceph-mon[215646]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:12.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 60812 60796 108.521 75.25 0.00617834 0.00920979 2023-09-23T15:08:12.994 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:13.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:13.055 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:12 smithi027 ceph-mon[195233]: pgmap v268: 89 pgs: 89 active+clean; 3.4 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:13.121 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:13.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:13.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:13.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:13.265 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:13.341 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:13.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:13.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:13.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:13.493 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:13.573 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:13.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:13.577 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:13.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:13.732 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:13.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:13.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:13.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:13.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:13.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 62003 61987 107.574 74.4375 0.00660409 0.00929124 2023-09-23T15:08:13.996 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:14.097 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:14.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:14.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:14.140 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:08:14.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:14.235 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:14.278 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:14.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:14.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:14.422 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:14.422 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:14.422 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:14.424 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:14.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:14.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:14.535 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:14.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:14.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:14.702 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:14.824 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:14.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:14.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:14.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:14.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 63108 63092 106.533 69.0625 0.0161134 0.00938231 2023-09-23T15:08:14.980 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:15.128 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:08:15.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:15.438 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:15.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:15.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:15.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T15:08:15.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: pgmap v270: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: pgmap v270: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T15:08:15.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:08:15.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:15.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:15.549 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:15.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/97688850' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: osdmap e125: 8 total, 8 up, 8 in 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: failed to encode map e125 with expected crc 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: pgmap v270: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2023-09-23T15:08:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:08:15.614 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:15.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 63928 63912 105.079 51.25 0.00992044 0.00951226 2023-09-23T15:08:16.201 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:16.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:16.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:16.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:16.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[215646]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:08:16.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[215646]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T15:08:16.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[215646]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.327 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[215646]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[222878]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:08:16.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[222878]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T15:08:16.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[222878]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.328 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:16 smithi017 ceph-mon[222878]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.379 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:16.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:08:16.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:16 smithi027 ceph-mon[195233]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/366689586' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:08:16.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:16 smithi027 ceph-mon[195233]: osdmap e126: 8 total, 8 up, 8 in 2023-09-23T15:08:16.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:16 smithi027 ceph-mon[195233]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:16 smithi027 ceph-mon[195233]: failed to encode map e126 with expected crc 2023-09-23T15:08:16.637 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:16.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:16.685 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:08:16.685 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:08:16.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:16.791 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:16.899 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:16.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:16.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:16.939 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 65174 65158 104.381 77.875 0.0145957 0.00957545 2023-09-23T15:08:17.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:17.072 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:17.170 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:17.175 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:17.275 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:17.323 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:17.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:08:17.940584+0000 min lat: 0.00107351 max lat: 0.291013 avg lat: 0.00966221 2023-09-23T15:08:17.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:08:17.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 66163 66147 103.316 61.8125 0.0874121 0.00966221 2023-09-23T15:08:18.017 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:08:18.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:17 smithi017 ceph-mon[222878]: pgmap v272: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T15:08:18.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:17 smithi017 ceph-mon[215646]: pgmap v272: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T15:08:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:17 smithi027 ceph-mon[195233]: pgmap v272: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2023-09-23T15:08:18.249 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:18.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:18.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:18.284 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:18] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:08:18.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:08:18.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:18.398 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:18.468 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:18.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:18.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:18.523 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:18.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:18.676 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:18.792 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:18.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:18.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:18.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:18.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 67342 67326 102.593 73.6875 0.013733 0.00974302 2023-09-23T15:08:18.947 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:19.039 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:19.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:19.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:19.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:19.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:19.247 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:19.251 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:08:19.258 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:19.286 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:19.327 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:19.367 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:19.400 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:19.425 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:19.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:19.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:19.438 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:19.470 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:19.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:08:19.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:19.601 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:19.702 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:19.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:19.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:19.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:19.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:19 smithi027 ceph-mon[195233]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:08:19.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/286519329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:19.869 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:19.940 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 68511 68495 101.89 73.0625 0.0107679 0.00980968 2023-09-23T15:08:20.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:19 smithi017 ceph-mon[222878]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:08:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/286519329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:20.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:19 smithi017 ceph-mon[215646]: pgmap v273: 89 pgs: 89 active+clean; 3.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:08:20.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/286519329' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:20.941 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 69503 69487 100.959 62 0.0137041 0.00990094 2023-09-23T15:08:20.980 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:20.993 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:21.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:21.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:21.157 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:21.241 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:21.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:21.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:21.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:21.397 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:21.477 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:21.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:21.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:21.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:21.642 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:21.747 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:21.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:21.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:21.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:21.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:21 smithi017 ceph-mon[215646]: pgmap v274: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:21.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:21 smithi017 ceph-mon[222878]: pgmap v274: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:21.927 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:21.941 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 70668 70652 100.32 72.8125 0.00377309 0.00996155 2023-09-23T15:08:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:21 smithi027 ceph-mon[195233]: pgmap v274: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T15:08:22.776 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:08:22.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 71575 71559 99.3503 56.6875 0.00932473 0.0100609 2023-09-23T15:08:22.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3892551262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:22.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3892551262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:23.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:08:23.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3892551262' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:23.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:08:23.311 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:23.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:23.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:23.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:23.470 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:23.517 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:23.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:23.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:23.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:23.678 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:23.781 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:23 smithi027 ceph-mon[195233]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-09-23T15:08:23.781 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:23 smithi027 ceph-mon[195233]: failed to encode map e127 with expected crc 2023-09-23T15:08:23.781 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:23 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:08:23.781 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:23 smithi027 ceph-mon[195233]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T15:08:23.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 72529 72513 98.4868 59.625 0.0089079 0.0101476 2023-09-23T15:08:23.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[222878]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-09-23T15:08:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[222878]: failed to encode map e127 with expected crc 2023-09-23T15:08:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:08:23.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[222878]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T15:08:23.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[215646]: pgmap v275: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 48 MiB/s wr, 767 op/s 2023-09-23T15:08:23.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[215646]: failed to encode map e127 with expected crc 2023-09-23T15:08:23.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:08:23.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:23 smithi017 ceph-mon[215646]: osdmap e127: 8 total, 8 up, 8 in 2023-09-23T15:08:24.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:08:24.052 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:24.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:24.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:24.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:24.204 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:24.266 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:24.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:24.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:24.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:24.430 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:24.527 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:24.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:24.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:24.580 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:08:24.581 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:08:24.581 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:08:24.581 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:08:24.581 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:08:24.595 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:24.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:24.675 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:24.735 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:24.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:24.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:24.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:24.913 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:24.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 73576 73560 97.7837 65.4375 0.0100975 0.0102212 2023-09-23T15:08:24.998 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:25.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:25.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:25.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:25.133 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:25.512 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:25.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:25.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:25.597 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:25.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:25.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:08:25.694 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:25.816 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:25.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:25.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:25.830 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:08:25.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:25.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 74544 74528 97.007 60.5 0.0185748 0.0103026 2023-09-23T15:08:25.977 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:26.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[215646]: pgmap v277: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:08:26.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4260671530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[222878]: pgmap v277: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:08:26.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4260671530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.083 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:26.093 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:25 smithi027 ceph-mon[195233]: pgmap v277: 89 pgs: 89 active+clean; 4.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:08:26.093 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4260671530' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.093 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:26.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:26.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:26.255 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:26.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 75557 75541 96.3194 63.3125 0.00922519 0.0103776 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[222878]: pgmap v279: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:08:27.032 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:27.032 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:27.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T15:08:27.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:26 smithi017 ceph-mon[215646]: pgmap v279: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:08:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: osdmap e128: 8 total, 8 up, 8 in 2023-09-23T15:08:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: failed to encode map e128 with expected crc 2023-09-23T15:08:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:26 smithi027 ceph-mon[195233]: pgmap v279: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:08:27.262 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:27.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:27.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:27.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:27.396 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:27.567 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:27.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:27.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:27.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:27.714 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:27.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 76602 76586 95.6993 65.3125 0.00916105 0.0104441 2023-09-23T15:08:27.972 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:27.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:27.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:28.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:28.117 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:28.397 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:28.403 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:28] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:08:28.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:28.405 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:28.505 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:28.590 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:28.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:28.720 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:28.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:28.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:28.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:28.891 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:28.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 77659 77643 95.1183 66.0625 0.00732282 0.0105076 2023-09-23T15:08:28.969 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:28.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:28.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:29.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:08:29.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:29.198 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:29 smithi027 ceph-mon[195233]: pgmap v280: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T15:08:29.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:29.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:29.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[222878]: pgmap v280: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T15:08:29.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:29.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:29.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:29 smithi017 ceph-mon[215646]: pgmap v280: 89 pgs: 89 active+clean; 4.5 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s 2023-09-23T15:08:29.580 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:29.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:29.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:29.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:29.752 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:29.755 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:29.755 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:08:29.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:29.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:29.852 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:29.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:29.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:29.866 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:29.900 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:29.938 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:29.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 78664 78648 94.497 62.8125 0.00904194 0.0105767 2023-09-23T15:08:29.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:29.976 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:29.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.991 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:08:29.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:08:30.024 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:30.119 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:30.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:30.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:30.161 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:08:30.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:30.280 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:30.378 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:30.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:30.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:08:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/822884130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:08:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/822884130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:30.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:30.564 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:08:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/822884130' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:30.652 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:30.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:30.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:30.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:30.786 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:30.830 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:30.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:30.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:30.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:30.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 79631 79615 93.8545 60.4375 0.0140149 0.0106492 2023-09-23T15:08:31.004 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: pgmap v282: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[222878]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: pgmap v282: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:31 smithi017 ceph-mon[215646]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.502 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:31.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:31.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1079569942' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:08:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: osdmap e129: 8 total, 8 up, 8 in 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: pgmap v282: 89 pgs: 89 active+clean; 4.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 85 MiB/s wr, 1.35k op/s 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:31 smithi027 ceph-mon[195233]: failed to encode map e129 with expected crc 2023-09-23T15:08:31.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:31.669 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:31.797 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:31.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:31.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:31.835 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:08:31.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:08:31.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:31.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 80831 80815 93.5053 75 0.0114722 0.0106899 2023-09-23T15:08:32.020 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:32.144 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:32.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:32.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:32.246 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:32.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:32.301 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:32.384 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:32.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:32.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:32.416 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:08:32.416 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:08:32.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:32.550 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:32.653 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:32.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:32.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:32.829 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:32.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 81878 81862 92.995 65.4375 0.0102623 0.0107479 2023-09-23T15:08:32.945 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:32.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:32.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:33.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:33.088 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:33.562 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:33.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:33.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:33.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:08:33.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:33.746 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:33.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:08:33.850 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:33 smithi017 ceph-mon[215646]: pgmap v283: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2023-09-23T15:08:33.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:33 smithi017 ceph-mon[222878]: pgmap v283: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2023-09-23T15:08:33.932 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:33.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:33.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:33.942 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 82737 82721 92.2932 53.6875 0.00474094 0.0108247 2023-09-23T15:08:34.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:34.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:33 smithi027 ceph-mon[195233]: pgmap v283: 89 pgs: 89 active+clean; 4.7 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 43 MiB/s wr, 682 op/s 2023-09-23T15:08:34.126 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:34.340 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:34.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:34.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:34.436 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:34.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:34.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:34.575 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:34.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:34.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:34.678 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:34.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:34.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:34.808 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:34.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:34.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:34.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:34.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 83773 83757 91.81 64.75 0.0121794 0.0108868 2023-09-23T15:08:34.982 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:35.070 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:35.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:35.105 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:35.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:35.256 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:35.775 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:08:35.882 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:35.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:35.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:35.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 84665 84649 91.1883 55.75 0.00910451 0.0109616 2023-09-23T15:08:35.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:36.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:35 smithi017 ceph-mon[215646]: pgmap v284: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:08:36.045 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:08:36.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:35 smithi017 ceph-mon[222878]: pgmap v284: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:08:36.045 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:08:36.050 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:36.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:35 smithi027 ceph-mon[195233]: pgmap v284: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:08:36.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:08:36.386 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:36.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:36.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:36.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:36.563 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[215646]: failed to encode map e130 with expected crc 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[215646]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[222878]: failed to encode map e130 with expected crc 2023-09-23T15:08:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:08:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:36 smithi017 ceph-mon[222878]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T15:08:36.944 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 85617 85601 90.6513 59.5 0.00212209 0.0110115 2023-09-23T15:08:37.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:36 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:37.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:36 smithi027 ceph-mon[195233]: failed to encode map e130 with expected crc 2023-09-23T15:08:37.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/535023494' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:08:37.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:36 smithi027 ceph-mon[195233]: osdmap e130: 8 total, 8 up, 8 in 2023-09-23T15:08:37.531 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:37.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:37.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:37.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:37.665 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:37.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:37 smithi017 ceph-mon[215646]: pgmap v286: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T15:08:37.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:08:37.943907+0000 min lat: 0.00107351 max lat: 0.291013 avg lat: 0.011088 2023-09-23T15:08:37.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:08:37.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 86585 86569 90.1488 60.5 0.0193092 0.011088 2023-09-23T15:08:37.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:37 smithi017 ceph-mon[222878]: pgmap v286: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T15:08:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:37 smithi027 ceph-mon[195233]: pgmap v286: 89 pgs: 89 active+clean; 5.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2023-09-23T15:08:38.215 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:38.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:38.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:38.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:38.348 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:38] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-09-23T15:08:38.351 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:38.781 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:38.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:38.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:38.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:38.941 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:38.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 87479 87463 89.5868 55.875 0.0142957 0.0111574 2023-09-23T15:08:39.034 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:39.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:39.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:39.193 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:39.293 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:39.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:39.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:39.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:39.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:08:39.489 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:39.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:39.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:08:39.621 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:39.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:39.673 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:39.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:39.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:39.718 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:39.754 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:08:39.762 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:39.763 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:08:39.778 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:39.799 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:39.809 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:39.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:39.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:08:39.885 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:39.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:39.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:39.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 88357 88341 89.027 54.875 0.013355 0.0112268 2023-09-23T15:08:39.959 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:08:39.960 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:08:39.960 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:08:39.960 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:08:39.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:40.027 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:40.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:39 smithi027 ceph-mon[195233]: pgmap v287: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T15:08:40.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1734895309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:40.144 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:40.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:40.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:39 smithi017 ceph-mon[215646]: pgmap v287: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T15:08:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1734895309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:39 smithi017 ceph-mon[222878]: pgmap v287: 89 pgs: 89 active+clean; 5.1 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 54 MiB/s wr, 866 op/s 2023-09-23T15:08:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1734895309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:40.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:40.318 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:40.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 89273 89257 88.5226 57.25 0.00807355 0.0112924 2023-09-23T15:08:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:40 smithi027 ceph-mon[195233]: from='client.54640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:41.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:40 smithi017 ceph-mon[215646]: from='client.54640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:41.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:40 smithi017 ceph-mon[222878]: from='client.54640 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:41.357 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":287,"stamp":"2023-09-23T15:08:38.152543+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5514532787,"num_objects":84100,"num_object_clones":0,"num_object_copies":252300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84100,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":85218,"num_write_kb":5395749,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":85264,"ondisk_log_size":85264,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16365856,"kb_used_data":15944992,"kb_used_omap":30,"kb_used_meta":420769,"kb_avail":733595360,"statfs":{"total":767960285184,"available":751201648640,"internally_reserved":0,"allocated":16327671808,"data_stored":16323202258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":430868056},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":454885376,"num_objects":6941,"num_object_clones":0,"num_object_copies":20823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6941,"num_write_kb":444224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.012795"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":106,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799491+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:08:25.799491+0000","last_peered":"2023-09-23T15:08:25.799491+0000","last_clean":"2023-09-23T15:08:25.799491+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:08:25.799491+0000","last_undegraded":"2023-09-23T15:08:25.799491+0000","last_fullsized":"2023-09-23T15:08:25.799491+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":54,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811361+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:08:25.811361+0000","last_peered":"2023-09-23T15:08:25.811361+0000","last_clean":"2023-09-23T15:08:25.811361+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:08:25.811361+0000","last_undegraded":"2023-09-23T15:08:25.811361+0000","last_fullsized":"2023-09-23T15:08:25.811361+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.798895+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:08:25.798895+0000","last_peered":"2023-09-23T15:08:25.798895+0000","last_clean":"2023-09-23T15:08:25.798895+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:08:25.798895+0000","last_undegraded":"2023-09-23T15:08:25.798895+0000","last_fullsized":"2023-09-23T15:08:25.798895+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804440+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:08:25.804440+0000","last_peered":"2023-09-23T15:08:25.804440+0000","last_clean":"2023-09-23T15:08:25.804440+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:08:25.804440+0000","last_undegraded":"2023-09-23T15:08:25.804440+0000","last_fullsized":"2023-09-23T15:08:25.804440+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":108,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811555+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:08:25.811555+0000","last_peered":"2023-09-23T15:08:25.811555+0000","last_clean":"2023-09-23T15:08:25.811555+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:08:25.811555+0000","last_undegraded":"2023-09-23T15:08:25.811555+0000","last_fullsized":"2023-09-23T15:08:25.811555+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.812464+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:08:25.812464+0000","last_peered":"2023-09-23T15:08:25.812464+0000","last_clean":"2023-09-23T15:08:25.812464+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:08:25.812464+0000","last_undegraded":"2023-09-23T15:08:25.812464+0000","last_fullsized":"2023-09-23T15:08:25.812464+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804867+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:08:25.804867+0000","last_peered":"2023-09-23T15:08:25.804867+0000","last_clean":"2023-09-23T15:08:25.804867+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:08:25.804867+0000","last_undegraded":"2023-09-23T15:08:25.804867+0000","last_fullsized":"2023-09-23T15:08:25.804867+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157779+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:08:30.157779+0000","last_peered":"2023-09-23T15:08:30.157779+0000","last_clean":"2023-09-23T15:08:30.157779+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:08:30.157779+0000","last_undegraded":"2023-09-23T15:08:30.157779+0000","last_fullsized":"2023-09-23T15:08:30.157779+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.805177+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:08:25.805177+0000","last_peered":"2023-09-23T15:08:25.805177+0000","last_clean":"2023-09-23T15:08:25.805177+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:08:25.805177+0000","last_undegraded":"2023-09-23T15:08:25.805177+0000","last_fullsized":"2023-09-23T15:08:25.805177+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":40,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154348+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:08:30.154348+0000","last_peered":"2023-09-23T15:08:30.154348+0000","last_clean":"2023-09-23T15:08:30.154348+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:08:30.154348+0000","last_undegraded":"2023-09-23T15:08:30.154348+0000","last_fullsized":"2023-09-23T15:08:30.154348+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.813982+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:08:25.813982+0000","last_peered":"2023-09-23T15:08:25.813982+0000","last_clean":"2023-09-23T15:08:25.813982+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:08:25.813982+0000","last_undegraded":"2023-09-23T15:08:25.813982+0000","last_fullsized":"2023-09-23T15:08:25.813982+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.802205+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:08:25.802205+0000","last_peered":"2023-09-23T15:08:25.802205+0000","last_clean":"2023-09-23T15:08:25.802205+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:08:25.802205+0000","last_undegraded":"2023-09-23T15:08:25.802205+0000","last_fullsized":"2023-09-23T15:08:25.802205+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":93,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799624+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:08:25.799624+0000","last_peered":"2023-09-23T15:08:25.799624+0000","last_clean":"2023-09-23T15:08:25.799624+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:08:25.799624+0000","last_undegraded":"2023-09-23T15:08:25.799624+0000","last_fullsized":"2023-09-23T15:08:25.799624+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799740+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:08:25.799740+0000","last_peered":"2023-09-23T15:08:25.799740+0000","last_clean":"2023-09-23T15:08:25.799740+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:08:25.799740+0000","last_undegraded":"2023-09-23T15:08:25.799740+0000","last_fullsized":"2023-09-23T15:08:25.799740+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.814297+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:08:25.814297+0000","last_peered":"2023-09-23T15:08:25.814297+0000","last_clean":"2023-09-23T15:08:25.814297+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:08:25.814297+0000","last_undegraded":"2023-09-23T15:08:25.814297+0000","last_fullsized":"2023-09-23T15:08:25.814297+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157088+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:08:30.157088+0000","last_peered":"2023-09-23T15:08:30.157088+0000","last_clean":"2023-09-23T15:08:30.157088+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:08:30.157088+0000","last_undegraded":"2023-09-23T15:08:30.157088+0000","last_fullsized":"2023-09-23T15:08:30.157088+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804965+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:08:25.804965+0000","last_peered":"2023-09-23T15:08:25.804965+0000","last_clean":"2023-09-23T15:08:25.804965+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:08:25.804965+0000","last_undegraded":"2023-09-23T15:08:25.804965+0000","last_fullsized":"2023-09-23T15:08:25.804965+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.805049+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:08:25.805049+0000","last_peered":"2023-09-23T15:08:25.805049+0000","last_clean":"2023-09-23T15:08:25.805049+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:08:25.805049+0000","last_undegraded":"2023-09-23T15:08:25.805049+0000","last_fullsized":"2023-09-23T15:08:25.805049+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.813582+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:08:25.813582+0000","last_peered":"2023-09-23T15:08:25.813582+0000","last_clean":"2023-09-23T15:08:25.813582+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:08:25.813582+0000","last_undegraded":"2023-09-23T15:08:25.813582+0000","last_fullsized":"2023-09-23T15:08:25.813582+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804366+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:08:25.804366+0000","last_peered":"2023-09-23T15:08:25.804366+0000","last_clean":"2023-09-23T15:08:25.804366+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:08:25.804366+0000","last_undegraded":"2023-09-23T15:08:25.804366+0000","last_fullsized":"2023-09-23T15:08:25.804366+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811254+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:08:25.811254+0000","last_peered":"2023-09-23T15:08:25.811254+0000","last_clean":"2023-09-23T15:08:25.811254+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:08:25.811254+0000","last_undegraded":"2023-09-23T15:08:25.811254+0000","last_fullsized":"2023-09-23T15:08:25.811254+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811574+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:08:25.811574+0000","last_peered":"2023-09-23T15:08:25.811574+0000","last_clean":"2023-09-23T15:08:25.811574+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:08:25.811574+0000","last_undegraded":"2023-09-23T15:08:25.811574+0000","last_fullsized":"2023-09-23T15:08:25.811574+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"129'5250","reported_seq":5272,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.059832+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:08:34.059832+0000","last_peered":"2023-09-23T15:08:34.059832+0000","last_clean":"2023-09-23T15:08:34.059832+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:08:34.059832+0000","last_undegraded":"2023-09-23T15:08:34.059832+0000","last_fullsized":"2023-09-23T15:08:34.059832+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5250,"log_dups_size":0,"ondisk_log_size":5250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344064000,"num_objects":5250,"num_object_clones":0,"num_object_copies":15750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5250,"num_write_kb":336000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.813847+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:08:25.813847+0000","last_peered":"2023-09-23T15:08:25.813847+0000","last_clean":"2023-09-23T15:08:25.813847+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:08:25.813847+0000","last_undegraded":"2023-09-23T15:08:25.813847+0000","last_fullsized":"2023-09-23T15:08:25.813847+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804775+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:08:25.804775+0000","last_peered":"2023-09-23T15:08:25.804775+0000","last_clean":"2023-09-23T15:08:25.804775+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:08:25.804775+0000","last_undegraded":"2023-09-23T15:08:25.804775+0000","last_fullsized":"2023-09-23T15:08:25.804775+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"130'5362","reported_seq":5387,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:37.821069+0000","last_change":"2023-09-23T15:07:30.123308+0000","last_active":"2023-09-23T15:08:37.821069+0000","last_peered":"2023-09-23T15:08:37.821069+0000","last_clean":"2023-09-23T15:08:37.821069+0000","last_became_active":"2023-09-23T15:07:30.122994+0000","last_became_peered":"2023-09-23T15:07:30.122994+0000","last_unstale":"2023-09-23T15:08:37.821069+0000","last_undegraded":"2023-09-23T15:08:37.821069+0000","last_fullsized":"2023-09-23T15:08:37.821069+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5362,"log_dups_size":0,"ondisk_log_size":5362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:33.236044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351404032,"num_objects":5362,"num_object_clones":0,"num_object_copies":16086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5362,"num_write_kb":343168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.814565+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:08:25.814565+0000","last_peered":"2023-09-23T15:08:25.814565+0000","last_clean":"2023-09-23T15:08:25.814565+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:08:25.814565+0000","last_undegraded":"2023-09-23T15:08:25.814565+0000","last_fullsized":"2023-09-23T15:08:25.814565+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":104,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799637+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:08:25.799637+0000","last_peered":"2023-09-23T15:08:25.799637+0000","last_clean":"2023-09-23T15:08:25.799637+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:08:25.799637+0000","last_undegraded":"2023-09-23T15:08:25.799637+0000","last_fullsized":"2023-09-23T15:08:25.799637+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"130'5441","reported_seq":5466,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:37.801818+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:08:37.801818+0000","last_peered":"2023-09-23T15:08:37.801818+0000","last_clean":"2023-09-23T15:08:37.801818+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:08:37.801818+0000","last_undegraded":"2023-09-23T15:08:37.801818+0000","last_fullsized":"2023-09-23T15:08:37.801818+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5441,"log_dups_size":0,"ondisk_log_size":5441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356581376,"num_objects":5441,"num_object_clones":0,"num_object_copies":16323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5441,"num_write_kb":348224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.814030+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:08:25.814030+0000","last_peered":"2023-09-23T15:08:25.814030+0000","last_clean":"2023-09-23T15:08:25.814030+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:08:25.814030+0000","last_undegraded":"2023-09-23T15:08:25.814030+0000","last_fullsized":"2023-09-23T15:08:25.814030+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811273+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:08:25.811273+0000","last_peered":"2023-09-23T15:08:25.811273+0000","last_clean":"2023-09-23T15:08:25.811273+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:08:25.811273+0000","last_undegraded":"2023-09-23T15:08:25.811273+0000","last_fullsized":"2023-09-23T15:08:25.811273+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"130'5374","reported_seq":5398,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:37.816346+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:08:37.816346+0000","last_peered":"2023-09-23T15:08:37.816346+0000","last_clean":"2023-09-23T15:08:37.816346+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:08:37.816346+0000","last_undegraded":"2023-09-23T15:08:37.816346+0000","last_fullsized":"2023-09-23T15:08:37.816346+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5374,"log_dups_size":0,"ondisk_log_size":5374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5374,"num_write_kb":343936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804959+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:08:25.804959+0000","last_peered":"2023-09-23T15:08:25.804959+0000","last_clean":"2023-09-23T15:08:25.804959+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:08:25.804959+0000","last_undegraded":"2023-09-23T15:08:25.804959+0000","last_fullsized":"2023-09-23T15:08:25.804959+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.803998+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:08:25.803998+0000","last_peered":"2023-09-23T15:08:25.803998+0000","last_clean":"2023-09-23T15:08:25.803998+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:08:25.803998+0000","last_undegraded":"2023-09-23T15:08:25.803998+0000","last_fullsized":"2023-09-23T15:08:25.803998+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"130'5382","reported_seq":5407,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:37.797869+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:08:37.797869+0000","last_peered":"2023-09-23T15:08:37.797869+0000","last_clean":"2023-09-23T15:08:37.797869+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:08:37.797869+0000","last_undegraded":"2023-09-23T15:08:37.797869+0000","last_fullsized":"2023-09-23T15:08:37.797869+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5382,"log_dups_size":0,"ondisk_log_size":5382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714752,"num_objects":5382,"num_object_clones":0,"num_object_copies":16146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5382,"num_write_kb":344448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:29.146706+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:08:29.146706+0000","last_peered":"2023-09-23T15:08:29.146706+0000","last_clean":"2023-09-23T15:08:29.146706+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:08:29.146706+0000","last_undegraded":"2023-09-23T15:08:29.146706+0000","last_fullsized":"2023-09-23T15:08:29.146706+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799719+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:08:25.799719+0000","last_peered":"2023-09-23T15:08:25.799719+0000","last_clean":"2023-09-23T15:08:25.799719+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:08:25.799719+0000","last_undegraded":"2023-09-23T15:08:25.799719+0000","last_fullsized":"2023-09-23T15:08:25.799719+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"129'5291","reported_seq":5315,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.659471+0000","last_change":"2023-09-23T15:07:30.127218+0000","last_active":"2023-09-23T15:08:35.659471+0000","last_peered":"2023-09-23T15:08:35.659471+0000","last_clean":"2023-09-23T15:08:35.659471+0000","last_became_active":"2023-09-23T15:07:30.126890+0000","last_became_peered":"2023-09-23T15:07:30.126890+0000","last_unstale":"2023-09-23T15:08:35.659471+0000","last_undegraded":"2023-09-23T15:08:35.659471+0000","last_fullsized":"2023-09-23T15:08:35.659471+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5291,"log_dups_size":0,"ondisk_log_size":5291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:09.910283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346750976,"num_objects":5291,"num_object_clones":0,"num_object_copies":15873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5291,"num_write_kb":338624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811367+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:08:25.811367+0000","last_peered":"2023-09-23T15:08:25.811367+0000","last_clean":"2023-09-23T15:08:25.811367+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:08:25.811367+0000","last_undegraded":"2023-09-23T15:08:25.811367+0000","last_fullsized":"2023-09-23T15:08:25.811367+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":104,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799764+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:08:25.799764+0000","last_peered":"2023-09-23T15:08:25.799764+0000","last_clean":"2023-09-23T15:08:25.799764+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:08:25.799764+0000","last_undegraded":"2023-09-23T15:08:25.799764+0000","last_fullsized":"2023-09-23T15:08:25.799764+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"129'5238","reported_seq":5260,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.055158+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:08:34.055158+0000","last_peered":"2023-09-23T15:08:34.055158+0000","last_clean":"2023-09-23T15:08:34.055158+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:08:34.055158+0000","last_undegraded":"2023-09-23T15:08:34.055158+0000","last_fullsized":"2023-09-23T15:08:34.055158+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5238,"log_dups_size":0,"ondisk_log_size":5238,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343277568,"num_objects":5238,"num_object_clones":0,"num_object_copies":15714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5238,"num_write_kb":335232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799340+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:08:25.799340+0000","last_peered":"2023-09-23T15:08:25.799340+0000","last_clean":"2023-09-23T15:08:25.799340+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:08:25.799340+0000","last_undegraded":"2023-09-23T15:08:25.799340+0000","last_fullsized":"2023-09-23T15:08:25.799340+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799669+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:08:25.799669+0000","last_peered":"2023-09-23T15:08:25.799669+0000","last_clean":"2023-09-23T15:08:25.799669+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:08:25.799669+0000","last_undegraded":"2023-09-23T15:08:25.799669+0000","last_fullsized":"2023-09-23T15:08:25.799669+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"129'5111","reported_seq":5134,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.036004+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:08:34.036004+0000","last_peered":"2023-09-23T15:08:34.036004+0000","last_clean":"2023-09-23T15:08:34.036004+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:08:34.036004+0000","last_undegraded":"2023-09-23T15:08:34.036004+0000","last_fullsized":"2023-09-23T15:08:34.036004+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5111,"log_dups_size":0,"ondisk_log_size":5111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334954496,"num_objects":5111,"num_object_clones":0,"num_object_copies":15333,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5111,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":327104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":92,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811714+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:08:25.811714+0000","last_peered":"2023-09-23T15:08:25.811714+0000","last_clean":"2023-09-23T15:08:25.811714+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:08:25.811714+0000","last_undegraded":"2023-09-23T15:08:25.811714+0000","last_fullsized":"2023-09-23T15:08:25.811714+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.800272+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:08:25.800272+0000","last_peered":"2023-09-23T15:08:25.800272+0000","last_clean":"2023-09-23T15:08:25.800272+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:08:25.800272+0000","last_undegraded":"2023-09-23T15:08:25.800272+0000","last_fullsized":"2023-09-23T15:08:25.800272+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":377,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799453+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:08:25.799453+0000","last_peered":"2023-09-23T15:08:25.799453+0000","last_clean":"2023-09-23T15:08:25.799453+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:08:25.799453+0000","last_undegraded":"2023-09-23T15:08:25.799453+0000","last_fullsized":"2023-09-23T15:08:25.799453+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":105,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.805158+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:08:25.805158+0000","last_peered":"2023-09-23T15:08:25.805158+0000","last_clean":"2023-09-23T15:08:25.805158+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:08:25.805158+0000","last_undegraded":"2023-09-23T15:08:25.805158+0000","last_fullsized":"2023-09-23T15:08:25.805158+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154162+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:08:30.154162+0000","last_peered":"2023-09-23T15:08:30.154162+0000","last_clean":"2023-09-23T15:08:30.154162+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:08:30.154162+0000","last_undegraded":"2023-09-23T15:08:30.154162+0000","last_fullsized":"2023-09-23T15:08:30.154162+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"129'5038","reported_seq":5061,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:33.964917+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:08:33.964917+0000","last_peered":"2023-09-23T15:08:33.964917+0000","last_clean":"2023-09-23T15:08:33.964917+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:08:33.964917+0000","last_undegraded":"2023-09-23T15:08:33.964917+0000","last_fullsized":"2023-09-23T15:08:33.964917+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5038,"log_dups_size":0,"ondisk_log_size":5038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5038,"num_write_kb":322432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"129'5255","reported_seq":5278,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:33.955079+0000","last_change":"2023-09-23T15:07:30.127943+0000","last_active":"2023-09-23T15:08:33.955079+0000","last_peered":"2023-09-23T15:08:33.955079+0000","last_clean":"2023-09-23T15:08:33.955079+0000","last_became_active":"2023-09-23T15:07:30.127634+0000","last_became_peered":"2023-09-23T15:07:30.127634+0000","last_unstale":"2023-09-23T15:08:33.955079+0000","last_undegraded":"2023-09-23T15:08:33.955079+0000","last_fullsized":"2023-09-23T15:08:33.955079+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5255,"log_dups_size":0,"ondisk_log_size":5255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:26.922651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344391680,"num_objects":5255,"num_object_clones":0,"num_object_copies":15765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5255,"num_write_kb":336320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":630,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804869+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:08:25.804869+0000","last_peered":"2023-09-23T15:08:25.804869+0000","last_clean":"2023-09-23T15:08:25.804869+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:08:25.804869+0000","last_undegraded":"2023-09-23T15:08:25.804869+0000","last_fullsized":"2023-09-23T15:08:25.804869+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":88,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799710+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:08:25.799710+0000","last_peered":"2023-09-23T15:08:25.799710+0000","last_clean":"2023-09-23T15:08:25.799710+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:08:25.799710+0000","last_undegraded":"2023-09-23T15:08:25.799710+0000","last_fullsized":"2023-09-23T15:08:25.799710+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":81,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799365+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:08:25.799365+0000","last_peered":"2023-09-23T15:08:25.799365+0000","last_clean":"2023-09-23T15:08:25.799365+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:08:25.799365+0000","last_undegraded":"2023-09-23T15:08:25.799365+0000","last_fullsized":"2023-09-23T15:08:25.799365+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"130'5393","reported_seq":5418,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:37.812765+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:08:37.812765+0000","last_peered":"2023-09-23T15:08:37.812765+0000","last_clean":"2023-09-23T15:08:37.812765+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:08:37.812765+0000","last_undegraded":"2023-09-23T15:08:37.812765+0000","last_fullsized":"2023-09-23T15:08:37.812765+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5393,"log_dups_size":0,"ondisk_log_size":5393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353435648,"num_objects":5393,"num_object_clones":0,"num_object_copies":16179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5393,"num_write_kb":345152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":739,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.155138+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:08:30.155138+0000","last_peered":"2023-09-23T15:08:30.155138+0000","last_clean":"2023-09-23T15:08:30.155138+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:08:30.155138+0000","last_undegraded":"2023-09-23T15:08:30.155138+0000","last_fullsized":"2023-09-23T15:08:30.155138+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.800563+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:08:25.800563+0000","last_peered":"2023-09-23T15:08:25.800563+0000","last_clean":"2023-09-23T15:08:25.800563+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:08:25.800563+0000","last_undegraded":"2023-09-23T15:08:25.800563+0000","last_fullsized":"2023-09-23T15:08:25.800563+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.800342+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:08:25.800342+0000","last_peered":"2023-09-23T15:08:25.800342+0000","last_clean":"2023-09-23T15:08:25.800342+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:08:25.800342+0000","last_undegraded":"2023-09-23T15:08:25.800342+0000","last_fullsized":"2023-09-23T15:08:25.800342+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":374,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804837+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:08:25.804837+0000","last_peered":"2023-09-23T15:08:25.804837+0000","last_clean":"2023-09-23T15:08:25.804837+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:08:25.804837+0000","last_undegraded":"2023-09-23T15:08:25.804837+0000","last_fullsized":"2023-09-23T15:08:25.804837+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":468,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.813606+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:08:25.813606+0000","last_peered":"2023-09-23T15:08:25.813606+0000","last_clean":"2023-09-23T15:08:25.813606+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:08:25.813606+0000","last_undegraded":"2023-09-23T15:08:25.813606+0000","last_fullsized":"2023-09-23T15:08:25.813606+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":40,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.819712+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:08:25.819712+0000","last_peered":"2023-09-23T15:08:25.819712+0000","last_clean":"2023-09-23T15:08:25.819712+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:08:25.819712+0000","last_undegraded":"2023-09-23T15:08:25.819712+0000","last_fullsized":"2023-09-23T15:08:25.819712+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":69,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811680+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:08:25.811680+0000","last_peered":"2023-09-23T15:08:25.811680+0000","last_clean":"2023-09-23T15:08:25.811680+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:08:25.811680+0000","last_undegraded":"2023-09-23T15:08:25.811680+0000","last_fullsized":"2023-09-23T15:08:25.811680+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"129'5126","reported_seq":5148,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.069531+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:08:34.069531+0000","last_peered":"2023-09-23T15:08:34.069531+0000","last_clean":"2023-09-23T15:08:34.069531+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:08:34.069531+0000","last_undegraded":"2023-09-23T15:08:34.069531+0000","last_fullsized":"2023-09-23T15:08:34.069531+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335937536,"num_objects":5126,"num_object_clones":0,"num_object_copies":15378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5126,"num_write_kb":328064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"129'5254","reported_seq":5277,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.051994+0000","last_change":"2023-09-23T15:07:30.124905+0000","last_active":"2023-09-23T15:08:34.051994+0000","last_peered":"2023-09-23T15:08:34.051994+0000","last_clean":"2023-09-23T15:08:34.051994+0000","last_became_active":"2023-09-23T15:07:30.124609+0000","last_became_peered":"2023-09-23T15:07:30.124609+0000","last_unstale":"2023-09-23T15:08:34.051994+0000","last_undegraded":"2023-09-23T15:08:34.051994+0000","last_fullsized":"2023-09-23T15:08:34.051994+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5254,"log_dups_size":0,"ondisk_log_size":5254,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:24.682277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344326144,"num_objects":5254,"num_object_clones":0,"num_object_copies":15762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5254,"num_write_kb":336256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":333,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.813684+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:08:25.813684+0000","last_peered":"2023-09-23T15:08:25.813684+0000","last_clean":"2023-09-23T15:08:25.813684+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:08:25.813684+0000","last_undegraded":"2023-09-23T15:08:25.813684+0000","last_fullsized":"2023-09-23T15:08:25.813684+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.803719+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:08:25.803719+0000","last_peered":"2023-09-23T15:08:25.803719+0000","last_clean":"2023-09-23T15:08:25.803719+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:08:25.803719+0000","last_undegraded":"2023-09-23T15:08:25.803719+0000","last_fullsized":"2023-09-23T15:08:25.803719+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":85,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804570+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:08:25.804570+0000","last_peered":"2023-09-23T15:08:25.804570+0000","last_clean":"2023-09-23T15:08:25.804570+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:08:25.804570+0000","last_undegraded":"2023-09-23T15:08:25.804570+0000","last_fullsized":"2023-09-23T15:08:25.804570+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":798,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.814007+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:08:25.814007+0000","last_peered":"2023-09-23T15:08:25.814007+0000","last_clean":"2023-09-23T15:08:25.814007+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:08:25.814007+0000","last_undegraded":"2023-09-23T15:08:25.814007+0000","last_fullsized":"2023-09-23T15:08:25.814007+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":66,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799543+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:08:25.799543+0000","last_peered":"2023-09-23T15:08:25.799543+0000","last_clean":"2023-09-23T15:08:25.799543+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:08:25.799543+0000","last_undegraded":"2023-09-23T15:08:25.799543+0000","last_fullsized":"2023-09-23T15:08:25.799543+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":55,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804256+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:08:25.804256+0000","last_peered":"2023-09-23T15:08:25.804256+0000","last_clean":"2023-09-23T15:08:25.804256+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:08:25.804256+0000","last_undegraded":"2023-09-23T15:08:25.804256+0000","last_fullsized":"2023-09-23T15:08:25.804256+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"129'5272","reported_seq":5294,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.049697+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:08:34.049697+0000","last_peered":"2023-09-23T15:08:34.049697+0000","last_clean":"2023-09-23T15:08:34.049697+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:08:34.049697+0000","last_undegraded":"2023-09-23T15:08:34.049697+0000","last_fullsized":"2023-09-23T15:08:34.049697+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5272,"log_dups_size":0,"ondisk_log_size":5272,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345505792,"num_objects":5272,"num_object_clones":0,"num_object_copies":15816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5272,"num_write_kb":337408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811591+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:08:25.811591+0000","last_peered":"2023-09-23T15:08:25.811591+0000","last_clean":"2023-09-23T15:08:25.811591+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:08:25.811591+0000","last_undegraded":"2023-09-23T15:08:25.811591+0000","last_fullsized":"2023-09-23T15:08:25.811591+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":659,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.806703+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:08:25.806703+0000","last_peered":"2023-09-23T15:08:25.806703+0000","last_clean":"2023-09-23T15:08:25.806703+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:08:25.806703+0000","last_undegraded":"2023-09-23T15:08:25.806703+0000","last_fullsized":"2023-09-23T15:08:25.806703+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":86,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.811627+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:08:25.811627+0000","last_peered":"2023-09-23T15:08:25.811627+0000","last_clean":"2023-09-23T15:08:25.811627+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:08:25.811627+0000","last_undegraded":"2023-09-23T15:08:25.811627+0000","last_fullsized":"2023-09-23T15:08:25.811627+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"129'5160","reported_seq":5183,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.049258+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:08:34.049258+0000","last_peered":"2023-09-23T15:08:34.049258+0000","last_clean":"2023-09-23T15:08:34.049258+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:08:34.049258+0000","last_undegraded":"2023-09-23T15:08:34.049258+0000","last_fullsized":"2023-09-23T15:08:34.049258+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5160,"log_dups_size":0,"ondisk_log_size":5160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338165760,"num_objects":5160,"num_object_clones":0,"num_object_copies":15480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5160,"num_write_kb":330240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":559,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.812410+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:08:25.812410+0000","last_peered":"2023-09-23T15:08:25.812410+0000","last_clean":"2023-09-23T15:08:25.812410+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:08:25.812410+0000","last_undegraded":"2023-09-23T15:08:25.812410+0000","last_fullsized":"2023-09-23T15:08:25.812410+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":73,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804509+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:08:25.804509+0000","last_peered":"2023-09-23T15:08:25.804509+0000","last_clean":"2023-09-23T15:08:25.804509+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:08:25.804509+0000","last_undegraded":"2023-09-23T15:08:25.804509+0000","last_fullsized":"2023-09-23T15:08:25.804509+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799444+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:08:25.799444+0000","last_peered":"2023-09-23T15:08:25.799444+0000","last_clean":"2023-09-23T15:08:25.799444+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:08:25.799444+0000","last_undegraded":"2023-09-23T15:08:25.799444+0000","last_fullsized":"2023-09-23T15:08:25.799444+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"129'5125","reported_seq":5148,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.124821+0000","last_change":"2023-09-23T15:07:30.128190+0000","last_active":"2023-09-23T15:08:34.124821+0000","last_peered":"2023-09-23T15:08:34.124821+0000","last_clean":"2023-09-23T15:08:34.124821+0000","last_became_active":"2023-09-23T15:07:30.127665+0000","last_became_peered":"2023-09-23T15:07:30.127665+0000","last_unstale":"2023-09-23T15:08:34.124821+0000","last_undegraded":"2023-09-23T15:08:34.124821+0000","last_fullsized":"2023-09-23T15:08:34.124821+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5125,"log_dups_size":0,"ondisk_log_size":5125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:24.648384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335872000,"num_objects":5125,"num_object_clones":0,"num_object_copies":15375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5125,"num_write_kb":328000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":58,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804169+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:08:25.804169+0000","last_peered":"2023-09-23T15:08:25.804169+0000","last_clean":"2023-09-23T15:08:25.804169+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:08:25.804169+0000","last_undegraded":"2023-09-23T15:08:25.804169+0000","last_fullsized":"2023-09-23T15:08:25.804169+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.153635+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:08:30.153635+0000","last_peered":"2023-09-23T15:08:30.153635+0000","last_clean":"2023-09-23T15:08:30.153635+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:08:30.153635+0000","last_undegraded":"2023-09-23T15:08:30.153635+0000","last_fullsized":"2023-09-23T15:08:30.153635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804964+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:08:25.804964+0000","last_peered":"2023-09-23T15:08:25.804964+0000","last_clean":"2023-09-23T15:08:25.804964+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:08:25.804964+0000","last_undegraded":"2023-09-23T15:08:25.804964+0000","last_fullsized":"2023-09-23T15:08:25.804964+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":104,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799817+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:08:25.799817+0000","last_peered":"2023-09-23T15:08:25.799817+0000","last_clean":"2023-09-23T15:08:25.799817+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:08:25.799817+0000","last_undegraded":"2023-09-23T15:08:25.799817+0000","last_fullsized":"2023-09-23T15:08:25.799817+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":73,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.805178+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:08:25.805178+0000","last_peered":"2023-09-23T15:08:25.805178+0000","last_clean":"2023-09-23T15:08:25.805178+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:08:25.805178+0000","last_undegraded":"2023-09-23T15:08:25.805178+0000","last_fullsized":"2023-09-23T15:08:25.805178+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":104,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.799466+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:08:25.799466+0000","last_peered":"2023-09-23T15:08:25.799466+0000","last_clean":"2023-09-23T15:08:25.799466+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:08:25.799466+0000","last_undegraded":"2023-09-23T15:08:25.799466+0000","last_fullsized":"2023-09-23T15:08:25.799466+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.810958+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:08:25.810958+0000","last_peered":"2023-09-23T15:08:25.810958+0000","last_clean":"2023-09-23T15:08:25.810958+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:08:25.810958+0000","last_undegraded":"2023-09-23T15:08:25.810958+0000","last_fullsized":"2023-09-23T15:08:25.810958+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":111,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.810747+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:08:25.810747+0000","last_peered":"2023-09-23T15:08:25.810747+0000","last_clean":"2023-09-23T15:08:25.810747+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:08:25.810747+0000","last_undegraded":"2023-09-23T15:08:25.810747+0000","last_fullsized":"2023-09-23T15:08:25.810747+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":73,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.804811+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:08:25.804811+0000","last_peered":"2023-09-23T15:08:25.804811+0000","last_clean":"2023-09-23T15:08:25.804811+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:08:25.804811+0000","last_undegraded":"2023-09-23T15:08:25.804811+0000","last_fullsized":"2023-09-23T15:08:25.804811+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":97,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T15:08:25.800298+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:08:25.800298+0000","last_peered":"2023-09-23T15:08:25.800298+0000","last_clean":"2023-09-23T15:08:25.800298+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:08:25.800298+0000","last_undegraded":"2023-09-23T15:08:25.800298+0000","last_fullsized":"2023-09-23T15:08:25.800298+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5509742592,"num_objects":84072,"num_object_clones":0,"num_object_copies":252216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":84072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":84072,"num_write_kb":5380608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16415326208,"data_stored":16415326208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84072,"ondisk_log_size":84072,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201446,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1698628,"kb_used_data":1657788,"kb_used_omap":0,"kb_used_meta":40832,"kb_avail":92046524,"statfs":{"total":95995035648,"available":94255640576,"internally_reserved":0,"allocated":1697574912,"data_stored":1697037039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":41811968},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.64500000000000002,"15min":0.60699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.8799999999999999,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":3.7749999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.64600000000000002,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.0210000000000008,"5min":8.0210000000000008,"15min":8.0210000000000008},"last":3.9660000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":3.7490000000000001},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":3.7269999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:07:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.61499999999999999,"15min":0.61599999999999999},"min":{"1min":0.41999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":170.63800000000001},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":170.47999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.78000000000000003,"15min":0.63700000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.1980000000000004,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":3.6960000000000002},{"interface":"front","average":{"1min":1.583,"5min":0.88800000000000001,"15min":0.77200000000000002},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":7.1849999999999996,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":3.806}]},{"osd":4,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.65400000000000003,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":3.9049999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.64000000000000001,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.23999999999999999},"max":{"1min":2.972,"5min":2.972,"15min":2.972},"last":0.69699999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.66500000000000004,"15min":0.59599999999999997},"min":{"1min":0.30599999999999999,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":3.5089999999999999,"5min":3.5089999999999999,"15min":3.5089999999999999},"last":3.851},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.72599999999999998,"15min":0.625},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8639999999999999,"5min":2.8639999999999999,"15min":2.8639999999999999},"last":3.8380000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.72999999999999998,"15min":0.63400000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.7909999999999999,"5min":2.7909999999999999,"15min":2.7909999999999999},"last":4.1589999999999998},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.307},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":3.9300000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463082,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1365500,"kb_used_data":1330100,"kb_used_omap":0,"kb_used_meta":35392,"kb_avail":92379652,"statfs":{"total":95995035648,"available":94596763648,"internally_reserved":0,"allocated":1362022400,"data_stored":1361492592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":36241408},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.56200000000000006,"15min":0.55000000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.60499999999999998,"15min":0.58699999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1219999999999999,"5min":2.1219999999999999,"15min":2.1219999999999999},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.375}]},{"osd":2,"last update":"Sat Sep 23 15:07:45 2023","interfaces":[{"interface":"back","average":{"1min":8.4719999999999995,"5min":2.1890000000000001,"15min":1.1419999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":180.374,"5min":180.374,"15min":180.374},"last":0.70999999999999996},{"interface":"front","average":{"1min":8.7289999999999992,"5min":2.242,"15min":1.161},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":181.685,"5min":181.685,"15min":181.685},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.68700000000000006,"15min":0.63100000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":2.2200000000000002},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.66900000000000004,"15min":0.63},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4710000000000001,"5min":2.4710000000000001,"15min":2.4710000000000001},"last":0.49399999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:08:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.57599999999999996,"15min":0.51900000000000002},"min":{"1min":0.33200000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":2.1600000000000001,"5min":2.1600000000000001,"15min":2.1600000000000001},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.64500000000000002,"15min":0.55100000000000005},"min":{"1min":0.33800000000000002,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:08:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.60499999999999998,"15min":0.55000000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.988,"5min":1.988,"15min":1.988},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.68200000000000005,"15min":0.58799999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"last":1.151}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.69899999999999995,"15min":0.61799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.28000000000000003},"max":{"1min":2.5939999999999999,"5min":2.5939999999999999,"15min":2.5939999999999999},"last":1.0109999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.77300000000000002,"15min":0.67000000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":2.927,"5min":2.927,"15min":2.927},"last":1.1850000000000001}]}]},{"osd":1,"up_from":100,"seq":429496729643,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1737536,"kb_used_data":1683704,"kb_used_omap":6,"kb_used_meta":53817,"kb_avail":92007616,"statfs":{"total":95995035648,"available":94215798784,"internally_reserved":0,"allocated":1724112896,"data_stored":1723521755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":55108683},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":2.9390000000000001},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":0.90900000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":0.70499999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":0.95199999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.439}]}]},{"osd":0,"up_from":95,"seq":408021893169,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2715736,"kb_used_data":2639760,"kb_used_omap":8,"kb_used_meta":75959,"kb_avail":91029416,"statfs":{"total":95995035648,"available":93214121984,"internally_reserved":0,"allocated":2703114240,"data_stored":2702501090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":77782612},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":4.1909999999999998},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.46200000000000002,"15min":0.42199999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.2490000000000001,"5min":3.2490000000000001,"15min":3.2490000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.42399999999999999,"15min":0.41399999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.403,"5min":1.419,"15min":1.419},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.63700000000000001,"15min":0.52500000000000002},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.0739999999999998,"5min":6.0739999999999998,"15min":6.0739999999999998},"last":4.0899999999999999},{"interface":"front","average":{"1min":1.407,"5min":1.087,"15min":1.0329999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1020000000000003,"5min":11.978,"15min":11.978},"last":4.1050000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.79200000000000004,"15min":0.69599999999999995},"min":{"1min":0.52300000000000002,"5min":0.375,"15min":0.375},"max":{"1min":3.093,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.75,"5min":0.66300000000000003,"15min":0.64500000000000002},"min":{"1min":0.46000000000000002,"5min":0.432,"15min":0.432},"max":{"1min":1.956,"5min":1.956,"15min":1.956},"last":0.93400000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.69799999999999995,"15min":0.69699999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.84999999999999998,"15min":0.77100000000000002},"min":{"1min":0.46899999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.9780000000000002,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.70299999999999996,"15min":0.69999999999999996},"min":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3009999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67200000000000004,"15min":0.67400000000000004},"min":{"1min":0.441,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.1339999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.83499999999999996,"15min":0.754},"min":{"1min":0.54000000000000004,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.0510000000000002,"5min":8.157,"15min":8.157},"last":3.1890000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566117,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2129344,"kb_used_data":2065720,"kb_used_omap":11,"kb_used_meta":63604,"kb_avail":91615808,"statfs":{"total":95995035648,"available":93814587392,"internally_reserved":0,"allocated":2115297280,"data_stored":2114712907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":65131280},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38700000000000001,"15min":0.38500000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38200000000000001,"15min":0.38400000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.52100000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":6.0910000000000002},{"interface":"front","average":{"1min":0.379,"5min":0.38600000000000001,"15min":0.38700000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.90000000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":1.0860000000000001,"15min":0.622},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":57.695,"5min":57.695,"15min":57.695},"last":6.0540000000000003},{"interface":"front","average":{"1min":4.2069999999999999,"5min":1.1619999999999999,"15min":0.65400000000000003},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":58.387999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":0.92900000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.63300000000000001,"15min":0.64500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.64700000000000002,"15min":0.65800000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":3.504}]},{"osd":5,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.622,"15min":0.63},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.90500000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":1.6339999999999999},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.65900000000000003,"15min":0.67500000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.88800000000000001,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":0.48099999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.69499999999999995,"15min":0.71899999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.83499999999999996,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":0.436},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.68899999999999995,"15min":0.70299999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.88500000000000001,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":3.46}]},{"osd":7,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68100000000000005,"15min":0.69899999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79900000000000004,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.67400000000000004,"15min":0.68999999999999995},"min":{"1min":0.49399999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85699999999999998,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":0.83899999999999997}]}]},{"osd":3,"up_from":110,"seq":472446402590,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1021916,"kb_used_data":986260,"kb_used_omap":3,"kb_used_meta":35644,"kb_avail":92723236,"statfs":{"total":95995035648,"available":94948593664,"internally_reserved":0,"allocated":1009930240,"data_stored":1009369574,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":36499625},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.48499999999999999,"15min":0.41199999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"last":1.4350000000000001},{"interface":"front","average":{"1min":1.27,"5min":0.56399999999999995,"15min":0.44600000000000001},"min":{"1min":0.27200000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8,"5min":8,"15min":8},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":1.268},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.35899999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.67800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.8109999999999999,"5min":4.4109999999999996,"15min":4.4109999999999996},"last":4.1500000000000004},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.63800000000000001,"15min":0.49399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.0570000000000004,"5min":8.0570000000000004,"15min":8.0570000000000004},"last":1.4910000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.67800000000000005,"15min":0.625},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.423,"5min":0.877,"15min":0.78600000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.4859999999999998,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":1.0369999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.72099999999999997,"15min":0.66600000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.2509999999999999,"5min":0.89300000000000002,"15min":0.83299999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":4.4450000000000003,"5min":4.4450000000000003,"15min":4.4450000000000003},"last":0.996}]},{"osd":6,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.70399999999999996,"15min":0.68400000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.891,"5min":2.891,"15min":2.891},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.2150000000000001,"5min":0.91100000000000003,"15min":0.85999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":1.0700000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":0.92900000000000005,"15min":0.83699999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":1.1060000000000001},{"interface":"front","average":{"1min":1.026,"5min":0.72599999999999998,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":4.1660000000000004,"5min":4.1660000000000004,"15min":4.1660000000000004},"last":0.42799999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986355,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2679740,"kb_used_data":2628340,"kb_used_omap":0,"kb_used_meta":51392,"kb_avail":91065412,"statfs":{"total":95995035648,"available":93250981888,"internally_reserved":0,"allocated":2691420160,"data_stored":2690889865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":52625408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:46 2023","interfaces":[{"interface":"back","average":{"1min":7.7060000000000004,"5min":1.972,"15min":1.0149999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":122.69199999999999,"5min":122.69199999999999,"15min":122.69199999999999},"last":4.0830000000000002},{"interface":"front","average":{"1min":7.7359999999999998,"5min":1.9890000000000001,"15min":1.02},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":122.745,"5min":122.745,"15min":122.745},"last":7.8860000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":7.9560000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.57899999999999996,"15min":0.58699999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":8.7390000000000008},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.72399999999999998,"15min":0.73299999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.4890000000000001,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":8.0050000000000008}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":0.78700000000000003,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.7240000000000002,"5min":4.7240000000000002,"15min":4.7240000000000002},"last":4.0099999999999998},{"interface":"front","average":{"1min":1.1830000000000001,"5min":0.72499999999999998,"15min":0.64900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":0.61899999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:08:30 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":0.71299999999999997,"15min":0.57799999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.2309999999999999,"5min":4.2309999999999999,"15min":4.2309999999999999},"last":3.976},{"interface":"front","average":{"1min":1.46,"5min":0.76500000000000001,"15min":0.60499999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":3.8799999999999999,"5min":3.8799999999999999,"15min":3.8799999999999999},"last":0.54100000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.65500000000000003,"15min":0.61099999999999999},"min":{"1min":0.39600000000000002,"5min":0.379,"15min":0.318},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":3.9369999999999998},{"interface":"front","average":{"1min":1.008,"5min":0.70899999999999996,"15min":0.61899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33400000000000002},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:08:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.70299999999999996,"15min":0.628},"min":{"1min":0.376,"5min":0.376,"15min":0.28799999999999998},"max":{"1min":2.8559999999999999,"5min":2.8559999999999999,"15min":2.8559999999999999},"last":8.1980000000000004},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.75800000000000001,"15min":0.68899999999999995},"min":{"1min":0.47899999999999998,"5min":0.432,"15min":0.38800000000000001},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":4.1079999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724718,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3017456,"kb_used_data":2953320,"kb_used_omap":0,"kb_used_meta":64128,"kb_avail":90727696,"statfs":{"total":95995035648,"available":92905160704,"internally_reserved":0,"allocated":3024199680,"data_stored":3023677436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":65667072},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.56499999999999995,"15min":0.55300000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":6.5289999999999999},{"interface":"front","average":{"1min":0.63,"5min":0.56000000000000005,"15min":0.54400000000000004},"min":{"1min":0.39700000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.2070000000000001,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":3.7290000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.39000000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.58599999999999997,"15min":0.56999999999999995},"min":{"1min":0.36599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.3759999999999999,"5min":2.3759999999999999,"15min":2.3759999999999999},"last":3.5139999999999998},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.61499999999999999,"15min":0.59699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.351,"5min":2.351,"15min":2.351},"last":8.1479999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.72099999999999997,"15min":0.64500000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.0279999999999996,"5min":5.0279999999999996,"15min":5.0279999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.774,"5min":0.85499999999999998,"15min":0.70199999999999996},"min":{"1min":0.64400000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":4.4249999999999998,"5min":4.4249999999999998,"15min":4.4249999999999998},"last":2.2050000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:07:31 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.53600000000000003,"15min":0.47399999999999998},"min":{"1min":0.38700000000000001,"5min":0.29999999999999999,"15min":0.26300000000000001},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.56200000000000006,"15min":0.51600000000000001},"min":{"1min":0.376,"5min":0.33000000000000002,"15min":0.27400000000000002},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.63,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":3.1499999999999999,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.67200000000000004,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":2.1230000000000002,"5min":2.1230000000000002,"15min":2.1230000000000002},"last":0.622}]},{"osd":7,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":0.754,"15min":0.64400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":5.21,"5min":5.21,"15min":5.21},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.80100000000000005,"15min":0.65000000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":1.232}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2714435584,"data_stored":2714435584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1737490432,"data_stored":1737490432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2117402624,"data_stored":2117402624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1012006912,"data_stored":1012006912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2718302208,"data_stored":2718302208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3054895104,"data_stored":3054895104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1363542016,"data_stored":1363542016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1697251328,"data_stored":1697251328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:08:41.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:08:41.484 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:41.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:41.588 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:41.643 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:41.737 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:41.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:41.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:41.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:41.930 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:41.943 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 90346 90330 88.1873 67.0625 0.0117709 0.0113356 2023-09-23T15:08:42.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:08:42.074 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:42.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:42.081 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:41 smithi017 ceph-mon[215646]: pgmap v288: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:08:42.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:42.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:41 smithi017 ceph-mon[222878]: pgmap v288: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:08:42.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:41 smithi027 ceph-mon[195233]: pgmap v288: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:08:42.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:42.197 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:42.276 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:42.329 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:42.445 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:42.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:42.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:42.650 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:42.804 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:42.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:42.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:42.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:42.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 91324 91308 87.7676 61.125 0.0237982 0.0113892 2023-09-23T15:08:42.963 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:43.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:43.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:43.068 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:43.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:43.197 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:43.241 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:43.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:43.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:43.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:43.376 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:43.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:08:43.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:43 smithi017 ceph-mon[215646]: pgmap v289: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:08:43.837 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:43.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 92117 92101 87.1888 49.5625 0.0133692 0.0114641 2023-09-23T15:08:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:43 smithi027 ceph-mon[195233]: pgmap v289: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:08:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:44.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:43 smithi017 ceph-mon[222878]: pgmap v289: 89 pgs: 89 active+clean; 5.3 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:08:44.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:44.617 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:44.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:44.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:44.770 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:44.772 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:44.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:44.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:44.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:44.882 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:44 smithi017 ceph-mon[222878]: pgmap v290: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:44.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:44 smithi017 ceph-mon[215646]: pgmap v290: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:44.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 92996 92980 86.7075 54.9375 0.0236665 0.0115279 2023-09-23T15:08:44.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:45.008 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:44 smithi027 ceph-mon[195233]: pgmap v290: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:45.116 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:45.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:45.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:45.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:08:45.244 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:45.331 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:45.866 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:08:45.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 93877 93861 86.2422 55.0625 0.00398749 0.0115759 2023-09-23T15:08:46.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3457447576' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:45 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3457447576' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:45 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.301 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:08:46.309 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:08:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3457447576' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:45 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:08:46.397 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:08:46.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:46.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:08:46.503 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:46.566 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:46.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:46.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:08:46.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: failed to encode map e131 with expected crc 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: pgmap v292: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[215646]: from='client.34842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: failed to encode map e131 with expected crc 2023-09-23T15:08:46.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:46.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T15:08:46.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: pgmap v292: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:46.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:46 smithi017 ceph-mon[222878]: from='client.34842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:46.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 94865 94849 85.8872 61.75 0.0286363 0.0116383 2023-09-23T15:08:47.009 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:47.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:47.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:47.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:47.141 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: failed to encode map e131 with expected crc 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: osdmap e131: 8 total, 8 up, 8 in 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: pgmap v292: 89 pgs: 89 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 938 op/s 2023-09-23T15:08:47.142 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:46 smithi027 ceph-mon[195233]: from='client.34842 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:08:47.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:47.188 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:47.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:47.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:47.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:47.347 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:47.685 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":292,"stamp":"2023-09-23T15:08:46.156216+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6034364339,"num_objects":92032,"num_object_clones":0,"num_object_copies":276096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92032,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":93150,"num_write_kb":5903397,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93196,"ondisk_log_size":93196,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18087576,"kb_used_data":17659672,"kb_used_omap":30,"kb_used_meta":427809,"kb_avail":731873640,"statfs":{"total":767960285184,"available":749438607360,"internally_reserved":0,"allocated":18083504128,"data_stored":18079033724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":438077016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,5,6,8,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":615251968,"num_objects":9388,"num_object_clones":0,"num_object_copies":28164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9388,"num_write_kb":600832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004102"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":107,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154832+0000","last_change":"2023-09-23T15:05:44.717329+0000","last_active":"2023-09-23T15:08:30.154832+0000","last_peered":"2023-09-23T15:08:30.154832+0000","last_clean":"2023-09-23T15:08:30.154832+0000","last_became_active":"2023-09-23T15:05:44.716919+0000","last_became_peered":"2023-09-23T15:05:44.716919+0000","last_unstale":"2023-09-23T15:08:30.154832+0000","last_undegraded":"2023-09-23T15:08:30.154832+0000","last_fullsized":"2023-09-23T15:08:30.154832+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:01.506007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":56,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.860501+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:08:35.860501+0000","last_peered":"2023-09-23T15:08:35.860501+0000","last_clean":"2023-09-23T15:08:35.860501+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:08:35.860501+0000","last_undegraded":"2023-09-23T15:08:35.860501+0000","last_fullsized":"2023-09-23T15:08:35.860501+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":67,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.143987+0000","last_change":"2023-09-23T15:06:16.129498+0000","last_active":"2023-09-23T15:08:30.143987+0000","last_peered":"2023-09-23T15:08:30.143987+0000","last_clean":"2023-09-23T15:08:30.143987+0000","last_became_active":"2023-09-23T15:06:16.129150+0000","last_became_peered":"2023-09-23T15:06:16.129150+0000","last_unstale":"2023-09-23T15:08:30.143987+0000","last_undegraded":"2023-09-23T15:08:30.143987+0000","last_fullsized":"2023-09-23T15:08:30.143987+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:29.090102+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.858415+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:08:35.858415+0000","last_peered":"2023-09-23T15:08:35.858415+0000","last_clean":"2023-09-23T15:08:35.858415+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:08:35.858415+0000","last_undegraded":"2023-09-23T15:08:35.858415+0000","last_fullsized":"2023-09-23T15:08:35.858415+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.860091+0000","last_change":"2023-09-23T15:06:16.132197+0000","last_active":"2023-09-23T15:08:35.860091+0000","last_peered":"2023-09-23T15:08:35.860091+0000","last_clean":"2023-09-23T15:08:35.860091+0000","last_became_active":"2023-09-23T15:06:16.131496+0000","last_became_peered":"2023-09-23T15:06:16.131496+0000","last_unstale":"2023-09-23T15:08:35.860091+0000","last_undegraded":"2023-09-23T15:08:35.860091+0000","last_fullsized":"2023-09-23T15:08:35.860091+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:07.527154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":87,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.160435+0000","last_change":"2023-09-23T15:05:12.648352+0000","last_active":"2023-09-23T15:08:30.160435+0000","last_peered":"2023-09-23T15:08:30.160435+0000","last_clean":"2023-09-23T15:08:30.160435+0000","last_became_active":"2023-09-23T15:05:12.647997+0000","last_became_peered":"2023-09-23T15:05:12.647997+0000","last_unstale":"2023-09-23T15:08:30.160435+0000","last_undegraded":"2023-09-23T15:08:30.160435+0000","last_fullsized":"2023-09-23T15:08:30.160435+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:58.568420+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151203+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:08:30.151203+0000","last_peered":"2023-09-23T15:08:30.151203+0000","last_clean":"2023-09-23T15:08:30.151203+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:08:30.151203+0000","last_undegraded":"2023-09-23T15:08:30.151203+0000","last_fullsized":"2023-09-23T15:08:30.151203+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.147339+0000","last_change":"2023-09-23T15:06:16.131810+0000","last_active":"2023-09-23T15:08:34.147339+0000","last_peered":"2023-09-23T15:08:34.147339+0000","last_clean":"2023-09-23T15:08:34.147339+0000","last_became_active":"2023-09-23T15:06:16.130713+0000","last_became_peered":"2023-09-23T15:06:16.130713+0000","last_unstale":"2023-09-23T15:08:34.147339+0000","last_undegraded":"2023-09-23T15:08:34.147339+0000","last_fullsized":"2023-09-23T15:08:34.147339+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:27:36.772379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.165078+0000","last_change":"2023-09-23T15:05:12.643813+0000","last_active":"2023-09-23T15:08:30.165078+0000","last_peered":"2023-09-23T15:08:30.165078+0000","last_clean":"2023-09-23T15:08:30.165078+0000","last_became_active":"2023-09-23T15:05:12.643430+0000","last_became_peered":"2023-09-23T15:05:12.643430+0000","last_unstale":"2023-09-23T15:08:30.165078+0000","last_undegraded":"2023-09-23T15:08:30.165078+0000","last_fullsized":"2023-09-23T15:08:30.165078+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:14:37.826592+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":41,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.147428+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:08:34.147428+0000","last_peered":"2023-09-23T15:08:34.147428+0000","last_clean":"2023-09-23T15:08:34.147428+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:08:34.147428+0000","last_undegraded":"2023-09-23T15:08:34.147428+0000","last_fullsized":"2023-09-23T15:08:34.147428+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.158001+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:08:30.158001+0000","last_peered":"2023-09-23T15:08:30.158001+0000","last_clean":"2023-09-23T15:08:30.158001+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:08:30.158001+0000","last_undegraded":"2023-09-23T15:08:30.158001+0000","last_fullsized":"2023-09-23T15:08:30.158001+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":67,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144277+0000","last_change":"2023-09-23T15:05:44.717525+0000","last_active":"2023-09-23T15:08:30.144277+0000","last_peered":"2023-09-23T15:08:30.144277+0000","last_clean":"2023-09-23T15:08:30.144277+0000","last_became_active":"2023-09-23T15:05:44.717211+0000","last_became_peered":"2023-09-23T15:05:44.717211+0000","last_unstale":"2023-09-23T15:08:30.144277+0000","last_undegraded":"2023-09-23T15:08:30.144277+0000","last_fullsized":"2023-09-23T15:08:30.144277+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:43.811395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":94,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154885+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:08:30.154885+0000","last_peered":"2023-09-23T15:08:30.154885+0000","last_clean":"2023-09-23T15:08:30.154885+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:08:30.154885+0000","last_undegraded":"2023-09-23T15:08:30.154885+0000","last_fullsized":"2023-09-23T15:08:30.154885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154734+0000","last_change":"2023-09-23T15:05:12.645440+0000","last_active":"2023-09-23T15:08:30.154734+0000","last_peered":"2023-09-23T15:08:30.154734+0000","last_clean":"2023-09-23T15:08:30.154734+0000","last_became_active":"2023-09-23T15:05:12.645150+0000","last_became_peered":"2023-09-23T15:05:12.645150+0000","last_unstale":"2023-09-23T15:08:30.154734+0000","last_undegraded":"2023-09-23T15:08:30.154734+0000","last_fullsized":"2023-09-23T15:08:30.154734+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:22.377370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":90,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157734+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:08:30.157734+0000","last_peered":"2023-09-23T15:08:30.157734+0000","last_clean":"2023-09-23T15:08:30.157734+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:08:30.157734+0000","last_undegraded":"2023-09-23T15:08:30.157734+0000","last_fullsized":"2023-09-23T15:08:30.157734+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:34.147467+0000","last_change":"2023-09-23T15:06:16.135551+0000","last_active":"2023-09-23T15:08:34.147467+0000","last_peered":"2023-09-23T15:08:34.147467+0000","last_clean":"2023-09-23T15:08:34.147467+0000","last_became_active":"2023-09-23T15:06:16.133626+0000","last_became_peered":"2023-09-23T15:06:16.133626+0000","last_unstale":"2023-09-23T15:08:34.147467+0000","last_undegraded":"2023-09-23T15:08:34.147467+0000","last_fullsized":"2023-09-23T15:08:34.147467+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.338568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.143996+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:08:30.143996+0000","last_peered":"2023-09-23T15:08:30.143996+0000","last_clean":"2023-09-23T15:08:30.143996+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:08:30.143996+0000","last_undegraded":"2023-09-23T15:08:30.143996+0000","last_fullsized":"2023-09-23T15:08:30.143996+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.163160+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:08:30.163160+0000","last_peered":"2023-09-23T15:08:30.163160+0000","last_clean":"2023-09-23T15:08:30.163160+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:08:30.163160+0000","last_undegraded":"2023-09-23T15:08:30.163160+0000","last_fullsized":"2023-09-23T15:08:30.163160+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157519+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:08:30.157519+0000","last_peered":"2023-09-23T15:08:30.157519+0000","last_clean":"2023-09-23T15:08:30.157519+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:08:30.157519+0000","last_undegraded":"2023-09-23T15:08:30.157519+0000","last_fullsized":"2023-09-23T15:08:30.157519+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151530+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:08:30.151530+0000","last_peered":"2023-09-23T15:08:30.151530+0000","last_clean":"2023-09-23T15:08:30.151530+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:08:30.151530+0000","last_undegraded":"2023-09-23T15:08:30.151530+0000","last_fullsized":"2023-09-23T15:08:30.151530+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.161807+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:08:30.161807+0000","last_peered":"2023-09-23T15:08:30.161807+0000","last_clean":"2023-09-23T15:08:30.161807+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:08:30.161807+0000","last_undegraded":"2023-09-23T15:08:30.161807+0000","last_fullsized":"2023-09-23T15:08:30.161807+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.860682+0000","last_change":"2023-09-23T15:06:16.131473+0000","last_active":"2023-09-23T15:08:35.860682+0000","last_peered":"2023-09-23T15:08:35.860682+0000","last_clean":"2023-09-23T15:08:35.860682+0000","last_became_active":"2023-09-23T15:06:16.131049+0000","last_became_peered":"2023-09-23T15:06:16.131049+0000","last_unstale":"2023-09-23T15:08:35.860682+0000","last_undegraded":"2023-09-23T15:08:35.860682+0000","last_fullsized":"2023-09-23T15:08:35.860682+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:07.948539+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"130'5809","reported_seq":5832,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.051354+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:08:44.051354+0000","last_peered":"2023-09-23T15:08:44.051354+0000","last_clean":"2023-09-23T15:08:44.051354+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:08:44.051354+0000","last_undegraded":"2023-09-23T15:08:44.051354+0000","last_fullsized":"2023-09-23T15:08:44.051354+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5809,"log_dups_size":0,"ondisk_log_size":5809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380698624,"num_objects":5809,"num_object_clones":0,"num_object_copies":17427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5809,"num_write_kb":371776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157925+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:08:30.157925+0000","last_peered":"2023-09-23T15:08:30.157925+0000","last_clean":"2023-09-23T15:08:30.157925+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:08:30.157925+0000","last_undegraded":"2023-09-23T15:08:30.157925+0000","last_fullsized":"2023-09-23T15:08:30.157925+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.163028+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:08:30.163028+0000","last_peered":"2023-09-23T15:08:30.163028+0000","last_clean":"2023-09-23T15:08:30.163028+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:08:30.163028+0000","last_undegraded":"2023-09-23T15:08:30.163028+0000","last_fullsized":"2023-09-23T15:08:30.163028+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"130'5673","reported_seq":5698,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:42.803181+0000","last_change":"2023-09-23T15:07:30.123308+0000","last_active":"2023-09-23T15:08:42.803181+0000","last_peered":"2023-09-23T15:08:42.803181+0000","last_clean":"2023-09-23T15:08:42.803181+0000","last_became_active":"2023-09-23T15:07:30.122994+0000","last_became_peered":"2023-09-23T15:07:30.122994+0000","last_unstale":"2023-09-23T15:08:42.803181+0000","last_undegraded":"2023-09-23T15:08:42.803181+0000","last_fullsized":"2023-09-23T15:08:42.803181+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5673,"log_dups_size":0,"ondisk_log_size":5673,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:33.236044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371785728,"num_objects":5673,"num_object_clones":0,"num_object_copies":17019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5673,"num_write_kb":363072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":90,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157959+0000","last_change":"2023-09-23T15:05:12.644370+0000","last_active":"2023-09-23T15:08:30.157959+0000","last_peered":"2023-09-23T15:08:30.157959+0000","last_clean":"2023-09-23T15:08:30.157959+0000","last_became_active":"2023-09-23T15:05:12.643996+0000","last_became_peered":"2023-09-23T15:05:12.643996+0000","last_unstale":"2023-09-23T15:08:30.157959+0000","last_undegraded":"2023-09-23T15:08:30.157959+0000","last_fullsized":"2023-09-23T15:08:30.157959+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:26.554064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154556+0000","last_change":"2023-09-23T15:06:16.131518+0000","last_active":"2023-09-23T15:08:30.154556+0000","last_peered":"2023-09-23T15:08:30.154556+0000","last_clean":"2023-09-23T15:08:30.154556+0000","last_became_active":"2023-09-23T15:06:16.131129+0000","last_became_peered":"2023-09-23T15:06:16.131129+0000","last_unstale":"2023-09-23T15:08:30.154556+0000","last_undegraded":"2023-09-23T15:08:30.154556+0000","last_fullsized":"2023-09-23T15:08:30.154556+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:47.620480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"130'5745","reported_seq":5770,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:42.815086+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:08:42.815086+0000","last_peered":"2023-09-23T15:08:42.815086+0000","last_clean":"2023-09-23T15:08:42.815086+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:08:42.815086+0000","last_undegraded":"2023-09-23T15:08:42.815086+0000","last_fullsized":"2023-09-23T15:08:42.815086+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5745,"log_dups_size":0,"ondisk_log_size":5745,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376504320,"num_objects":5745,"num_object_clones":0,"num_object_copies":17235,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5745,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5745,"num_write_kb":367680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":72,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157947+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:08:30.157947+0000","last_peered":"2023-09-23T15:08:30.157947+0000","last_clean":"2023-09-23T15:08:30.157947+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:08:30.157947+0000","last_undegraded":"2023-09-23T15:08:30.157947+0000","last_fullsized":"2023-09-23T15:08:30.157947+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":87,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.160595+0000","last_change":"2023-09-23T15:05:12.642504+0000","last_active":"2023-09-23T15:08:30.160595+0000","last_peered":"2023-09-23T15:08:30.160595+0000","last_clean":"2023-09-23T15:08:30.160595+0000","last_became_active":"2023-09-23T15:05:12.642155+0000","last_became_peered":"2023-09-23T15:05:12.642155+0000","last_unstale":"2023-09-23T15:08:30.160595+0000","last_undegraded":"2023-09-23T15:08:30.160595+0000","last_fullsized":"2023-09-23T15:08:30.160595+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:54.403174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"130'5666","reported_seq":5691,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:42.800390+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:08:42.800390+0000","last_peered":"2023-09-23T15:08:42.800390+0000","last_clean":"2023-09-23T15:08:42.800390+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:08:42.800390+0000","last_undegraded":"2023-09-23T15:08:42.800390+0000","last_fullsized":"2023-09-23T15:08:42.800390+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5666,"log_dups_size":0,"ondisk_log_size":5666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371326976,"num_objects":5666,"num_object_clones":0,"num_object_copies":16998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5666,"num_write_kb":362624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.162980+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:08:30.162980+0000","last_peered":"2023-09-23T15:08:30.162980+0000","last_clean":"2023-09-23T15:08:30.162980+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:08:30.162980+0000","last_undegraded":"2023-09-23T15:08:30.162980+0000","last_fullsized":"2023-09-23T15:08:30.162980+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151290+0000","last_change":"2023-09-23T15:05:12.645592+0000","last_active":"2023-09-23T15:08:30.151290+0000","last_peered":"2023-09-23T15:08:30.151290+0000","last_clean":"2023-09-23T15:08:30.151290+0000","last_became_active":"2023-09-23T15:05:12.645197+0000","last_became_peered":"2023-09-23T15:05:12.645197+0000","last_unstale":"2023-09-23T15:08:30.151290+0000","last_undegraded":"2023-09-23T15:08:30.151290+0000","last_fullsized":"2023-09-23T15:08:30.151290+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:31.029928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"130'5693","reported_seq":5717,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:42.831343+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:08:42.831343+0000","last_peered":"2023-09-23T15:08:42.831343+0000","last_clean":"2023-09-23T15:08:42.831343+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:08:42.831343+0000","last_undegraded":"2023-09-23T15:08:42.831343+0000","last_fullsized":"2023-09-23T15:08:42.831343+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5693,"log_dups_size":0,"ondisk_log_size":5693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373096448,"num_objects":5693,"num_object_clones":0,"num_object_copies":17079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5693,"num_write_kb":364352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":90,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.159559+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:08:30.159559+0000","last_peered":"2023-09-23T15:08:30.159559+0000","last_clean":"2023-09-23T15:08:30.159559+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:08:30.159559+0000","last_undegraded":"2023-09-23T15:08:30.159559+0000","last_fullsized":"2023-09-23T15:08:30.159559+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154930+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:08:30.154930+0000","last_peered":"2023-09-23T15:08:30.154930+0000","last_clean":"2023-09-23T15:08:30.154930+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:08:30.154930+0000","last_undegraded":"2023-09-23T15:08:30.154930+0000","last_fullsized":"2023-09-23T15:08:30.154930+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"130'5895","reported_seq":5920,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:45.658250+0000","last_change":"2023-09-23T15:07:30.127218+0000","last_active":"2023-09-23T15:08:45.658250+0000","last_peered":"2023-09-23T15:08:45.658250+0000","last_clean":"2023-09-23T15:08:45.658250+0000","last_became_active":"2023-09-23T15:07:30.126890+0000","last_became_peered":"2023-09-23T15:07:30.126890+0000","last_unstale":"2023-09-23T15:08:45.658250+0000","last_undegraded":"2023-09-23T15:08:45.658250+0000","last_fullsized":"2023-09-23T15:08:45.658250+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5895,"log_dups_size":0,"ondisk_log_size":5895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:09.910283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386334720,"num_objects":5895,"num_object_clones":0,"num_object_copies":17685,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5895,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5895,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.160729+0000","last_change":"2023-09-23T15:05:12.643666+0000","last_active":"2023-09-23T15:08:30.160729+0000","last_peered":"2023-09-23T15:08:30.160729+0000","last_clean":"2023-09-23T15:08:30.160729+0000","last_became_active":"2023-09-23T15:05:12.643319+0000","last_became_peered":"2023-09-23T15:05:12.643319+0000","last_unstale":"2023-09-23T15:08:30.160729+0000","last_undegraded":"2023-09-23T15:08:30.160729+0000","last_fullsized":"2023-09-23T15:08:30.160729+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:30:10.755600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.156281+0000","last_change":"2023-09-23T15:05:44.717800+0000","last_active":"2023-09-23T15:08:30.156281+0000","last_peered":"2023-09-23T15:08:30.156281+0000","last_clean":"2023-09-23T15:08:30.156281+0000","last_became_active":"2023-09-23T15:05:44.717482+0000","last_became_peered":"2023-09-23T15:05:44.717482+0000","last_unstale":"2023-09-23T15:08:30.156281+0000","last_undegraded":"2023-09-23T15:08:30.156281+0000","last_fullsized":"2023-09-23T15:08:30.156281+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:08.318333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"130'5825","reported_seq":5849,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.047066+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:08:44.047066+0000","last_peered":"2023-09-23T15:08:44.047066+0000","last_clean":"2023-09-23T15:08:44.047066+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:08:44.047066+0000","last_undegraded":"2023-09-23T15:08:44.047066+0000","last_fullsized":"2023-09-23T15:08:44.047066+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5825,"log_dups_size":0,"ondisk_log_size":5825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":381747200,"num_objects":5825,"num_object_clones":0,"num_object_copies":17475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5825,"num_write_kb":372800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154438+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:08:30.154438+0000","last_peered":"2023-09-23T15:08:30.154438+0000","last_clean":"2023-09-23T15:08:30.154438+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:08:30.154438+0000","last_undegraded":"2023-09-23T15:08:30.154438+0000","last_fullsized":"2023-09-23T15:08:30.154438+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":88,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.156405+0000","last_change":"2023-09-23T15:05:12.647492+0000","last_active":"2023-09-23T15:08:30.156405+0000","last_peered":"2023-09-23T15:08:30.156405+0000","last_clean":"2023-09-23T15:08:30.156405+0000","last_became_active":"2023-09-23T15:05:12.646910+0000","last_became_peered":"2023-09-23T15:05:12.646910+0000","last_unstale":"2023-09-23T15:08:30.156405+0000","last_undegraded":"2023-09-23T15:08:30.156405+0000","last_fullsized":"2023-09-23T15:08:30.156405+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:34.744612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"130'5647","reported_seq":5671,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.056986+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:08:44.056986+0000","last_peered":"2023-09-23T15:08:44.056986+0000","last_clean":"2023-09-23T15:08:44.056986+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:08:44.056986+0000","last_undegraded":"2023-09-23T15:08:44.056986+0000","last_fullsized":"2023-09-23T15:08:44.056986+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5647,"log_dups_size":0,"ondisk_log_size":5647,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":370081792,"num_objects":5647,"num_object_clones":0,"num_object_copies":16941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5647,"num_write_kb":361408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157987+0000","last_change":"2023-09-23T15:06:16.132588+0000","last_active":"2023-09-23T15:08:30.157987+0000","last_peered":"2023-09-23T15:08:30.157987+0000","last_clean":"2023-09-23T15:08:30.157987+0000","last_became_active":"2023-09-23T15:06:16.132288+0000","last_became_peered":"2023-09-23T15:06:16.132288+0000","last_unstale":"2023-09-23T15:08:30.157987+0000","last_undegraded":"2023-09-23T15:08:30.157987+0000","last_fullsized":"2023-09-23T15:08:30.157987+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:17.454766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":67,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.147249+0000","last_change":"2023-09-23T15:06:16.128245+0000","last_active":"2023-09-23T15:08:30.147249+0000","last_peered":"2023-09-23T15:08:30.147249+0000","last_clean":"2023-09-23T15:08:30.147249+0000","last_became_active":"2023-09-23T15:06:16.127897+0000","last_became_peered":"2023-09-23T15:06:16.127897+0000","last_unstale":"2023-09-23T15:08:30.147249+0000","last_undegraded":"2023-09-23T15:08:30.147249+0000","last_fullsized":"2023-09-23T15:08:30.147249+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:29.713387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":378,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144156+0000","last_change":"2023-09-23T15:05:12.645438+0000","last_active":"2023-09-23T15:08:30.144156+0000","last_peered":"2023-09-23T15:08:30.144156+0000","last_clean":"2023-09-23T15:08:30.144156+0000","last_became_active":"2023-09-23T15:05:12.644633+0000","last_became_peered":"2023-09-23T15:05:12.644633+0000","last_unstale":"2023-09-23T15:08:30.144156+0000","last_undegraded":"2023-09-23T15:08:30.144156+0000","last_fullsized":"2023-09-23T15:08:30.144156+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:31:38.640431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":106,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.163292+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:08:30.163292+0000","last_peered":"2023-09-23T15:08:30.163292+0000","last_clean":"2023-09-23T15:08:30.163292+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:08:30.163292+0000","last_undegraded":"2023-09-23T15:08:30.163292+0000","last_fullsized":"2023-09-23T15:08:30.163292+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":48,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154162+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:08:30.154162+0000","last_peered":"2023-09-23T15:08:30.154162+0000","last_clean":"2023-09-23T15:08:30.154162+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:08:30.154162+0000","last_undegraded":"2023-09-23T15:08:30.154162+0000","last_fullsized":"2023-09-23T15:08:30.154162+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"130'5625","reported_seq":5649,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:43.936497+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:08:43.936497+0000","last_peered":"2023-09-23T15:08:43.936497+0000","last_clean":"2023-09-23T15:08:43.936497+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:08:43.936497+0000","last_undegraded":"2023-09-23T15:08:43.936497+0000","last_fullsized":"2023-09-23T15:08:43.936497+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5625,"log_dups_size":0,"ondisk_log_size":5625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368640000,"num_objects":5625,"num_object_clones":0,"num_object_copies":16875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5625,"num_write_kb":360000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"130'5858","reported_seq":5882,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:43.970182+0000","last_change":"2023-09-23T15:07:30.127943+0000","last_active":"2023-09-23T15:08:43.970182+0000","last_peered":"2023-09-23T15:08:43.970182+0000","last_clean":"2023-09-23T15:08:43.970182+0000","last_became_active":"2023-09-23T15:07:30.127634+0000","last_became_peered":"2023-09-23T15:07:30.127634+0000","last_unstale":"2023-09-23T15:08:43.970182+0000","last_undegraded":"2023-09-23T15:08:43.970182+0000","last_fullsized":"2023-09-23T15:08:43.970182+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:26.922651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":383909888,"num_objects":5858,"num_object_clones":0,"num_object_copies":17574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5858,"num_write_kb":374912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":631,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.162962+0000","last_change":"2023-09-23T15:05:12.647321+0000","last_active":"2023-09-23T15:08:30.162962+0000","last_peered":"2023-09-23T15:08:30.162962+0000","last_clean":"2023-09-23T15:08:30.162962+0000","last_became_active":"2023-09-23T15:05:12.646847+0000","last_became_peered":"2023-09-23T15:05:12.646847+0000","last_unstale":"2023-09-23T15:08:30.162962+0000","last_undegraded":"2023-09-23T15:08:30.162962+0000","last_fullsized":"2023-09-23T15:08:30.162962+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:27.514430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":89,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154696+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:08:30.154696+0000","last_peered":"2023-09-23T15:08:30.154696+0000","last_clean":"2023-09-23T15:08:30.154696+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:08:30.154696+0000","last_undegraded":"2023-09-23T15:08:30.154696+0000","last_fullsized":"2023-09-23T15:08:30.154696+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":82,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144073+0000","last_change":"2023-09-23T15:06:16.129423+0000","last_active":"2023-09-23T15:08:30.144073+0000","last_peered":"2023-09-23T15:08:30.144073+0000","last_clean":"2023-09-23T15:08:30.144073+0000","last_became_active":"2023-09-23T15:06:16.129128+0000","last_became_peered":"2023-09-23T15:06:16.129128+0000","last_unstale":"2023-09-23T15:08:30.144073+0000","last_undegraded":"2023-09-23T15:08:30.144073+0000","last_fullsized":"2023-09-23T15:08:30.144073+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:22.887225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"130'5684","reported_seq":5709,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:42.819859+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:08:42.819859+0000","last_peered":"2023-09-23T15:08:42.819859+0000","last_clean":"2023-09-23T15:08:42.819859+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:08:42.819859+0000","last_undegraded":"2023-09-23T15:08:42.819859+0000","last_fullsized":"2023-09-23T15:08:42.819859+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5684,"log_dups_size":0,"ondisk_log_size":5684,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372506624,"num_objects":5684,"num_object_clones":0,"num_object_copies":17052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5684,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":739,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.155138+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:08:30.155138+0000","last_peered":"2023-09-23T15:08:30.155138+0000","last_clean":"2023-09-23T15:08:30.155138+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:08:30.155138+0000","last_undegraded":"2023-09-23T15:08:30.155138+0000","last_fullsized":"2023-09-23T15:08:30.155138+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":67,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144438+0000","last_change":"2023-09-23T15:05:44.716950+0000","last_active":"2023-09-23T15:08:30.144438+0000","last_peered":"2023-09-23T15:08:30.144438+0000","last_clean":"2023-09-23T15:08:30.144438+0000","last_became_active":"2023-09-23T15:05:44.716646+0000","last_became_peered":"2023-09-23T15:05:44.716646+0000","last_unstale":"2023-09-23T15:08:30.144438+0000","last_undegraded":"2023-09-23T15:08:30.144438+0000","last_fullsized":"2023-09-23T15:08:30.144438+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:19.294819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":96,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.147323+0000","last_change":"2023-09-23T15:05:12.639664+0000","last_active":"2023-09-23T15:08:30.147323+0000","last_peered":"2023-09-23T15:08:30.147323+0000","last_clean":"2023-09-23T15:08:30.147323+0000","last_became_active":"2023-09-23T15:05:12.639336+0000","last_became_peered":"2023-09-23T15:05:12.639336+0000","last_unstale":"2023-09-23T15:08:30.147323+0000","last_undegraded":"2023-09-23T15:08:30.147323+0000","last_fullsized":"2023-09-23T15:08:30.147323+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:19:26.519133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":375,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.165249+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:08:30.165249+0000","last_peered":"2023-09-23T15:08:30.165249+0000","last_clean":"2023-09-23T15:08:30.165249+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:08:30.165249+0000","last_undegraded":"2023-09-23T15:08:30.165249+0000","last_fullsized":"2023-09-23T15:08:30.165249+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":469,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157480+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:08:30.157480+0000","last_peered":"2023-09-23T15:08:30.157480+0000","last_clean":"2023-09-23T15:08:30.157480+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:08:30.157480+0000","last_undegraded":"2023-09-23T15:08:30.157480+0000","last_fullsized":"2023-09-23T15:08:30.157480+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":41,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154047+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:08:30.154047+0000","last_peered":"2023-09-23T15:08:30.154047+0000","last_clean":"2023-09-23T15:08:30.154047+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:08:30.154047+0000","last_undegraded":"2023-09-23T15:08:30.154047+0000","last_fullsized":"2023-09-23T15:08:30.154047+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":70,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.162511+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:08:30.162511+0000","last_peered":"2023-09-23T15:08:30.162511+0000","last_clean":"2023-09-23T15:08:30.162511+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:08:30.162511+0000","last_undegraded":"2023-09-23T15:08:30.162511+0000","last_fullsized":"2023-09-23T15:08:30.162511+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"130'5701","reported_seq":5724,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.016439+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:08:44.016439+0000","last_peered":"2023-09-23T15:08:44.016439+0000","last_clean":"2023-09-23T15:08:44.016439+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:08:44.016439+0000","last_undegraded":"2023-09-23T15:08:44.016439+0000","last_fullsized":"2023-09-23T15:08:44.016439+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5701,"log_dups_size":0,"ondisk_log_size":5701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":373620736,"num_objects":5701,"num_object_clones":0,"num_object_copies":17103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5701,"num_write_kb":364864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"130'5832","reported_seq":5856,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.031909+0000","last_change":"2023-09-23T15:07:30.124905+0000","last_active":"2023-09-23T15:08:44.031909+0000","last_peered":"2023-09-23T15:08:44.031909+0000","last_clean":"2023-09-23T15:08:44.031909+0000","last_became_active":"2023-09-23T15:07:30.124609+0000","last_became_peered":"2023-09-23T15:07:30.124609+0000","last_unstale":"2023-09-23T15:08:44.031909+0000","last_undegraded":"2023-09-23T15:08:44.031909+0000","last_fullsized":"2023-09-23T15:08:44.031909+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5832,"log_dups_size":0,"ondisk_log_size":5832,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:24.682277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382205952,"num_objects":5832,"num_object_clones":0,"num_object_copies":17496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5832,"num_write_kb":373248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":334,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157483+0000","last_change":"2023-09-23T15:05:12.642118+0000","last_active":"2023-09-23T15:08:30.157483+0000","last_peered":"2023-09-23T15:08:30.157483+0000","last_clean":"2023-09-23T15:08:30.157483+0000","last_became_active":"2023-09-23T15:05:12.641696+0000","last_became_peered":"2023-09-23T15:05:12.641696+0000","last_unstale":"2023-09-23T15:08:30.157483+0000","last_undegraded":"2023-09-23T15:08:30.157483+0000","last_fullsized":"2023-09-23T15:08:30.157483+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:30.203630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.143696+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:08:30.143696+0000","last_peered":"2023-09-23T15:08:30.143696+0000","last_clean":"2023-09-23T15:08:30.143696+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:08:30.143696+0000","last_undegraded":"2023-09-23T15:08:30.143696+0000","last_fullsized":"2023-09-23T15:08:30.143696+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":86,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151455+0000","last_change":"2023-09-23T15:05:44.716591+0000","last_active":"2023-09-23T15:08:30.151455+0000","last_peered":"2023-09-23T15:08:30.151455+0000","last_clean":"2023-09-23T15:08:30.151455+0000","last_became_active":"2023-09-23T15:05:44.716274+0000","last_became_peered":"2023-09-23T15:05:44.716274+0000","last_unstale":"2023-09-23T15:08:30.151455+0000","last_undegraded":"2023-09-23T15:08:30.151455+0000","last_fullsized":"2023-09-23T15:08:30.151455+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.246252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":799,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157851+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:08:30.157851+0000","last_peered":"2023-09-23T15:08:30.157851+0000","last_clean":"2023-09-23T15:08:30.157851+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:08:30.157851+0000","last_undegraded":"2023-09-23T15:08:30.157851+0000","last_fullsized":"2023-09-23T15:08:30.157851+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":67,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144244+0000","last_change":"2023-09-23T15:05:44.722272+0000","last_active":"2023-09-23T15:08:30.144244+0000","last_peered":"2023-09-23T15:08:30.144244+0000","last_clean":"2023-09-23T15:08:30.144244+0000","last_became_active":"2023-09-23T15:05:44.721949+0000","last_became_peered":"2023-09-23T15:05:44.721949+0000","last_unstale":"2023-09-23T15:08:30.144244+0000","last_undegraded":"2023-09-23T15:08:30.144244+0000","last_fullsized":"2023-09-23T15:08:30.144244+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:04.833932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":56,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151566+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:08:30.151566+0000","last_peered":"2023-09-23T15:08:30.151566+0000","last_clean":"2023-09-23T15:08:30.151566+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:08:30.151566+0000","last_undegraded":"2023-09-23T15:08:30.151566+0000","last_fullsized":"2023-09-23T15:08:30.151566+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"130'5894","reported_seq":5917,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.079814+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:08:44.079814+0000","last_peered":"2023-09-23T15:08:44.079814+0000","last_clean":"2023-09-23T15:08:44.079814+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:08:44.079814+0000","last_undegraded":"2023-09-23T15:08:44.079814+0000","last_fullsized":"2023-09-23T15:08:44.079814+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5894,"log_dups_size":0,"ondisk_log_size":5894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386269184,"num_objects":5894,"num_object_clones":0,"num_object_copies":17682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5894,"num_write_kb":377216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":90,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.153931+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:08:30.153931+0000","last_peered":"2023-09-23T15:08:30.153931+0000","last_clean":"2023-09-23T15:08:30.153931+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:08:30.153931+0000","last_undegraded":"2023-09-23T15:08:30.153931+0000","last_fullsized":"2023-09-23T15:08:30.153931+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":660,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.144769+0000","last_change":"2023-09-23T15:05:12.643560+0000","last_active":"2023-09-23T15:08:30.144769+0000","last_peered":"2023-09-23T15:08:30.144769+0000","last_clean":"2023-09-23T15:08:30.144769+0000","last_became_active":"2023-09-23T15:05:12.643116+0000","last_became_peered":"2023-09-23T15:05:12.643116+0000","last_unstale":"2023-09-23T15:08:30.144769+0000","last_undegraded":"2023-09-23T15:08:30.144769+0000","last_fullsized":"2023-09-23T15:08:30.144769+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:40.554636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":87,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.157918+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:08:30.157918+0000","last_peered":"2023-09-23T15:08:30.157918+0000","last_clean":"2023-09-23T15:08:30.157918+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:08:30.157918+0000","last_undegraded":"2023-09-23T15:08:30.157918+0000","last_fullsized":"2023-09-23T15:08:30.157918+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"130'5749","reported_seq":5773,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.051096+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:08:44.051096+0000","last_peered":"2023-09-23T15:08:44.051096+0000","last_clean":"2023-09-23T15:08:44.051096+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:08:44.051096+0000","last_undegraded":"2023-09-23T15:08:44.051096+0000","last_fullsized":"2023-09-23T15:08:44.051096+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5749,"log_dups_size":0,"ondisk_log_size":5749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376766464,"num_objects":5749,"num_object_clones":0,"num_object_copies":17247,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5749,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5749,"num_write_kb":367936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":560,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.160363+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:08:30.160363+0000","last_peered":"2023-09-23T15:08:30.160363+0000","last_clean":"2023-09-23T15:08:30.160363+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:08:30.160363+0000","last_undegraded":"2023-09-23T15:08:30.160363+0000","last_fullsized":"2023-09-23T15:08:30.160363+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":74,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151363+0000","last_change":"2023-09-23T15:05:12.646177+0000","last_active":"2023-09-23T15:08:30.151363+0000","last_peered":"2023-09-23T15:08:30.151363+0000","last_clean":"2023-09-23T15:08:30.151363+0000","last_became_active":"2023-09-23T15:05:12.645772+0000","last_became_peered":"2023-09-23T15:05:12.645772+0000","last_unstale":"2023-09-23T15:08:30.151363+0000","last_undegraded":"2023-09-23T15:08:30.151363+0000","last_fullsized":"2023-09-23T15:08:30.151363+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:47.060337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154522+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:08:30.154522+0000","last_peered":"2023-09-23T15:08:30.154522+0000","last_clean":"2023-09-23T15:08:30.154522+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:08:30.154522+0000","last_undegraded":"2023-09-23T15:08:30.154522+0000","last_fullsized":"2023-09-23T15:08:30.154522+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"130'5708","reported_seq":5732,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:44.113561+0000","last_change":"2023-09-23T15:07:30.128190+0000","last_active":"2023-09-23T15:08:44.113561+0000","last_peered":"2023-09-23T15:08:44.113561+0000","last_clean":"2023-09-23T15:08:44.113561+0000","last_became_active":"2023-09-23T15:07:30.127665+0000","last_became_peered":"2023-09-23T15:07:30.127665+0000","last_unstale":"2023-09-23T15:08:44.113561+0000","last_undegraded":"2023-09-23T15:08:44.113561+0000","last_fullsized":"2023-09-23T15:08:44.113561+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5708,"log_dups_size":0,"ondisk_log_size":5708,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:24.648384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374079488,"num_objects":5708,"num_object_clones":0,"num_object_copies":17124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5708,"num_write_kb":365312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":59,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151448+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:08:30.151448+0000","last_peered":"2023-09-23T15:08:30.151448+0000","last_clean":"2023-09-23T15:08:30.151448+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:08:30.151448+0000","last_undegraded":"2023-09-23T15:08:30.151448+0000","last_fullsized":"2023-09-23T15:08:30.151448+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.153635+0000","last_change":"2023-09-23T15:06:16.131256+0000","last_active":"2023-09-23T15:08:30.153635+0000","last_peered":"2023-09-23T15:08:30.153635+0000","last_clean":"2023-09-23T15:08:30.153635+0000","last_became_active":"2023-09-23T15:06:16.130894+0000","last_became_peered":"2023-09-23T15:06:16.130894+0000","last_unstale":"2023-09-23T15:08:30.153635+0000","last_undegraded":"2023-09-23T15:08:30.153635+0000","last_fullsized":"2023-09-23T15:08:30.153635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:27.672471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.163080+0000","last_change":"2023-09-23T15:05:12.649372+0000","last_active":"2023-09-23T15:08:30.163080+0000","last_peered":"2023-09-23T15:08:30.163080+0000","last_clean":"2023-09-23T15:08:30.163080+0000","last_became_active":"2023-09-23T15:05:12.648942+0000","last_became_peered":"2023-09-23T15:05:12.648942+0000","last_unstale":"2023-09-23T15:08:30.163080+0000","last_undegraded":"2023-09-23T15:08:30.163080+0000","last_fullsized":"2023-09-23T15:08:30.163080+0000","mapping_epoch":100,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:32.181266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154779+0000","last_change":"2023-09-23T15:06:16.138486+0000","last_active":"2023-09-23T15:08:30.154779+0000","last_peered":"2023-09-23T15:08:30.154779+0000","last_clean":"2023-09-23T15:08:30.154779+0000","last_became_active":"2023-09-23T15:06:16.138075+0000","last_became_peered":"2023-09-23T15:06:16.138075+0000","last_unstale":"2023-09-23T15:08:30.154779+0000","last_undegraded":"2023-09-23T15:08:30.154779+0000","last_fullsized":"2023-09-23T15:08:30.154779+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:15.015838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":74,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.163367+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:08:30.163367+0000","last_peered":"2023-09-23T15:08:30.163367+0000","last_clean":"2023-09-23T15:08:30.163367+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:08:30.163367+0000","last_undegraded":"2023-09-23T15:08:30.163367+0000","last_fullsized":"2023-09-23T15:08:30.163367+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":105,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154739+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:08:30.154739+0000","last_peered":"2023-09-23T15:08:30.154739+0000","last_clean":"2023-09-23T15:08:30.154739+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:08:30.154739+0000","last_undegraded":"2023-09-23T15:08:30.154739+0000","last_fullsized":"2023-09-23T15:08:30.154739+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.154855+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:08:30.154855+0000","last_peered":"2023-09-23T15:08:30.154855+0000","last_clean":"2023-09-23T15:08:30.154855+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:08:30.154855+0000","last_undegraded":"2023-09-23T15:08:30.154855+0000","last_fullsized":"2023-09-23T15:08:30.154855+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T15:08:35.860109+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:08:35.860109+0000","last_peered":"2023-09-23T15:08:35.860109+0000","last_clean":"2023-09-23T15:08:35.860109+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:08:35.860109+0000","last_undegraded":"2023-09-23T15:08:35.860109+0000","last_fullsized":"2023-09-23T15:08:35.860109+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":74,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.151124+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:08:30.151124+0000","last_peered":"2023-09-23T15:08:30.151124+0000","last_clean":"2023-09-23T15:08:30.151124+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:08:30.151124+0000","last_undegraded":"2023-09-23T15:08:30.151124+0000","last_fullsized":"2023-09-23T15:08:30.151124+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":98,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T15:08:30.152431+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:08:30.152431+0000","last_peered":"2023-09-23T15:08:30.152431+0000","last_clean":"2023-09-23T15:08:30.152431+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:08:30.152431+0000","last_undegraded":"2023-09-23T15:08:30.152431+0000","last_fullsized":"2023-09-23T15:08:30.152431+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6029574144,"num_objects":92004,"num_object_clones":0,"num_object_copies":276012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":92004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":92004,"num_write_kb":5888256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18142527488,"data_stored":18142527488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":92004,"ondisk_log_size":92004,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201448,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1884300,"kb_used_data":1844676,"kb_used_omap":0,"kb_used_meta":39616,"kb_avail":91860852,"statfs":{"total":95995035648,"available":94065512448,"internally_reserved":0,"allocated":1888948224,"data_stored":1888410229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":40566784},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:47 2023","interfaces":[{"interface":"back","average":{"1min":0.91900000000000004,"5min":0.64500000000000002,"15min":0.60699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":7.8799999999999999,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":6.2110000000000003},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.64600000000000002,"15min":0.60599999999999998},"min":{"1min":0.36899999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":8.0210000000000008,"5min":8.0210000000000008,"15min":8.0210000000000008},"last":3.9790000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":5},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":3.6240000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:07:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.61499999999999999,"15min":0.61599999999999999},"min":{"1min":0.41999999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":6.6130000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.56999999999999995,"15min":0.56699999999999995},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":2.1880000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.78000000000000003,"15min":0.63700000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.1980000000000004,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":6.8449999999999998},{"interface":"front","average":{"1min":1.583,"5min":0.88800000000000001,"15min":0.77200000000000002},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":7.1849999999999996,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":6.7960000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.65400000000000003,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":1.857},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.64000000000000001,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.23999999999999999},"max":{"1min":2.972,"5min":2.972,"15min":2.972},"last":0.33900000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.66500000000000004,"15min":0.59599999999999997},"min":{"1min":0.30599999999999999,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":3.5089999999999999,"5min":3.5089999999999999,"15min":3.5089999999999999},"last":4.9589999999999996},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.72599999999999998,"15min":0.625},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8639999999999999,"5min":2.8639999999999999,"15min":2.8639999999999999},"last":6.2629999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.72999999999999998,"15min":0.63400000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.7909999999999999,"5min":2.7909999999999999,"15min":2.7909999999999999},"last":1.9299999999999999},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.307},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":2.0950000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463084,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512004,"kb_used_data":1482428,"kb_used_omap":0,"kb_used_meta":29568,"kb_avail":92233148,"statfs":{"total":95995035648,"available":94446743552,"internally_reserved":0,"allocated":1518006272,"data_stored":1517476342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":30277632},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.56200000000000006,"15min":0.55000000000000004},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.60499999999999998,"15min":0.58699999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":2.1219999999999999,"5min":2.1219999999999999,"15min":2.1219999999999999},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:07:45 2023","interfaces":[{"interface":"back","average":{"1min":8.4719999999999995,"5min":2.1890000000000001,"15min":1.1419999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":180.374,"5min":180.374,"15min":180.374},"last":0.46300000000000002},{"interface":"front","average":{"1min":8.7289999999999992,"5min":2.242,"15min":1.161},"min":{"1min":0.40200000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":181.685,"5min":181.685,"15min":181.685},"last":0.38}]},{"osd":3,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.68700000000000006,"15min":0.63100000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":3.2690000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.66900000000000004,"15min":0.63},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4710000000000001,"5min":2.4710000000000001,"15min":2.4710000000000001},"last":0.80900000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:08:01 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.57599999999999996,"15min":0.51900000000000002},"min":{"1min":0.33200000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":2.1600000000000001,"5min":2.1600000000000001,"15min":2.1600000000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.82199999999999995,"5min":0.64500000000000002,"15min":0.55100000000000005},"min":{"1min":0.33800000000000002,"5min":0.31900000000000001,"15min":0.25600000000000001},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":0.498}]},{"osd":5,"last update":"Sat Sep 23 15:08:01 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.60499999999999998,"15min":0.55000000000000004},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.30199999999999999},"max":{"1min":1.988,"5min":1.988,"15min":1.988},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.85599999999999998,"5min":0.68200000000000005,"15min":0.58799999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.31900000000000001},"max":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.69899999999999995,"15min":0.61799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.28000000000000003},"max":{"1min":2.5939999999999999,"5min":2.5939999999999999,"15min":2.5939999999999999},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.77300000000000002,"15min":0.67000000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":2.927,"5min":2.927,"15min":2.927},"last":0.55500000000000005}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893171,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3023264,"kb_used_data":2947288,"kb_used_omap":8,"kb_used_meta":75959,"kb_avail":90721888,"statfs":{"total":95995035648,"available":92899213312,"internally_reserved":0,"allocated":3018022912,"data_stored":3017409640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":77782612},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6859999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":0.46200000000000002,"15min":0.42199999999999999},"min":{"1min":0.29299999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":3.2490000000000001,"5min":3.2490000000000001,"15min":3.2490000000000001},"last":4.0140000000000002},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.42399999999999999,"15min":0.41399999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":1.403,"5min":1.419,"15min":1.419},"last":3.2080000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.63700000000000001,"15min":0.52500000000000002},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.0739999999999998,"5min":6.0739999999999998,"15min":6.0739999999999998},"last":0.88800000000000001},{"interface":"front","average":{"1min":1.407,"5min":1.087,"15min":1.0329999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1020000000000003,"5min":11.978,"15min":11.978},"last":7.5949999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.878,"5min":0.79200000000000004,"15min":0.69599999999999995},"min":{"1min":0.52300000000000002,"5min":0.375,"15min":0.375},"max":{"1min":3.093,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.76500000000000001},{"interface":"front","average":{"1min":0.75,"5min":0.66300000000000003,"15min":0.64500000000000002},"min":{"1min":0.46000000000000002,"5min":0.432,"15min":0.432},"max":{"1min":1.956,"5min":1.956,"15min":1.956},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.71299999999999997,"5min":0.69799999999999995,"15min":0.69699999999999995},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.6519999999999999,"5min":1.6519999999999999,"15min":1.6519999999999999},"last":3.1779999999999999},{"interface":"front","average":{"1min":0.86399999999999999,"5min":0.84999999999999998,"15min":0.77100000000000002},"min":{"1min":0.46899999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":2.9780000000000002,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76900000000000002,"5min":0.70299999999999996,"15min":0.69999999999999996},"min":{"1min":0.45000000000000001,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":1.9930000000000001,"5min":1.9930000000000001,"15min":1.9930000000000001},"last":1.3140000000000001},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.67000000000000004,"15min":0.68000000000000005},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":1.3009999999999999,"5min":1.3919999999999999,"15min":1.3919999999999999},"last":3.0990000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67200000000000004,"15min":0.67400000000000004},"min":{"1min":0.441,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.1339999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":3.1459999999999999},{"interface":"front","average":{"1min":0.86099999999999999,"5min":0.83499999999999996,"15min":0.754},"min":{"1min":0.54000000000000004,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":3.0510000000000002,"5min":8.157,"15min":8.157},"last":0.442}]}]},{"osd":2,"up_from":105,"seq":450971566118,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2255872,"kb_used_data":2184056,"kb_used_omap":11,"kb_used_meta":71796,"kb_avail":91489280,"statfs":{"total":95995035648,"available":93685022720,"internally_reserved":0,"allocated":2236473344,"data_stored":2235888971,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":73519888},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39900000000000002,"15min":0.40000000000000002},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":1.171,"5min":1.171,"15min":1.171},"last":5.1340000000000003},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.38700000000000001,"15min":0.38500000000000001},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.155,"5min":1.155,"15min":1.155},"last":4.1399999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.38200000000000001,"15min":0.38400000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.52100000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":2.9199999999999999},{"interface":"front","average":{"1min":0.379,"5min":0.38600000000000001,"15min":0.38700000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":6.5069999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":1.0860000000000001,"15min":0.622},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":57.695,"5min":57.695,"15min":57.695},"last":9.7579999999999991},{"interface":"front","average":{"1min":4.2069999999999999,"5min":1.1619999999999999,"15min":0.65400000000000003},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":58.387999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":5.7279999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.63300000000000001,"15min":0.64500000000000002},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.88100000000000001,"5min":0.88100000000000001,"15min":0.88100000000000001},"last":2.4580000000000002},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.64700000000000002,"15min":0.65800000000000003},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.91500000000000004,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":2.9580000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.622,"15min":0.63},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.90500000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":4.5750000000000002},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.65900000000000003,"15min":0.67500000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.88800000000000001,"5min":0.93400000000000005,"15min":0.93400000000000005},"last":2.5110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.69499999999999995,"15min":0.71899999999999997},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.83499999999999996,"5min":1.9870000000000001,"15min":1.9870000000000001},"last":2.5369999999999999},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.68899999999999995,"15min":0.70299999999999996},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.88500000000000001,"5min":0.95499999999999996,"15min":0.95499999999999996},"last":5.1669999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.68100000000000005,"15min":0.69899999999999995},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.79900000000000004,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":4.5439999999999996},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.67400000000000004,"15min":0.68999999999999995},"min":{"1min":0.49399999999999999,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.85699999999999998,"5min":0.92900000000000005,"15min":0.92900000000000005},"last":4.71}]}]},{"osd":3,"up_from":110,"seq":472446402592,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138724,"kb_used_data":1098972,"kb_used_omap":3,"kb_used_meta":39740,"kb_avail":92606428,"statfs":{"total":95995035648,"available":94828982272,"internally_reserved":0,"allocated":1125347328,"data_stored":1124786540,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":40693929},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.48499999999999999,"15min":0.41199999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"last":7.6529999999999996},{"interface":"front","average":{"1min":1.27,"5min":0.56399999999999995,"15min":0.44600000000000001},"min":{"1min":0.27200000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8,"5min":8,"15min":8},"last":7.875}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.67800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.8109999999999999,"5min":4.4109999999999996,"15min":4.4109999999999996},"last":16.901},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.63800000000000001,"15min":0.49399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.0570000000000004,"5min":8.0570000000000004,"15min":8.0570000000000004},"last":20.274999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.67800000000000005,"15min":0.625},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.54400000000000004},{"interface":"front","average":{"1min":1.423,"5min":0.877,"15min":0.78600000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.4859999999999998,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":0.748}]},{"osd":5,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.72099999999999997,"15min":0.66600000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.2509999999999999,"5min":0.89300000000000002,"15min":0.83299999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":4.4450000000000003,"5min":4.4450000000000003,"15min":4.4450000000000003},"last":0.71899999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.70399999999999996,"15min":0.68400000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.891,"5min":2.891,"15min":2.891},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":0.91100000000000003,"15min":0.85999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.76700000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":0.92900000000000005,"15min":0.83699999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.026,"5min":0.72599999999999998,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":4.1660000000000004,"5min":4.1660000000000004,"15min":4.1660000000000004},"last":0.55600000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986357,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2988292,"kb_used_data":2932220,"kb_used_omap":0,"kb_used_meta":56064,"kb_avail":90756860,"statfs":{"total":95995035648,"available":92935024640,"internally_reserved":0,"allocated":3002593280,"data_stored":3002062863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":57409536},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:46 2023","interfaces":[{"interface":"back","average":{"1min":7.7060000000000004,"5min":1.972,"15min":1.0149999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":122.69199999999999,"5min":122.69199999999999,"15min":122.69199999999999},"last":1.1499999999999999},{"interface":"front","average":{"1min":7.7359999999999998,"5min":1.9890000000000001,"15min":1.02},"min":{"1min":0.38200000000000001,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":122.745,"5min":122.745,"15min":122.745},"last":3.7290000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":1.6160000000000001},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":5.1429999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.57899999999999996,"15min":0.58699999999999997},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.749,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":4.0309999999999997},{"interface":"front","average":{"1min":0.66900000000000004,"5min":0.72399999999999998,"15min":0.73299999999999998},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.4890000000000001,"5min":2.6619999999999999,"15min":2.6619999999999999},"last":9.093}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":0.78700000000000003,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.7240000000000002,"5min":4.7240000000000002,"15min":4.7240000000000002},"last":1.089},{"interface":"front","average":{"1min":1.1830000000000001,"5min":0.72499999999999998,"15min":0.64900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":1.6719999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:08:30 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":0.71299999999999997,"15min":0.57799999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.2309999999999999,"5min":4.2309999999999999,"15min":4.2309999999999999},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.46,"5min":0.76500000000000001,"15min":0.60499999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":3.8799999999999999,"5min":3.8799999999999999,"15min":3.8799999999999999},"last":3.9399999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:08:00 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.65500000000000003,"15min":0.61099999999999999},"min":{"1min":0.39600000000000002,"5min":0.379,"15min":0.318},"max":{"1min":3.798,"5min":3.798,"15min":3.798},"last":2.331},{"interface":"front","average":{"1min":1.008,"5min":0.70899999999999996,"15min":0.61899999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.33400000000000002},"max":{"1min":3.8159999999999998,"5min":3.8159999999999998,"15min":3.8159999999999998},"last":2.7669999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:08:08 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":0.70299999999999996,"15min":0.628},"min":{"1min":0.376,"5min":0.376,"15min":0.28799999999999998},"max":{"1min":2.8559999999999999,"5min":2.8559999999999999,"15min":2.8559999999999999},"last":2.8519999999999999},{"interface":"front","average":{"1min":1.0149999999999999,"5min":0.75800000000000001,"15min":0.68899999999999995},"min":{"1min":0.47899999999999998,"5min":0.432,"15min":0.38800000000000001},"max":{"1min":3.8359999999999999,"5min":3.8359999999999999,"15min":3.8359999999999999},"last":3.9140000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724720,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3367544,"kb_used_data":3298096,"kb_used_omap":0,"kb_used_meta":69440,"kb_avail":90377608,"statfs":{"total":95995035648,"available":92546670592,"internally_reserved":0,"allocated":3377250304,"data_stored":3376727938,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":71106560},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:07:44 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.56499999999999995,"15min":0.55300000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":1.163,"5min":1.163,"15min":1.163},"last":0.38},{"interface":"front","average":{"1min":0.63,"5min":0.56000000000000005,"15min":0.54400000000000004},"min":{"1min":0.39700000000000002,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.2070000000000001,"5min":1.2070000000000001,"15min":1.2070000000000001},"last":3.827}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":3.665},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.53200000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.58599999999999997,"15min":0.56999999999999995},"min":{"1min":0.36599999999999999,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":2.3759999999999999,"5min":2.3759999999999999,"15min":2.3759999999999999},"last":3.6970000000000001},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.61499999999999999,"15min":0.59699999999999998},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":2.351,"5min":2.351,"15min":2.351},"last":3.7909999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.72099999999999997,"15min":0.64500000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.0279999999999996,"5min":5.0279999999999996,"15min":5.0279999999999996},"last":3.6269999999999998},{"interface":"front","average":{"1min":1.774,"5min":0.85499999999999998,"15min":0.70199999999999996},"min":{"1min":0.64400000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":4.4249999999999998,"5min":4.4249999999999998,"15min":4.4249999999999998},"last":0.499}]},{"osd":4,"last update":"Sat Sep 23 15:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":0.73999999999999999,"15min":0.54800000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26300000000000001},"max":{"1min":5.306,"5min":5.306,"15min":5.306},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.089,"5min":0.67000000000000004,"15min":0.55900000000000005},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.25,"5min":3.25,"15min":3.25},"last":0.34799999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:07:53 2023","interfaces":[{"interface":"back","average":{"1min":0.82099999999999995,"5min":0.63,"15min":0.55100000000000005},"min":{"1min":0.41999999999999998,"5min":0.34599999999999997,"15min":0.29899999999999999},"max":{"1min":3.1499999999999999,"5min":3.1499999999999999,"15min":3.1499999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.67200000000000004,"15min":0.61399999999999999},"min":{"1min":0.44500000000000001,"5min":0.33000000000000002,"15min":0.32700000000000001},"max":{"1min":2.1230000000000002,"5min":2.1230000000000002,"15min":2.1230000000000002},"last":0.40999999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":0.754,"15min":0.64400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":5.21,"5min":5.21,"15min":5.21},"last":3.52},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.80100000000000005,"15min":0.65000000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.75700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3025403904,"data_stored":3025403904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2235367424,"data_stored":2235367424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1127546880,"data_stored":1127546880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3023568896,"data_stored":3023568896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3398959104,"data_stored":3398959104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1517944832,"data_stored":1517944832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1886584832,"data_stored":1886584832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:08:47.691 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:08:47.691 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:08:47.691 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T15:08:47.691 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T15:08:47.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T15:08:47.786 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:47.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:47.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:47.895 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:47.944 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:08:47 smithi017 systemd[1]: Stopping Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:08:47.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 95757 95741 85.4567 55.75 0.0104029 0.0116972 2023-09-23T15:08:47.952 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:48.037 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:48.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:48.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:48.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:48.194 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:48.264 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:48.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:48.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:48.355 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:48] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-09-23T15:08:48.365 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:08:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:08:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:08:48.365 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:08:48.365 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:08:48.380 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:48.433 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:48.477 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:48.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:48.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:48.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:48.627 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:48.719 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:48.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:48.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:48.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:48.873 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:48.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 96611 96595 85.0049 53.375 0.00613786 0.0117579 2023-09-23T15:08:49.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:49 smithi017 ceph-mon[215646]: pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T15:08:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:49 smithi017 ceph-mon[222878]: pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T15:08:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:49 smithi027 ceph-mon[195233]: pgmap v293: 89 pgs: 89 active+clean; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 932 op/s 2023-09-23T15:08:49.675 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:49.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:49.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:49.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:08:49.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:08:49.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:49.891 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:49.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:08:49 smithi017 conmon[247394]: 2023-09-23T15:08:49.841+0000 7f5b12ea7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:08:49.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:08:49 smithi017 conmon[247394]: 2023-09-23T15:08:49.841+0000 7f5b12ea7700 -1 osd.1 131 *** Got signal Terminated *** 2023-09-23T15:08:49.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:08:49 smithi017 conmon[247394]: 2023-09-23T15:08:49.841+0000 7f5b12ea7700 -1 osd.1 131 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:08:49.946 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 97564 97548 84.6515 59.5625 0.0274283 0.0118079 2023-09-23T15:08:50.122 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:50.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:50.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:50.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:50.307 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:50.369 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:50.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:50.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:50.427 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:50.436 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:50.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:50 smithi017 ceph-mon[215646]: osd.1 marked itself down and dead 2023-09-23T15:08:50.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:50 smithi017 ceph-mon[222878]: osd.1 marked itself down and dead 2023-09-23T15:08:50.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:50.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:50.482 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:50.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:50 smithi027 ceph-mon[195233]: osd.1 marked itself down and dead 2023-09-23T15:08:50.501 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:50.505 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:50.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:50.538 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:50.545 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:50.548 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:50.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:50.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:50.570 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:50.570 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.570 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:08:50.570 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.571 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:08:50.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:50.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:50.836 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:50.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:50.942 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:50.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 97866 97850 83.7506 18.875 0.00416578 0.0118276 2023-09-23T15:08:50.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:51.015 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:51.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:51.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:51.148 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:51.190 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:51.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:51.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:51.446 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:51.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: pgmap v294: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1095414839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:51.578 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:08:51 smithi017 bash[304710]: 1c41de2c6f27e7c172002e402cd3117eb8939903b224dac97ab1f7186360cbb8 2023-09-23T15:08:51.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: pgmap v294: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1095414839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: pgmap v294: 89 pgs: 89 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 913 op/s 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T15:08:51.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: failed to encode map e132 with expected crc 2023-09-23T15:08:51.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1095414839' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:51.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 98476 98460 83.1341 38.125 0.0160104 0.012024 2023-09-23T15:08:51.994 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:08:51.995 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:08:52.004 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:52.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:52.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:52.195 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.250 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.251 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[215646]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.287 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:52.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:52.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:52.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:08:52.467 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:52 smithi027 ceph-mon[195233]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:52 smithi017 ceph-mon[222878]: failed to encode map e133 with expected crc 2023-09-23T15:08:52.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 99539 99523 82.9114 66.4375 0.0248875 0.0120564 2023-09-23T15:08:52.996 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:08:53.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:53.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:08:53.107 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:53.169 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:53.269 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:53.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:53.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:53.383 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:53.446 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:53.566 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:53.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:53.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:53.739 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:53.838 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:53.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:53.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:53.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:08:53.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 100575 100559 82.6725 64.75 0.0209869 0.0120913 2023-09-23T15:08:53.980 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:53 smithi027 ceph-mon[195233]: pgmap v297: 89 pgs: 10 stale+active+clean, 79 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2023-09-23T15:08:53.985 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:53.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:08:54.066 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:53 smithi017 ceph-mon[215646]: pgmap v297: 89 pgs: 10 stale+active+clean, 79 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2023-09-23T15:08:54.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:53 smithi017 ceph-mon[222878]: pgmap v297: 89 pgs: 10 stale+active+clean, 79 active+clean; 5.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 665 op/s 2023-09-23T15:08:54.073 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:08:54.074 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:08:54.074 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:08:54.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:54.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:08:54.179 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:54.238 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:54.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:54.351 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:54.451 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:54.529 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:54.667 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:54.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:54.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:54.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:54.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/60479611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:54.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:54 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:54.795 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:08:54.875 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:54.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 101514 101498 82.361 58.6875 0.0142801 0.0121364 2023-09-23T15:08:54.993 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:55.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:55.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:55.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/60479611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:55.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:54 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:55.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/60479611' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:55.149 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:54 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:08:55.179 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:55.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:08:55.767 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:55.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:55.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:55.802 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:08:55.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:55.947 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 102385 102369 82.003 54.4375 0.0273275 0.0121901 2023-09-23T15:08:55.957 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:56.040 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:56.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:56.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: pgmap v298: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 31554/300927 objects degraded (10.486%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:56.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:55 smithi027 ceph-mon[195233]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T15:08:56.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: pgmap v298: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 31554/300927 objects degraded (10.486%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:56.082 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[222878]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T15:08:56.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: pgmap v298: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:56.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 31554/300927 objects degraded (10.486%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:08:56.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:08:56.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: failed to encode map e134 with expected crc 2023-09-23T15:08:56.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:08:56.084 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:55 smithi017 ceph-mon[215646]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T15:08:56.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:08:56.203 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:56.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:08:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:56.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:08:56.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:56.472 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:56.589 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:56.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T15:08:56.607 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:08:56.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:56.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:08:56.667 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:08:56.667 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T15:08:56.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:56.789 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:56.877 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:56.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:56.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:56.949 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 103341 103325 81.719 59.75 0.00743347 0.012232 2023-09-23T15:08:56.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:08:57.052 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:57.264 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:08:57.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:57.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:08:57.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:08:57.397 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:57.802 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:57 smithi027 ceph-mon[195233]: pgmap v300: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:57.802 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:08:57.955 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:08:57.950473+0000 min lat: 0.00107351 max lat: 1.03287 avg lat: 0.012295 2023-09-23T15:08:57.955 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:08:57.955 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 104115 104099 81.3022 48.375 0.0115895 0.012295 2023-09-23T15:08:57.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:08:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:57 smithi017 ceph-mon[222878]: pgmap v300: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:08:58.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:08:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:08:58] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-09-23T15:08:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:57 smithi017 ceph-mon[215646]: pgmap v300: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 43 MiB/s wr, 681 op/s; 31554/300927 objects degraded (10.486%) 2023-09-23T15:08:58.201 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:08:58.647 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:08:58.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:08:58.719 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:08:58.780 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:08:58.819 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:08:58.856 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:08:58.883 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:08:58.883 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.884 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:08:58.895 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:08:58.949 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 105004 104988 80.9844 55.5625 0.00402825 0.0123303 2023-09-23T15:08:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[215646]: pgmap v301: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 31881/305349 objects degraded (10.441%) 2023-09-23T15:08:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2621439146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[222878]: pgmap v301: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 31881/305349 objects degraded (10.441%) 2023-09-23T15:08:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:08:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2621439146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:59.306 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:08:59.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:59.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:08:59.398 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:08:59.399 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:08:59.399 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:59 smithi027 ceph-mon[195233]: pgmap v301: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 44 MiB/s wr, 702 op/s; 31881/305349 objects degraded (10.441%) 2023-09-23T15:08:59.399 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:08:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2621439146' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:08:59.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:08:59.499 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:59.704 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:08:59.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:08:59.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:08:59.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:08:59.848 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:08:59.849 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:08:59.893 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:08:59.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 105628 105612 80.4725 39 0.01666 0.0124218 2023-09-23T15:09:00.088 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:00.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:00.091 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:105788 5.f 5:fd837f3a:::benchmark_data_smithi017_288048_object105787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.150402+0000", 2023-09-23T15:09:00.172 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.020004853, 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020045971999999999, 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 105788 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:00.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150402+0000", 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150402+0000", 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150401+0000", 2023-09-23T15:09:00.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150489+0000", 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8740999999999996e-05 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150504+0000", 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4946999999999999e-05 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.150511+0000", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3119999999999999e-06 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.152784+0000", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022732540000000002 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.152932+0000", 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000148169 2023-09-23T15:09:00.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.153109+0000", 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176909 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.158571+0000", 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054623550000000003 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:09:00.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.160824+0000", 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022526389999999999 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:105800 5.b e134/114 5:d945e720:::benchmark_data_smithi017_288048_object105799:head v 134'6626, mlcod=134'6624)", 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.162697+0000", 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0077098899999999996, 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078482240000000009, 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:00.178 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162697+0000", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162697+0000", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:00.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162696+0000", 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162737+0000", 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0936000000000002e-05 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162750+0000", 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2972e-05 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162757+0000", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6599999999999998e-06 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162798+0000", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1006e-05 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.162818+0000", 2023-09-23T15:09:00.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0244000000000001e-05 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:105794 5.4 e134/114 5:272d00bf:::benchmark_data_smithi017_288048_object105793:head v 134'6536, mlcod=134'6533)", 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.164357+0000", 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.006049797, 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062580789999999997, 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:00.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164357+0000", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164357+0000", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164356+0000", 2023-09-23T15:09:00.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164383+0000", 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7039e-05 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164398+0000", 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4576000000000001e-05 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164404+0000", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5500000000000002e-06 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164444+0000", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0738999999999997e-05 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.164463+0000", 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9048000000000001e-05 2023-09-23T15:09:00.185 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:105801 5.4 e134/114 5:2a1485be:::benchmark_data_smithi017_288048_object105800:head v 134'6537, mlcod=134'6533)", 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.167884+0000", 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0025232190000000002, 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027933810000000002, 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:00.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167884+0000", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167884+0000", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167883+0000", 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:09:00.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167912+0000", 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9342000000000001e-05 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167932+0000", 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9762000000000002e-05 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167938+0000", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6340000000000003e-06 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.167990+0000", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1243999999999999e-05 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168009+0000", 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9511e-05 2023-09-23T15:09:00.189 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:105803 5.f 5:ff2b2865:::benchmark_data_smithi017_288048_object105802:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.168184+0000", 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.002223048, 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025640239999999998, 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:09:00.190 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 105803 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168184+0000", 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168184+0000", 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168182+0000", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168222+0000", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0107000000000001e-05 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168236+0000", 2023-09-23T15:09:00.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3125999999999999e-05 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168243+0000", 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0929999999999997e-06 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168297+0000", 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4452e-05 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168433+0000", 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135907 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.168583+0000", 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014958300000000001 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:00.194 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:09:00.195 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 5 2023-09-23T15:09:00.195 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:09:00.195 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:00.304 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:00 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 32909/314259 objects degraded (10.472%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:00.438 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:00.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:00.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:00 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 32909/314259 objects degraded (10.472%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:00.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:00 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 32909/314259 objects degraded (10.472%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:00.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:00.581 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:00.848 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:00.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:00.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 106191 106175 79.927 35.1875 0.00856795 0.0124736 2023-09-23T15:09:00.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:01.003 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:01.277 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:01 smithi027 ceph-mon[195233]: pgmap v302: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:01.371 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:01.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:01.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:01 smithi017 ceph-mon[215646]: pgmap v302: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:01 smithi017 ceph-mon[222878]: pgmap v302: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 956 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:01.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:09:01.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:01.534 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:01.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:01.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:01.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:01.668 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:09:01.668 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:09:01.668 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T15:09:01.668 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:09:01.669 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:09:01.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T15:09:01.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:01.827 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:01.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:09:01.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 107045 107029 79.6109 53.375 0.0262877 0.0125553 2023-09-23T15:09:02.110 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:02.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:02.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:02.344 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:02.430 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:02.462 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:02.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:02.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:02.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:02.628 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:02.686 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:02.749 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:02.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:02.757 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:02.778 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2945436864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.778 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:02 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.783 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:02.793 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:09:02.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:02.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2945436864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:02 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2945436864' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:02 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:02.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 107840 107824 79.2589 49.6875 0.00645458 0.0126111 2023-09-23T15:09:02.959 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:03.196 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:03.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:03.217 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:03.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:03.347 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:03.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: pgmap v303: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: failed to encode map e135 with expected crc 2023-09-23T15:09:03.784 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:03.785 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:03.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 108595 108579 78.886 47.1875 0.00984119 0.012671 2023-09-23T15:09:04.019 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:04.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:04.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:04.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: pgmap v303: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:04.151 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:04.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: pgmap v303: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 53 MiB/s wr, 852 op/s; 32909/314259 objects degraded (10.472%) 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: failed to encode map e135 with expected crc 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:04.153 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:04.154 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:04.225 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:04.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:04.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:04.363 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:04.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:04.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:04.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 109329 109313 78.5065 45.875 0.0407108 0.0127331 2023-09-23T15:09:04.976 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:05.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:05.061 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:05.064 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:05.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:05.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:05.177 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:05.209 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.236 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.237 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:09:05.508 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:05.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:05.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:05.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:05.656 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:05.722 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:05.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:05.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:05.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:05 smithi027 ceph-mon[195233]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 33780/324654 objects degraded (10.405%) 2023-09-23T15:09:05.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3866717986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:05.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:05.827 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:09:05.827 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:09:05.827 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:09:05.882 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:05.882 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:05.926 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:05.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:05 smithi017 ceph-mon[215646]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 33780/324654 objects degraded (10.405%) 2023-09-23T15:09:05.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3866717986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:05 smithi017 ceph-mon[222878]: pgmap v305: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 53 MiB/s wr, 842 op/s; 33780/324654 objects degraded (10.405%) 2023-09-23T15:09:05.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3866717986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:05.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 110118 110102 78.1747 49.3125 0.00730432 0.0127862 2023-09-23T15:09:05.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:09:06.104 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:06.105 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:06.540 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:06.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 111078 111062 77.9705 60 0.0170925 0.0128202 2023-09-23T15:09:07.085 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:06 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 33780/324654 objects degraded (10.405%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:07.086 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:06 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 33780/324654 objects degraded (10.405%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:06 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 33780/324654 objects degraded (10.405%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:07.239 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:07.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:07.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:07.376 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:07.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:09:07.587 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:07.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:07.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:07.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:07.737 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:07.821 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:07.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:07.857 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:07 smithi027 ceph-mon[195233]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 33999/325311 objects degraded (10.451%) 2023-09-23T15:09:07.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:07.950 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0158 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 111766 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 77.6017 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 33.5667 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 210.625 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 18.875 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1241 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 537.067 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3370 2023-09-23T15:09:07.951 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 302 2023-09-23T15:09:07.952 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.012882 2023-09-23T15:09:07.952 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0197575 2023-09-23T15:09:07.952 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 1.03287 2023-09-23T15:09:07.952 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00107351 2023-09-23T15:09:07.956 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:09:07.957 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:08.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:07 smithi017 ceph-mon[215646]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 33999/325311 objects degraded (10.451%) 2023-09-23T15:09:08.027 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:08] "GET /metrics HTTP/1.1" 200 36298 "" "Prometheus/2.43.0" 2023-09-23T15:09:08.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:07 smithi017 ceph-mon[222878]: pgmap v306: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 33999/325311 objects degraded (10.451%) 2023-09-23T15:09:08.072 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:08.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:08.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:09:08.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:08.274 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:08.432 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:08.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:08.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:08.469 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:08.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:08.619 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:08.671 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:08.737 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:08.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:08.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:08.826 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:09:08.855 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:08.897 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:08.991 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:09.096 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:09.097 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:09.098 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:09.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:09.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:09.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:09.298 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:09.482 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:09.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:09.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:09.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:09.628 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:10.033 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:10.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:10.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 34229/328974 objects degraded (10.405%) 2023-09-23T15:09:10.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:09 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:10.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 34229/328974 objects degraded (10.405%) 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.184 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: pgmap v307: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 49 MiB/s wr, 787 op/s; 34229/328974 objects degraded (10.405%) 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3566914016' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: failed to encode map e136 with expected crc 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:09 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.188 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:10.248 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:10.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:10.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:10.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:10.395 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:10.587 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:10.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:10.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:10.810 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:10.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:09:10.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:09:10.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:10.938 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:10.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:10.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:11.047 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:11.085 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:11.086 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:11.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:09:11.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:09:11.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:11.138 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:11.140 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:11.174 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:11.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:09:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:09:11.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:09:11.201 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:11.238 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:11.271 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:11.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:11.280 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:11.313 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.346 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.346 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.346 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:09:11.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:11.451 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:11.573 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:11.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:11.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:11.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:11.732 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:11.780 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:11.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:11.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:11.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:09:11.983 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:11 smithi027 ceph-mon[195233]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 960 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:11.983 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1063404895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:11.983 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:11 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 34772/333309 objects degraded (10.432%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:11.986 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:12.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:09:12.152 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:12.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[215646]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 960 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1063404895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 34772/333309 objects degraded (10.432%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:12.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[222878]: pgmap v309: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 55 MiB/s wr, 960 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:12.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1063404895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:12.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:11 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 34772/333309 objects degraded (10.432%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:12.257 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:12.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:12.337 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:09:12.337 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:09:12.337 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T15:09:12.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:09:12.345 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:12.456 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:12.470 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:12.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:12.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:12.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:12.651 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:12.760 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:12.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:12.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:12.865 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:12.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:12.970 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:13.191 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:13.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:13.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:13.329 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:13.373 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:13.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:13.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:13.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:13.522 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:13.738 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:13.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:13.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:13.766 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:13.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:13.884 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:14.031 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:14.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:14.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:13 smithi027 ceph-mon[195233]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 818 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:14.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:14.160 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:13 smithi017 ceph-mon[215646]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 818 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:14.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:14.161 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:13 smithi017 ceph-mon[222878]: pgmap v310: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 47 MiB/s wr, 818 op/s; 34772/333309 objects degraded (10.432%) 2023-09-23T15:09:14.161 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:14.164 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:14.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:09:14.266 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:14.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:14.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:14.372 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:14.457 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:14.585 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:14.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:14.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:14.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:14.746 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:14.855 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:09:14.881 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:14.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:14.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:15.063 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:15.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3714698972' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3714698972' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3714698972' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:15.192 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:09:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:09:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:09:15.192 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:09:15.644 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:15.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:15.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:15.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:15.793 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:15.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:15.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 734 op/s; 33968/324081 objects degraded (10.481%) 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[222878]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 734 op/s; 33968/324081 objects degraded (10.481%) 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:15 smithi017 ceph-mon[215646]: failed to encode map e137 with expected crc 2023-09-23T15:09:15.949 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:15.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:15.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:16.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: pgmap v311: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 102 B/s rd, 22 MiB/s wr, 734 op/s; 33968/324081 objects degraded (10.481%) 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: failed to encode map e137 with expected crc 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: failed to encode map e137 with expected crc 2023-09-23T15:09:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:15 smithi027 ceph-mon[195233]: failed to encode map e137 with expected crc 2023-09-23T15:09:16.123 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:16.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:09:16.238 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:16.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:16.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:16.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:16.404 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:09:16.407 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:09:16.426 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:16.518 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:16.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:16.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:16.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:16.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:16.806 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:16.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[222878]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 862 op/s; 33463/321090 objects degraded (10.422%) 2023-09-23T15:09:16.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[222878]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:16.851 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 33968/324081 objects degraded (10.481%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:17.116 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:16 smithi027 ceph-mon[195233]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 862 op/s; 33463/321090 objects degraded (10.422%) 2023-09-23T15:09:17.116 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:16 smithi027 ceph-mon[195233]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:17.116 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:16 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 33968/324081 objects degraded (10.481%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:17.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[215646]: pgmap v313: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 127 B/s rd, 17 MiB/s wr, 862 op/s; 33463/321090 objects degraded (10.422%) 2023-09-23T15:09:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[215646]: from='client.44816 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:16 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 33968/324081 objects degraded (10.481%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:17.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:09:17.755 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":313,"stamp":"2023-09-23T15:09:16.165210+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":7017207755,"num_objects":107030,"num_object_clones":0,"num_object_copies":321090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33463,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107030,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":117678,"num_write_kb":7168166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117724,"ondisk_log_size":117724,"up":230,"acting":230,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20848296,"kb_used_data":20322536,"kb_used_omap":30,"kb_used_meta":525473,"kb_avail":729112920,"statfs":{"total":767960285184,"available":746611630080,"internally_reserved":0,"allocated":20810276864,"data_stored":20805742848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":538084952},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,8,2,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-172294120,"num_objects":-2628,"num_object_clones":0,"num_object_copies":-7884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-766,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2628,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6902,"num_write_kb":136705,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002503"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":120,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.822568+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:09:02.822568+0000","last_peered":"2023-09-23T15:09:02.822568+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:09:02.822568+0000","last_undegraded":"2023-09-23T15:09:02.822568+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":62,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.791569+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:09:02.791569+0000","last_peered":"2023-09-23T15:09:02.791569+0000","last_clean":"2023-09-23T15:09:02.791569+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:09:02.791569+0000","last_undegraded":"2023-09-23T15:09:02.791569+0000","last_fullsized":"2023-09-23T15:09:02.791569+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":50,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.791050+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:09:02.791050+0000","last_peered":"2023-09-23T15:09:02.791050+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:09:02.791050+0000","last_undegraded":"2023-09-23T15:09:02.791050+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":78,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.788817+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:09:02.788817+0000","last_peered":"2023-09-23T15:09:02.788817+0000","last_clean":"2023-09-23T15:09:02.788817+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:09:02.788817+0000","last_undegraded":"2023-09-23T15:09:02.788817+0000","last_fullsized":"2023-09-23T15:09:02.788817+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":122,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.790845+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:09:02.790845+0000","last_peered":"2023-09-23T15:09:02.790845+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:09:02.790845+0000","last_undegraded":"2023-09-23T15:09:02.790845+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":100,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.798600+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:09:02.798600+0000","last_peered":"2023-09-23T15:09:02.798600+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:09:02.798600+0000","last_undegraded":"2023-09-23T15:09:02.798600+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.789225+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:09:02.789225+0000","last_peered":"2023-09-23T15:09:02.789225+0000","last_clean":"2023-09-23T15:09:02.789225+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:09:02.789225+0000","last_undegraded":"2023-09-23T15:09:02.789225+0000","last_fullsized":"2023-09-23T15:09:02.789225+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.790701+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:09:02.790701+0000","last_peered":"2023-09-23T15:09:02.790701+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:09:02.790701+0000","last_undegraded":"2023-09-23T15:09:02.790701+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":102,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.805562+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:09:02.805562+0000","last_peered":"2023-09-23T15:09:02.805562+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:09:02.805562+0000","last_undegraded":"2023-09-23T15:09:02.805562+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":47,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.790969+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:09:02.790969+0000","last_peered":"2023-09-23T15:09:02.790969+0000","last_clean":"2023-09-23T15:09:02.790969+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:09:02.790969+0000","last_undegraded":"2023-09-23T15:09:02.790969+0000","last_fullsized":"2023-09-23T15:09:02.790969+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":97,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794914+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:09:02.794914+0000","last_peered":"2023-09-23T15:09:02.794914+0000","last_clean":"2023-09-23T15:09:02.794914+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:09:02.794914+0000","last_undegraded":"2023-09-23T15:09:02.794914+0000","last_fullsized":"2023-09-23T15:09:02.794914+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.797722+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:09:02.797722+0000","last_peered":"2023-09-23T15:09:02.797722+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:09:02.797722+0000","last_undegraded":"2023-09-23T15:09:02.797722+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":100,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.822451+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:09:02.822451+0000","last_peered":"2023-09-23T15:09:02.822451+0000","last_clean":"2023-09-23T15:09:02.822451+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:09:02.822451+0000","last_undegraded":"2023-09-23T15:09:02.822451+0000","last_fullsized":"2023-09-23T15:09:02.822451+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.822438+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:09:02.822438+0000","last_peered":"2023-09-23T15:09:02.822438+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:09:02.822438+0000","last_undegraded":"2023-09-23T15:09:02.822438+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":96,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794615+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:09:02.794615+0000","last_peered":"2023-09-23T15:09:02.794615+0000","last_clean":"2023-09-23T15:09:02.794615+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:09:02.794615+0000","last_undegraded":"2023-09-23T15:09:02.794615+0000","last_fullsized":"2023-09-23T15:09:02.794615+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.791309+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:09:02.791309+0000","last_peered":"2023-09-23T15:09:02.791309+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:09:02.791309+0000","last_undegraded":"2023-09-23T15:09:02.791309+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.788628+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:09:02.788628+0000","last_peered":"2023-09-23T15:09:02.788628+0000","last_clean":"2023-09-23T15:09:02.788628+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:09:02.788628+0000","last_undegraded":"2023-09-23T15:09:02.788628+0000","last_fullsized":"2023-09-23T15:09:02.788628+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":94,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.801559+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:09:02.801559+0000","last_peered":"2023-09-23T15:09:02.801559+0000","last_clean":"2023-09-23T15:09:02.801559+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:09:02.801559+0000","last_undegraded":"2023-09-23T15:09:02.801559+0000","last_fullsized":"2023-09-23T15:09:02.801559+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794327+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:09:02.794327+0000","last_peered":"2023-09-23T15:09:02.794327+0000","last_clean":"2023-09-23T15:09:02.794327+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:09:02.794327+0000","last_undegraded":"2023-09-23T15:09:02.794327+0000","last_fullsized":"2023-09-23T15:09:02.794327+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.788800+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:09:02.788800+0000","last_peered":"2023-09-23T15:09:02.788800+0000","last_clean":"2023-09-23T15:09:02.788800+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:09:02.788800+0000","last_undegraded":"2023-09-23T15:09:02.788800+0000","last_fullsized":"2023-09-23T15:09:02.788800+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":102,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.797823+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:09:02.797823+0000","last_peered":"2023-09-23T15:09:02.797823+0000","last_clean":"2023-09-23T15:09:02.797823+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:09:02.797823+0000","last_undegraded":"2023-09-23T15:09:02.797823+0000","last_fullsized":"2023-09-23T15:09:02.797823+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.794527+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:09:02.794527+0000","last_peered":"2023-09-23T15:09:02.794527+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:09:02.794527+0000","last_undegraded":"2023-09-23T15:09:02.794527+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"136'7346","reported_seq":7373,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:13.997371+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:09:13.997371+0000","last_peered":"2023-09-23T15:09:13.997371+0000","last_clean":"2023-09-23T15:09:13.997371+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:09:13.997371+0000","last_undegraded":"2023-09-23T15:09:13.997371+0000","last_fullsized":"2023-09-23T15:09:13.997371+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7346,"log_dups_size":0,"ondisk_log_size":7346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441188352,"num_objects":6732,"num_object_clones":0,"num_object_copies":20196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7346,"num_write_kb":450496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.797412+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:09:02.797412+0000","last_peered":"2023-09-23T15:09:02.797412+0000","last_clean":"2023-09-23T15:09:02.797412+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:09:02.797412+0000","last_undegraded":"2023-09-23T15:09:02.797412+0000","last_fullsized":"2023-09-23T15:09:02.797412+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.795979+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:09:02.795979+0000","last_peered":"2023-09-23T15:09:02.795979+0000","last_clean":"2023-09-23T15:09:02.795979+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:09:02.795979+0000","last_undegraded":"2023-09-23T15:09:02.795979+0000","last_fullsized":"2023-09-23T15:09:02.795979+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"136'7222","reported_seq":7258,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:12.858547+0000","last_change":"2023-09-23T15:08:51.249550+0000","last_active":"2023-09-23T15:09:12.858547+0000","last_peered":"2023-09-23T15:09:12.858547+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:08:51.249550+0000","last_became_peered":"2023-09-23T15:08:51.249550+0000","last_unstale":"2023-09-23T15:09:12.858547+0000","last_undegraded":"2023-09-23T15:08:51.245609+0000","last_fullsized":"2023-09-23T15:08:51.245485+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.928709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441057280,"num_objects":6730,"num_object_clones":0,"num_object_copies":20190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6730,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":446464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":6730}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":103,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.801059+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:09:02.801059+0000","last_peered":"2023-09-23T15:09:02.801059+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:09:02.801059+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.823460+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:09:02.823460+0000","last_peered":"2023-09-23T15:09:02.823460+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:09:02.823460+0000","last_undegraded":"2023-09-23T15:09:02.823460+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"136'7251","reported_seq":7282,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:12.857554+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:09:12.857554+0000","last_peered":"2023-09-23T15:09:12.857554+0000","last_clean":"2023-09-23T15:09:12.857554+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:09:12.857554+0000","last_undegraded":"2023-09-23T15:09:12.857554+0000","last_fullsized":"2023-09-23T15:09:12.857554+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7251,"log_dups_size":0,"ondisk_log_size":7251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443613184,"num_objects":6769,"num_object_clones":0,"num_object_copies":20307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7251,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":78,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.795066+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:09:02.795066+0000","last_peered":"2023-09-23T15:09:02.795066+0000","last_clean":"2023-09-23T15:09:02.795066+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:09:02.795066+0000","last_undegraded":"2023-09-23T15:09:02.795066+0000","last_fullsized":"2023-09-23T15:09:02.795066+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":100,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.798046+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:09:02.798046+0000","last_peered":"2023-09-23T15:09:02.798046+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:09:02.798046+0000","last_undegraded":"2023-09-23T15:09:02.798046+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"136'7222","reported_seq":7253,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:12.832462+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:09:12.832462+0000","last_peered":"2023-09-23T15:09:12.832462+0000","last_clean":"2023-09-23T15:09:12.832462+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:09:12.832462+0000","last_undegraded":"2023-09-23T15:09:12.832462+0000","last_fullsized":"2023-09-23T15:09:12.832462+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":443285504,"num_objects":6764,"num_object_clones":0,"num_object_copies":20292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.802141+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:02.802141+0000","last_peered":"2023-09-23T15:09:02.802141+0000","last_clean":"2023-09-23T15:09:02.802141+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:09:02.802141+0000","last_undegraded":"2023-09-23T15:09:02.802141+0000","last_fullsized":"2023-09-23T15:09:02.802141+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":84,"reported_epoch":136,"state":"active+undersized","last_fresh":"2023-09-23T15:09:08.817373+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:09:08.817373+0000","last_peered":"2023-09-23T15:09:08.817373+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:09:08.817373+0000","last_undegraded":"2023-09-23T15:09:08.817373+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"136'7236","reported_seq":7267,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:12.846788+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:09:12.846788+0000","last_peered":"2023-09-23T15:09:12.846788+0000","last_clean":"2023-09-23T15:09:12.846788+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:09:12.846788+0000","last_undegraded":"2023-09-23T15:09:12.846788+0000","last_fullsized":"2023-09-23T15:09:12.846788+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7236,"log_dups_size":0,"ondisk_log_size":7236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437190680,"num_objects":6672,"num_object_clones":0,"num_object_copies":20016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6672,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7236,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":96,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.790952+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:09:02.790952+0000","last_peered":"2023-09-23T15:09:02.790952+0000","last_clean":"2023-09-23T15:09:02.790952+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:09:02.790952+0000","last_undegraded":"2023-09-23T15:09:02.790952+0000","last_fullsized":"2023-09-23T15:09:02.790952+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.822535+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:09:02.822535+0000","last_peered":"2023-09-23T15:09:02.822535+0000","last_clean":"2023-09-23T15:09:02.822535+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:09:02.822535+0000","last_undegraded":"2023-09-23T15:09:02.822535+0000","last_fullsized":"2023-09-23T15:09:02.822535+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"136'7297","reported_seq":7333,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:14.013437+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:09:14.013437+0000","last_peered":"2023-09-23T15:09:14.013437+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:09:14.013437+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7297,"log_dups_size":0,"ondisk_log_size":7297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437452800,"num_objects":6675,"num_object_clones":0,"num_object_copies":20025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6675,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6675,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7297,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":6675}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.798377+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:09:02.798377+0000","last_peered":"2023-09-23T15:09:02.798377+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:09:02.798377+0000","last_undegraded":"2023-09-23T15:09:02.798377+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.823056+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:09:02.823056+0000","last_peered":"2023-09-23T15:09:02.823056+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:09:02.823056+0000","last_undegraded":"2023-09-23T15:09:02.823056+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"136'7462","reported_seq":7492,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:13.967085+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:09:13.967085+0000","last_peered":"2023-09-23T15:09:13.967085+0000","last_clean":"2023-09-23T15:09:13.967085+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:09:13.967085+0000","last_undegraded":"2023-09-23T15:09:13.967085+0000","last_fullsized":"2023-09-23T15:09:13.967085+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7462,"log_dups_size":0,"ondisk_log_size":7462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":442892288,"num_objects":6758,"num_object_clones":0,"num_object_copies":20274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7462,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.823223+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:09:02.823223+0000","last_peered":"2023-09-23T15:09:02.823223+0000","last_clean":"2023-09-23T15:09:02.823223+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:09:02.823223+0000","last_undegraded":"2023-09-23T15:09:02.823223+0000","last_fullsized":"2023-09-23T15:09:02.823223+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":101,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.822914+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:09:02.822914+0000","last_peered":"2023-09-23T15:09:02.822914+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:09:02.822914+0000","last_undegraded":"2023-09-23T15:09:02.822914+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"136'7235","reported_seq":7265,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:14.164773+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:09:14.164773+0000","last_peered":"2023-09-23T15:09:14.164773+0000","last_clean":"2023-09-23T15:09:14.164773+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:09:14.164773+0000","last_undegraded":"2023-09-23T15:09:14.164773+0000","last_fullsized":"2023-09-23T15:09:14.164773+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7235,"log_dups_size":0,"ondisk_log_size":7235,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431685632,"num_objects":6587,"num_object_clones":0,"num_object_copies":19761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7235,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":105,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.790957+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:09:02.790957+0000","last_peered":"2023-09-23T15:09:02.790957+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:09:02.790957+0000","last_undegraded":"2023-09-23T15:09:02.790957+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":50,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.794665+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:09:02.794665+0000","last_peered":"2023-09-23T15:09:02.794665+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:09:02.794665+0000","last_undegraded":"2023-09-23T15:09:02.794665+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":327,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.788869+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:09:02.788869+0000","last_peered":"2023-09-23T15:09:02.788869+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:09:02.788869+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":112,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.801709+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:02.801709+0000","last_peered":"2023-09-23T15:09:02.801709+0000","last_clean":"2023-09-23T15:09:02.801709+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:09:02.801709+0000","last_undegraded":"2023-09-23T15:09:02.801709+0000","last_fullsized":"2023-09-23T15:09:02.801709+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.779254+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:09:02.779254+0000","last_peered":"2023-09-23T15:09:02.779254+0000","last_clean":"2023-09-23T15:09:02.779254+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:09:02.779254+0000","last_undegraded":"2023-09-23T15:09:02.779254+0000","last_fullsized":"2023-09-23T15:09:02.779254+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"136'7088","reported_seq":7118,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:13.907532+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:09:13.907532+0000","last_peered":"2023-09-23T15:09:13.907532+0000","last_clean":"2023-09-23T15:09:13.907532+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:09:13.907532+0000","last_undegraded":"2023-09-23T15:09:13.907532+0000","last_fullsized":"2023-09-23T15:09:13.907532+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7088,"log_dups_size":0,"ondisk_log_size":7088,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425852928,"num_objects":6498,"num_object_clones":0,"num_object_copies":19494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7088,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"136'7425","reported_seq":7461,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:13.998433+0000","last_change":"2023-09-23T15:08:51.256935+0000","last_active":"2023-09-23T15:09:13.998433+0000","last_peered":"2023-09-23T15:09:13.998433+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:08:51.256935+0000","last_became_peered":"2023-09-23T15:08:51.256935+0000","last_unstale":"2023-09-23T15:09:13.998433+0000","last_undegraded":"2023-09-23T15:08:51.248529+0000","last_fullsized":"2023-09-23T15:08:51.248430+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7425,"log_dups_size":0,"ondisk_log_size":7425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:33.609757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":440467456,"num_objects":6721,"num_object_clones":0,"num_object_copies":20163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6721,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7425,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":6721}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":644,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.796180+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:09:02.796180+0000","last_peered":"2023-09-23T15:09:02.796180+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:09:02.796180+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":95,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.822335+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:09:02.822335+0000","last_peered":"2023-09-23T15:09:02.822335+0000","last_clean":"2023-09-23T15:09:02.822335+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:09:02.822335+0000","last_undegraded":"2023-09-23T15:09:02.822335+0000","last_fullsized":"2023-09-23T15:09:02.822335+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":65,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.779646+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:09:02.779646+0000","last_peered":"2023-09-23T15:09:02.779646+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:09:02.779646+0000","last_undegraded":"2023-09-23T15:09:02.779646+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"136'7222","reported_seq":7251,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:12.817381+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:09:12.817381+0000","last_peered":"2023-09-23T15:09:12.817381+0000","last_clean":"2023-09-23T15:09:12.817381+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:09:12.817381+0000","last_undegraded":"2023-09-23T15:09:12.817381+0000","last_fullsized":"2023-09-23T15:09:12.817381+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":439746560,"num_objects":6710,"num_object_clones":0,"num_object_copies":20130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":445824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":745,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.779098+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:09:02.779098+0000","last_peered":"2023-09-23T15:09:02.779098+0000","last_clean":"2023-09-23T15:09:02.779098+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:09:02.779098+0000","last_undegraded":"2023-09-23T15:09:02.779098+0000","last_fullsized":"2023-09-23T15:09:02.779098+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":45,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.778947+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:09:02.778947+0000","last_peered":"2023-09-23T15:09:02.778947+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:09:02.778947+0000","last_undegraded":"2023-09-23T15:09:02.778947+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":70,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.806503+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:09:02.806503+0000","last_peered":"2023-09-23T15:09:02.806503+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:09:02.806503+0000","last_undegraded":"2023-09-23T15:09:02.806503+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":381,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.796863+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:09:02.796863+0000","last_peered":"2023-09-23T15:09:02.796863+0000","last_clean":"2023-09-23T15:09:02.796863+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:09:02.796863+0000","last_undegraded":"2023-09-23T15:09:02.796863+0000","last_fullsized":"2023-09-23T15:09:02.796863+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":475,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794095+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:09:02.794095+0000","last_peered":"2023-09-23T15:09:02.794095+0000","last_clean":"2023-09-23T15:09:02.794095+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:09:02.794095+0000","last_undegraded":"2023-09-23T15:09:02.794095+0000","last_fullsized":"2023-09-23T15:09:02.794095+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":47,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.791754+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:09:02.791754+0000","last_peered":"2023-09-23T15:09:02.791754+0000","last_clean":"2023-09-23T15:09:02.791754+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:09:02.791754+0000","last_undegraded":"2023-09-23T15:09:02.791754+0000","last_fullsized":"2023-09-23T15:09:02.791754+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":76,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.797997+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:09:02.797997+0000","last_peered":"2023-09-23T15:09:02.797997+0000","last_clean":"2023-09-23T15:09:02.797997+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:09:02.797997+0000","last_undegraded":"2023-09-23T15:09:02.797997+0000","last_fullsized":"2023-09-23T15:09:02.797997+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"136'7217","reported_seq":7246,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:13.995978+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:09:13.995978+0000","last_peered":"2023-09-23T15:09:13.995978+0000","last_clean":"2023-09-23T15:09:13.995978+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:09:13.995978+0000","last_undegraded":"2023-09-23T15:09:13.995978+0000","last_fullsized":"2023-09-23T15:09:13.995978+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7217,"log_dups_size":0,"ondisk_log_size":7217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432603136,"num_objects":6601,"num_object_clones":0,"num_object_copies":19803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7217,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"136'7373","reported_seq":7410,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:14.003176+0000","last_change":"2023-09-23T15:08:51.252737+0000","last_active":"2023-09-23T15:09:14.003176+0000","last_peered":"2023-09-23T15:09:14.003176+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:08:51.252737+0000","last_became_peered":"2023-09-23T15:08:51.252737+0000","last_unstale":"2023-09-23T15:09:14.003176+0000","last_undegraded":"2023-09-23T15:08:51.248004+0000","last_fullsized":"2023-09-23T15:08:51.247871+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7373,"log_dups_size":0,"ondisk_log_size":7373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.686400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":441909248,"num_objects":6743,"num_object_clones":0,"num_object_copies":20229,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6743,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6743,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7373,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":6743}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":347,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.794207+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:09:02.794207+0000","last_peered":"2023-09-23T15:09:02.794207+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:09:02.794207+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.786531+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:09:02.786531+0000","last_peered":"2023-09-23T15:09:02.786531+0000","last_clean":"2023-09-23T15:09:02.786531+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:09:02.786531+0000","last_undegraded":"2023-09-23T15:09:02.786531+0000","last_fullsized":"2023-09-23T15:09:02.786531+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":99,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.788928+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:09:02.788928+0000","last_peered":"2023-09-23T15:09:02.788928+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:09:02.788928+0000","last_undegraded":"2023-09-23T15:09:02.788928+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":805,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794989+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:09:02.794989+0000","last_peered":"2023-09-23T15:09:02.794989+0000","last_clean":"2023-09-23T15:09:02.794989+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:09:02.794989+0000","last_undegraded":"2023-09-23T15:09:02.794989+0000","last_fullsized":"2023-09-23T15:09:02.794989+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":45,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.779717+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:09:02.779717+0000","last_peered":"2023-09-23T15:09:02.779717+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:09:02.779717+0000","last_undegraded":"2023-09-23T15:09:02.779717+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":63,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.788996+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:09:02.788996+0000","last_peered":"2023-09-23T15:09:02.788996+0000","last_clean":"2023-09-23T15:09:02.788996+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:09:02.788996+0000","last_undegraded":"2023-09-23T15:09:02.788996+0000","last_fullsized":"2023-09-23T15:09:02.788996+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"136'7437","reported_seq":7466,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:13.931887+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:09:13.931887+0000","last_peered":"2023-09-23T15:09:13.931887+0000","last_clean":"2023-09-23T15:09:13.931887+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:09:13.931887+0000","last_undegraded":"2023-09-23T15:09:13.931887+0000","last_fullsized":"2023-09-23T15:09:13.931887+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7437,"log_dups_size":0,"ondisk_log_size":7437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":444530688,"num_objects":6783,"num_object_clones":0,"num_object_copies":20349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7437,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":96,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794568+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:09:02.794568+0000","last_peered":"2023-09-23T15:09:02.794568+0000","last_clean":"2023-09-23T15:09:02.794568+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:09:02.794568+0000","last_undegraded":"2023-09-23T15:09:02.794568+0000","last_fullsized":"2023-09-23T15:09:02.794568+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":630,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.798190+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:09:02.798190+0000","last_peered":"2023-09-23T15:09:02.798190+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:09:02.798190+0000","last_undegraded":"2023-09-23T15:09:02.798190+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":93,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.790856+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:09:02.790856+0000","last_peered":"2023-09-23T15:09:02.790856+0000","last_clean":"2023-09-23T15:09:02.790856+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:09:02.790856+0000","last_undegraded":"2023-09-23T15:09:02.790856+0000","last_fullsized":"2023-09-23T15:09:02.790856+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"136'7277","reported_seq":7306,"reported_epoch":136,"state":"active+clean","last_fresh":"2023-09-23T15:09:14.168884+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:09:14.168884+0000","last_peered":"2023-09-23T15:09:14.168884+0000","last_clean":"2023-09-23T15:09:14.168884+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:09:14.168884+0000","last_undegraded":"2023-09-23T15:09:14.168884+0000","last_fullsized":"2023-09-23T15:09:14.168884+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7277,"log_dups_size":0,"ondisk_log_size":7277,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":437190656,"num_objects":6671,"num_object_clones":0,"num_object_copies":20013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6671,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7277,"num_write_kb":446336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":566,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.798518+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:09:02.798518+0000","last_peered":"2023-09-23T15:09:02.798518+0000","last_clean":"2023-09-23T15:09:02.798518+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:09:02.798518+0000","last_undegraded":"2023-09-23T15:09:02.798518+0000","last_fullsized":"2023-09-23T15:09:02.798518+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":87,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.788885+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:09:02.788885+0000","last_peered":"2023-09-23T15:09:02.788885+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:09:02.788885+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":77,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.823364+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:09:02.823364+0000","last_peered":"2023-09-23T15:09:02.823364+0000","last_clean":"2023-09-23T15:09:02.823364+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:09:02.823364+0000","last_undegraded":"2023-09-23T15:09:02.823364+0000","last_fullsized":"2023-09-23T15:09:02.823364+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"136'7222","reported_seq":7258,"reported_epoch":136,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:14.169776+0000","last_change":"2023-09-23T15:08:51.257194+0000","last_active":"2023-09-23T15:09:14.169776+0000","last_peered":"2023-09-23T15:09:14.169776+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:08:51.257194+0000","last_became_peered":"2023-09-23T15:08:51.257194+0000","last_unstale":"2023-09-23T15:09:14.169776+0000","last_undegraded":"2023-09-23T15:08:51.246208+0000","last_fullsized":"2023-09-23T15:08:51.246087+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7222,"log_dups_size":0,"ondisk_log_size":7222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.554621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431751168,"num_objects":6588,"num_object_clones":0,"num_object_copies":19764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6588,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7222,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":6588}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":65,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.788830+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:09:02.788830+0000","last_peered":"2023-09-23T15:09:02.788830+0000","last_clean":"2023-09-23T15:09:02.788830+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:09:02.788830+0000","last_undegraded":"2023-09-23T15:09:02.788830+0000","last_fullsized":"2023-09-23T15:09:02.788830+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":124,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.779545+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:09:02.779545+0000","last_peered":"2023-09-23T15:09:02.779545+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:09:02.779545+0000","last_undegraded":"2023-09-23T15:09:02.779545+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":104,"reported_epoch":135,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:02.802256+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:09:02.802256+0000","last_peered":"2023-09-23T15:09:02.802256+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:09:02.802256+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":118,"reported_epoch":135,"state":"active+undersized","last_fresh":"2023-09-23T15:09:02.822636+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:09:02.822636+0000","last_peered":"2023-09-23T15:09:02.822636+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:09:02.822636+0000","last_undegraded":"2023-09-23T15:09:02.822636+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":80,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.801795+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:09:02.801795+0000","last_peered":"2023-09-23T15:09:02.801795+0000","last_clean":"2023-09-23T15:09:02.801795+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:09:02.801795+0000","last_undegraded":"2023-09-23T15:09:02.801795+0000","last_fullsized":"2023-09-23T15:09:02.801795+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.822454+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:09:02.822454+0000","last_peered":"2023-09-23T15:09:02.822454+0000","last_clean":"2023-09-23T15:09:02.822454+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:09:02.822454+0000","last_undegraded":"2023-09-23T15:09:02.822454+0000","last_fullsized":"2023-09-23T15:09:02.822454+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":104,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.794685+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:09:02.794685+0000","last_peered":"2023-09-23T15:09:02.794685+0000","last_clean":"2023-09-23T15:09:02.794685+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:09:02.794685+0000","last_undegraded":"2023-09-23T15:09:02.794685+0000","last_fullsized":"2023-09-23T15:09:02.794685+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":119,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.790818+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:09:02.790818+0000","last_peered":"2023-09-23T15:09:02.790818+0000","last_clean":"2023-09-23T15:09:02.790818+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:09:02.790818+0000","last_undegraded":"2023-09-23T15:09:02.790818+0000","last_fullsized":"2023-09-23T15:09:02.790818+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":81,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.789158+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:09:02.789158+0000","last_peered":"2023-09-23T15:09:02.789158+0000","last_clean":"2023-09-23T15:09:02.789158+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:09:02.789158+0000","last_undegraded":"2023-09-23T15:09:02.789158+0000","last_fullsized":"2023-09-23T15:09:02.789158+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":135,"state":"active+clean","last_fresh":"2023-09-23T15:09:02.812438+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:09:02.812438+0000","last_peered":"2023-09-23T15:09:02.812438+0000","last_clean":"2023-09-23T15:09:02.812438+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:09:02.812438+0000","last_undegraded":"2023-09-23T15:09:02.812438+0000","last_fullsized":"2023-09-23T15:09:02.812438+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":7012417560,"num_objects":107002,"num_object_clones":0,"num_object_copies":321006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":33457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":107002,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":116532,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20742549504,"data_stored":20742537288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116532,"ondisk_log_size":116532,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201454,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2193280,"kb_used_data":2138488,"kb_used_omap":0,"kb_used_meta":54784,"kb_avail":91551872,"statfs":{"total":95995035648,"available":93749116928,"internally_reserved":0,"allocated":2189811712,"data_stored":2189268913,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":56098816},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":0.94499999999999995,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.2110000000000003,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.006,"15min":0.72599999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.0310000000000006,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":6.8120000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:08:50 2023","interfaces":[{"interface":"back","average":{"1min":11.904,"5min":2.8719999999999999,"15min":1.3680000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":170.63800000000001,"5min":170.63800000000001,"15min":170.63800000000001},"last":0.76600000000000001},{"interface":"front","average":{"1min":11.869,"5min":2.8300000000000001,"15min":1.3200000000000001},"min":{"1min":0.46000000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":170.47999999999999,"5min":170.47999999999999,"15min":170.47999999999999},"last":6.9470000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:08:24 2023","interfaces":[{"interface":"back","average":{"1min":1.6379999999999999,"5min":0.78000000000000003,"15min":0.63700000000000001},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.1980000000000004,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":3.5179999999999998},{"interface":"front","average":{"1min":1.583,"5min":0.88800000000000001,"15min":0.77200000000000002},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":7.1849999999999996,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":0.86499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.95599999999999996,"5min":0.65400000000000003,"15min":0.55400000000000005},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.1909999999999998,"5min":5.1909999999999998,"15min":5.1909999999999998},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.64000000000000001,"15min":0.56599999999999995},"min":{"1min":0.31900000000000001,"5min":0.308,"15min":0.23999999999999999},"max":{"1min":2.972,"5min":2.972,"15min":2.972},"last":6.766}]},{"osd":5,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.66500000000000004,"15min":0.59599999999999997},"min":{"1min":0.30599999999999999,"5min":0.29299999999999998,"15min":0.26200000000000001},"max":{"1min":3.5089999999999999,"5min":3.5089999999999999,"15min":3.5089999999999999},"last":3.4809999999999999},{"interface":"front","average":{"1min":1.0089999999999999,"5min":0.72599999999999998,"15min":0.625},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.8639999999999999,"5min":2.8639999999999999,"15min":2.8639999999999999},"last":6.8620000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.72999999999999998,"15min":0.63400000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.7909999999999999,"5min":2.7909999999999999,"15min":2.7909999999999999},"last":2.4630000000000001},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.71399999999999997,"15min":0.63500000000000001},"min":{"1min":0.371,"5min":0.36299999999999999,"15min":0.307},"max":{"1min":3.9399999999999999,"5min":3.9399999999999999,"15min":3.9399999999999999},"last":7.0190000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463090,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1758452,"kb_used_data":1715052,"kb_used_omap":0,"kb_used_meta":43392,"kb_avail":91986700,"statfs":{"total":95995035648,"available":94194380800,"internally_reserved":0,"allocated":1756213248,"data_stored":1755682586,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44433408},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":13.331,"5min":3.1190000000000002,"15min":1.403},"min":{"1min":0.26200000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":108.39400000000001,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.32400000000000001},{"interface":"front","average":{"1min":13.425000000000001,"5min":3.1739999999999999,"15min":1.444},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":108.452,"5min":108.452,"15min":108.452},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":26.603999999999999,"5min":7.3860000000000001,"15min":2.8740000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":117.768,"5min":180.374,"15min":180.374},"last":0.48099999999999998},{"interface":"front","average":{"1min":27.042999999999999,"5min":7.5270000000000001,"15min":2.923},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":119.129,"5min":181.685,"15min":181.685},"last":2.0600000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":0.68700000000000006,"15min":0.63100000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.5219999999999998,"5min":2.5219999999999998,"15min":2.5219999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.90400000000000003,"5min":0.66900000000000004,"15min":0.63},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.4710000000000001,"5min":2.4710000000000001,"15min":2.4710000000000001},"last":2.0880000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.70399999999999996,"15min":0.56499999999999995},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":7.4539999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":1.5309999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.76300000000000001,"15min":0.59599999999999997},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.25600000000000001},"max":{"1min":3.8210000000000002,"5min":3.8210000000000002,"15min":3.8210000000000002},"last":0.42399999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":0.79900000000000004,"15min":0.61599999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9669999999999996,"5min":7.9669999999999996,"15min":7.9669999999999996},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0720000000000001,"5min":0.78600000000000003,"15min":0.627},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.9260000000000002,"5min":3.9260000000000002,"15min":3.9260000000000002},"last":3.1869999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.69899999999999995,"15min":0.61799999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.28000000000000003},"max":{"1min":2.5939999999999999,"5min":2.5939999999999999,"15min":2.5939999999999999},"last":2.0139999999999998},{"interface":"front","average":{"1min":1.0620000000000001,"5min":0.77300000000000002,"15min":0.67000000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.373},"max":{"1min":2.927,"5min":2.927,"15min":2.927},"last":0.57199999999999995}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893177,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3535968,"kb_used_data":3432024,"kb_used_omap":8,"kb_used_meta":103863,"kb_avail":90209184,"statfs":{"total":95995035648,"available":92374204416,"internally_reserved":0,"allocated":3514392576,"data_stored":3513764403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":106356308},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":1.3640000000000001,"15min":0.72299999999999998},"min":{"1min":0.308,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":39.130000000000003,"5min":39.130000000000003,"15min":39.130000000000003},"last":2.1850000000000001},{"interface":"front","average":{"1min":3.9500000000000002,"5min":1.1319999999999999,"15min":0.65000000000000002},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":39.034999999999997,"5min":39.034999999999997,"15min":39.034999999999997},"last":1.0429999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3109999999999999,"5min":0.63700000000000001,"15min":0.52500000000000002},"min":{"1min":0.30499999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":6.0739999999999998,"5min":6.0739999999999998,"15min":6.0739999999999998},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.407,"5min":1.087,"15min":1.0329999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.1020000000000003,"5min":11.978,"15min":11.978},"last":1.1459999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":0.90000000000000002,"15min":0.73199999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5800000000000001,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.3700000000000001,"5min":1.01,"15min":0.76100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.0039999999999996,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":1.1020000000000001,"15min":0.83199999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":13.49,"5min":13.49,"15min":13.49},"last":1.075},{"interface":"front","average":{"1min":1.198,"5min":0.94299999999999995,"15min":0.80200000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.181,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.67700000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":1.01,"15min":0.80200000000000005},"min":{"1min":0.39700000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.0540000000000003,"5min":9.0540000000000003,"15min":9.0540000000000003},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.194,"5min":0.97199999999999998,"15min":0.78000000000000003},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.4960000000000004,"5min":5.4960000000000004,"15min":5.4960000000000004},"last":1.125}]},{"osd":7,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":0.99099999999999999,"15min":0.78100000000000003},"min":{"1min":0.48399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.7919999999999998,"5min":5.7919999999999998,"15min":5.7919999999999998},"last":1.101},{"interface":"front","average":{"1min":1.9299999999999999,"5min":1.0780000000000001,"15min":0.83499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":8.1760000000000002,"5min":8.1760000000000002,"15min":8.1760000000000002},"last":6.7240000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566124,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2670664,"kb_used_data":2602944,"kb_used_omap":11,"kb_used_meta":67700,"kb_avail":91074488,"statfs":{"total":95995035648,"available":93260275712,"internally_reserved":0,"allocated":2665414656,"data_stored":2664807545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":69325584},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":0.61899999999999999,"15min":0.47299999999999998},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.6189999999999998,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":1.8740000000000001},{"interface":"front","average":{"1min":1.4059999999999999,"5min":0.59099999999999997,"15min":0.45300000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.38100000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":3.8730000000000002,"5min":1.0860000000000001,"15min":0.622},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":57.695,"5min":57.695,"15min":57.695},"last":1.841},{"interface":"front","average":{"1min":4.2069999999999999,"5min":1.1619999999999999,"15min":0.65400000000000003},"min":{"1min":0.32400000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":58.387999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":1.968}]},{"osd":4,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":0.84599999999999997,"15min":0.71599999999999997},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":4.8250000000000002},"last":0.30299999999999999},{"interface":"front","average":{"1min":1.611,"5min":0.83599999999999997,"15min":0.72099999999999997},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0350000000000001,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":3.419}]},{"osd":5,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":1.0289999999999999,"15min":0.76600000000000001},"min":{"1min":0.441,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.202,"5min":6.202,"15min":6.202},"last":2.0419999999999998},{"interface":"front","average":{"1min":1.855,"5min":0.89300000000000002,"15min":0.753},"min":{"1min":0.48099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.1150000000000002,"5min":4.1150000000000002,"15min":4.1150000000000002},"last":0.47899999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":0.94499999999999995,"15min":0.80300000000000005},"min":{"1min":0.39300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":6.9790000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":2.552,"5min":1.0569999999999999,"15min":0.82499999999999996},"min":{"1min":0.47799999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":6.7720000000000002,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":1.784}]},{"osd":7,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":0.88700000000000001,"15min":0.76700000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.5439999999999996,"5min":4.5439999999999996,"15min":4.5439999999999996},"last":3.0270000000000001},{"interface":"front","average":{"1min":2.2349999999999999,"5min":0.98199999999999998,"15min":0.79200000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.9580000000000002,"5min":4.9580000000000002,"15min":4.9580000000000002},"last":0.41999999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402598,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1340900,"kb_used_data":1283100,"kb_used_omap":3,"kb_used_meta":57788,"kb_avail":92404252,"statfs":{"total":95995035648,"available":94621954048,"internally_reserved":0,"allocated":1313894400,"data_stored":1313318711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":59175081},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":0.48499999999999999,"15min":0.41199999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":2.6589999999999998,"5min":2.6589999999999998,"15min":2.6589999999999998},"last":0.19600000000000001},{"interface":"front","average":{"1min":1.27,"5min":0.56399999999999995,"15min":0.44600000000000001},"min":{"1min":0.27200000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8,"5min":8,"15min":8},"last":0.33300000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":0.67800000000000005,"15min":0.64000000000000001},"min":{"1min":0.34300000000000003,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.8109999999999999,"5min":4.4109999999999996,"15min":4.4109999999999996},"last":0.252},{"interface":"front","average":{"1min":1.4990000000000001,"5min":0.63800000000000001,"15min":0.49399999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":8.0570000000000004,"5min":8.0570000000000004,"15min":8.0570000000000004},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.67800000000000005,"15min":0.625},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":2.8300000000000001,"5min":2.8300000000000001,"15min":2.8300000000000001},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.423,"5min":0.877,"15min":0.78600000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.4859999999999998,"5min":4.4859999999999998,"15min":4.4859999999999998},"last":1.397}]},{"osd":5,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":0.72099999999999997,"15min":0.66600000000000004},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.9430000000000001,"5min":2.9430000000000001,"15min":2.9430000000000001},"last":3.6379999999999999},{"interface":"front","average":{"1min":1.2509999999999999,"5min":0.89300000000000002,"15min":0.83299999999999996},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":4.4450000000000003,"5min":4.4450000000000003,"15min":4.4450000000000003},"last":3.6549999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.70399999999999996,"15min":0.68400000000000005},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":2.891,"5min":2.891,"15min":2.891},"last":1.01},{"interface":"front","average":{"1min":1.2150000000000001,"5min":0.91100000000000003,"15min":0.85999999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":1.349}]},{"osd":7,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":0.92900000000000005,"15min":0.83699999999999997},"min":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"max":{"1min":4.5620000000000003,"5min":4.5620000000000003,"15min":4.5620000000000003},"last":3.6219999999999999},{"interface":"front","average":{"1min":1.026,"5min":0.72599999999999998,"15min":0.67600000000000005},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":4.1660000000000004,"5min":4.1660000000000004,"15min":4.1660000000000004},"last":3.738}]}]},{"osd":4,"up_from":31,"seq":133143986363,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3513140,"kb_used_data":3439532,"kb_used_omap":0,"kb_used_meta":73472,"kb_avail":90232012,"statfs":{"total":95995035648,"available":92397580288,"internally_reserved":0,"allocated":3522080768,"data_stored":3521549619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":75235328},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":2.2469999999999999,"15min":1.107},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7489999999999997,"5min":122.69199999999999,"15min":122.69199999999999},"last":6.8419999999999996},{"interface":"front","average":{"1min":2.992,"5min":2.48,"15min":1.1839999999999999},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.265000000000001,"5min":122.745,"15min":122.745},"last":3.3860000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":5.101,"5min":1.4810000000000001,"15min":0.88800000000000001},"min":{"1min":0.434,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":36.884,"5min":36.884,"15min":36.884},"last":2.4350000000000001},{"interface":"front","average":{"1min":5.8090000000000002,"5min":1.738,"15min":1.071},"min":{"1min":0.58099999999999996,"5min":0.433,"15min":0.433},"max":{"1min":44.177999999999997,"5min":44.177999999999997,"15min":44.177999999999997},"last":2.9319999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":0.78700000000000003,"15min":0.63600000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":4.7240000000000002,"5min":4.7240000000000002,"15min":4.7240000000000002},"last":6.8920000000000003},{"interface":"front","average":{"1min":1.1830000000000001,"5min":0.72499999999999998,"15min":0.64900000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":3.855,"5min":3.855,"15min":3.855},"last":0.40000000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:08:30 2023","interfaces":[{"interface":"back","average":{"1min":1.389,"5min":0.71299999999999997,"15min":0.57799999999999996},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.2309999999999999,"5min":4.2309999999999999,"15min":4.2309999999999999},"last":10.478},{"interface":"front","average":{"1min":1.46,"5min":0.76500000000000001,"15min":0.60499999999999998},"min":{"1min":0.317,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":3.8799999999999999,"5min":3.8799999999999999,"15min":3.8799999999999999},"last":0.35999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":0.93600000000000005,"15min":0.70499999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":10.509},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.92000000000000004,"15min":0.68899999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33400000000000002},"max":{"1min":5.6470000000000002,"5min":5.6470000000000002,"15min":5.6470000000000002},"last":5.7309999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:09:10 2023","interfaces":[{"interface":"back","average":{"1min":3.524,"5min":1.292,"15min":0.82599999999999996},"min":{"1min":0.40300000000000002,"5min":0.376,"15min":0.28799999999999998},"max":{"1min":11.711,"5min":11.711,"15min":11.711},"last":2.3929999999999998},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.296,"15min":0.87},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":14.653,"5min":14.653,"15min":14.653},"last":14.653}]}]},{"osd":5,"up_from":39,"seq":167503724726,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3918316,"kb_used_data":3839460,"kb_used_omap":0,"kb_used_meta":78848,"kb_avail":89826836,"statfs":{"total":95995035648,"available":91982680064,"internally_reserved":0,"allocated":3931607040,"data_stored":3931079870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80740352},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.8460000000000001,"5min":1.0249999999999999,"15min":0.70599999999999996},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":24.962,"5min":24.962,"15min":24.962},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.589,"5min":0.97099999999999997,"15min":0.68100000000000005},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":25.317,"5min":25.317,"15min":25.317},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.0289999999999999,"15min":0.71699999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":19.236999999999998,"5min":19.236999999999998,"15min":19.236999999999998},"last":0.25600000000000001},{"interface":"front","average":{"1min":3.036,"5min":1.1040000000000001,"15min":0.76000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.699999999999999,"5min":19.699999999999999,"15min":19.699999999999999},"last":0.309}]},{"osd":3,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.177,"5min":0.72099999999999997,"15min":0.64500000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":5.0279999999999996,"5min":5.0279999999999996,"15min":5.0279999999999996},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.774,"5min":0.85499999999999998,"15min":0.70199999999999996},"min":{"1min":0.64400000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":4.4249999999999998,"5min":4.4249999999999998,"15min":4.4249999999999998},"last":0.57999999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:08:34 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":0.73999999999999999,"15min":0.54800000000000004},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.26300000000000001},"max":{"1min":5.306,"5min":5.306,"15min":5.306},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.089,"5min":0.67000000000000004,"15min":0.55900000000000005},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.25,"5min":3.25,"15min":3.25},"last":0.38600000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.75,"15min":0.59299999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.4610000000000003,"5min":4.4610000000000003,"15min":4.4610000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.6659999999999999,"5min":2.1230000000000002,"15min":2.1230000000000002},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2989999999999999,"5min":0.754,"15min":0.64400000000000002},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":5.21,"5min":5.21,"15min":5.21},"last":12.311},{"interface":"front","average":{"1min":1.3380000000000001,"5min":0.80100000000000005,"15min":0.65000000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.625}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3498639360,"data_stored":3498639360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2641301504,"data_stored":2641297432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1308819456,"data_stored":1308819456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3502702592,"data_stored":3502702592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3928625152,"data_stored":3928621080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1754267648,"data_stored":1754267648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2181042176,"data_stored":2181038104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:09:17.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T15:09:17.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:17.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:17.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:17.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:18.005 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:18.029 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:18] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.43.0" 2023-09-23T15:09:18.234 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:18.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:18.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:18.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:18.405 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:18.546 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:18.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:18.640 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:18.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:18.663 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:18.671 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:18.709 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:18.742 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:18.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:18.781 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:18.823 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:18.854 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:18.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:18.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:18.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:18.904 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:18.904 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.904 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:18.904 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.904 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.905 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.905 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.905 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.905 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:09:18.906 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:18.906 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:18.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:19.008 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:19.140 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:19.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:19.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:19.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:19.324 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:19.372 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:19.374 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:19 smithi027 ceph-mon[195233]: pgmap v314: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 14 MiB/s wr, 867 op/s; 33215/317460 objects degraded (10.463%) 2023-09-23T15:09:19.374 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2150698252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:19.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:19.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:19 smithi017 ceph-mon[215646]: pgmap v314: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 14 MiB/s wr, 867 op/s; 33215/317460 objects degraded (10.463%) 2023-09-23T15:09:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2150698252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:19.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:19 smithi017 ceph-mon[222878]: pgmap v314: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 109 B/s rd, 14 MiB/s wr, 867 op/s; 33215/317460 objects degraded (10.463%) 2023-09-23T15:09:19.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2150698252' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:19.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:19.502 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:19.802 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:19.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:19.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:19.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:19.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:09:20.017 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:20.168 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:20.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:20.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:20.268 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:20.268 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:20.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:20.329 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:20.655 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:20.859 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:20.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:20.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:20.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:21.001 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:21.113 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:21.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:21.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:21.218 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:21.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:21 smithi027 ceph-mon[195233]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:21.319 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:21.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:21 smithi017 ceph-mon[215646]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:21.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:21 smithi017 ceph-mon[222878]: pgmap v315: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:21.463 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:21.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:21.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:21.571 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:21.627 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:21.671 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:21.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:21.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:21.809 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:22.052 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:22.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:22.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:22.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:22.206 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:22.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:09:22.292 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:22.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 32175/308973 objects degraded (10.414%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T15:09:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:22.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:22.473 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:22.598 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:22.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:22.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 32175/308973 objects degraded (10.414%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T15:09:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 32175/308973 objects degraded (10.414%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T15:09:22.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:22.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:22.789 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:22.790 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:09:22.813 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 1 2023-09-23T15:09:22.824 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:22.933 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:22.933 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:22.956 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:22.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:23.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:23.107 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:23.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[222878]: pgmap v316: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[222878]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[215646]: pgmap v316: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:23 smithi017 ceph-mon[215646]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T15:09:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:23 smithi027 ceph-mon[195233]: pgmap v316: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 147 KiB/s wr, 815 op/s; 32175/308973 objects degraded (10.414%) 2023-09-23T15:09:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:23 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1231086087' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T15:09:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2229643097' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:23 smithi027 ceph-mon[195233]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T15:09:23.730 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:23.733 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:23.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:23.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:09:23.923 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:23.976 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:23.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:23.982 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:24.083 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:24.172 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:24.297 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:24.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:24.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:24.521 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:24 smithi027 ceph-mon[195233]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.521 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:24 smithi027 ceph-mon[195233]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.526 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:24.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:24 smithi017 ceph-mon[222878]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:24 smithi017 ceph-mon[222878]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:24 smithi017 ceph-mon[215646]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:24 smithi017 ceph-mon[215646]: failed to encode map e138 with expected crc 2023-09-23T15:09:24.658 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:24.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:24.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:24.675 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:24.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:24.838 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:24.894 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:24.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:24.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:24.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:25.045 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:25.259 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:25.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:25.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:25.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:25.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:25 smithi017 ceph-mon[215646]: pgmap v318: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 31699/301698 objects degraded (10.507%) 2023-09-23T15:09:25.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:25 smithi017 ceph-mon[222878]: pgmap v318: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 31699/301698 objects degraded (10.507%) 2023-09-23T15:09:25.407 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:25.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:25 smithi027 ceph-mon[195233]: pgmap v318: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 31699/301698 objects degraded (10.507%) 2023-09-23T15:09:25.589 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:25.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:25.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:25.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:25.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:25.772 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:25.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:25.789 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:25.808 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:25.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:25.865 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:25.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:25.898 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:25.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:25.935 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:25.967 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:09:25.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:09:26.013 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:26.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:26.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:26.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:26.167 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:26.248 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:26.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:26.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:26.379 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:26.422 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:26.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:26.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:26.535 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1985659062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:26.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1985659062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:26.555 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1985659062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:26.557 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:26.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:09:26.811 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:26.811 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:26.826 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:26.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:26.908 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126764 5.9 5:9b5a15cb:::benchmark_data_smithi017_288048_object14995:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.885176+0000", 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.014353551000000001, 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014396300000000001, 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:09:26.909 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126764 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885176+0000", 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885176+0000", 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885174+0000", 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885182+0000", 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.028e-06 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885200+0000", 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8423e-05 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.885209+0000", 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2849999999999995e-06 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.887283+0000", 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020740799999999998 2023-09-23T15:09:26.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.887396+0000", 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113757 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.887520+0000", 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123256 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.913 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126769 5.9 5:92e8ec9f:::benchmark_data_smithi017_288048_object15000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.890326+0000", 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0092027759999999993, 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093319249999999996, 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126769 2023-09-23T15:09:26.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890326+0000", 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890326+0000", 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890324+0000", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890340+0000", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5707999999999999e-05 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890354+0000", 2023-09-23T15:09:26.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4331000000000001e-05 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890362+0000", 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1960000000000002e-06 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.895372+0000", 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005010629 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.895497+0000", 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012437599999999999 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.895607+0000", 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110227 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:126765 5.d e138/114 5:bc5405b5:::benchmark_data_smithi017_288048_object14996:head v 138'7898, mlcod=138'7895)", 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.890882+0000", 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0086467439999999996, 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088533459999999998, 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890882+0000", 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890882+0000", 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890881+0000", 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890889+0000", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6979999999999998e-06 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890911+0000", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1831000000000002e-05 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890917+0000", 2023-09-23T15:09:26.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3759999999999996e-06 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890973+0000", 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5943000000000003e-05 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.890992+0000", 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8711000000000001e-05 2023-09-23T15:09:26.922 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:126770 5.1 e138/114 5:80e77d2a:::benchmark_data_smithi017_288048_object15001:head v 138'7883, mlcod=138'7881)", 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.891078+0000", 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0084508099999999996, 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00871977, 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891078+0000", 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891078+0000", 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891078+0000", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891085+0000", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6109999999999996e-06 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891097+0000", 2023-09-23T15:09:26.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1569999999999999e-05 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891102+0000", 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3430000000000003e-06 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891620+0000", 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000518357 2023-09-23T15:09:26.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.891639+0000", 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8006e-05 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:126773 5.d e138/114 5:b0f02887:::benchmark_data_smithi017_288048_object15004:head v 138'7899, mlcod=138'7896)", 2023-09-23T15:09:26.927 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.899355+0000", 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.000174535, 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00050770500000000001, 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899355+0000", 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899355+0000", 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899353+0000", 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899362+0000", 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9279999999999999e-06 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899374+0000", 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1769e-05 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899381+0000", 2023-09-23T15:09:26.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4710000000000004e-06 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899419+0000", 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8136000000000002e-05 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899438+0000", 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9536999999999999e-05 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.931 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126778 5.4 5:2d273c6e:::benchmark_data_smithi017_288048_object15009:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.899439+0000", 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 9.0041000000000001e-05, 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00048446900000000001, 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:26.932 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126778 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899439+0000", 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899439+0000", 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899438+0000", 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899443+0000", 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.254e-06 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899453+0000", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0166000000000001e-05 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899459+0000", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4489999999999998e-06 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899505+0000", 2023-09-23T15:09:26.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6644000000000003e-05 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899629+0000", 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012378899999999999 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899742+0000", 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112656 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:126764 5.9 e138/114)", 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.899699+0000", 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9998302, 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029488600000000001, 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899699+0000", 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899699+0000", 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899698+0000", 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899702+0000", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7179999999999998e-06 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899714+0000", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2343e-05 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899720+0000", 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8649999999999998e-06 2023-09-23T15:09:26.939 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:126769 5.9 e138/114)", 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.899967+0000", 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9995627, 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2717000000000002e-05, 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899967+0000", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899967+0000", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899966+0000", 2023-09-23T15:09:26.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899971+0000", 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5920000000000002e-06 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899980+0000", 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4050000000000006e-06 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:09:26.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.899985+0000", 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0900000000000004e-06 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 8 2023-09-23T15:09:26.943 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:09:26.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:27.003 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:27.080 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:27.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:27.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:27.129 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:27.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:27.293 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:27.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:27 smithi017 ceph-mon[222878]: pgmap v319: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 30977/297330 objects degraded (10.418%) 2023-09-23T15:09:27.359 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 30977/297330 objects degraded (10.418%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:27.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:27 smithi017 ceph-mon[215646]: pgmap v319: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 30977/297330 objects degraded (10.418%) 2023-09-23T15:09:27.360 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 30977/297330 objects degraded (10.418%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:27.428 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:27.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:27.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:27.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:27.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:27 smithi027 ceph-mon[195233]: pgmap v319: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 791 op/s; 30977/297330 objects degraded (10.418%) 2023-09-23T15:09:27.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 30977/297330 objects degraded (10.418%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:27.613 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:27.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:27.852 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:09:27.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:27.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:27.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:27.981 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:28.054 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:28.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:28.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:28.172 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:28.224 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:28.299 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:28.300 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:28] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.43.0" 2023-09-23T15:09:28.300 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:28.373 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:28.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:28.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:28.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:28.575 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:28.596 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:28.729 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:28.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:28.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:28.813 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:09:28.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:28.925 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:29.023 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:29.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:29.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:29.130 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:29.180 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:29.261 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:29.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:29.334 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:09:29.368 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[215646]: pgmap v320: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 30747/293847 objects degraded (10.464%) 2023-09-23T15:09:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1441782851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[222878]: pgmap v320: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 30747/293847 objects degraded (10.464%) 2023-09-23T15:09:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1441782851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:29 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.462 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:29.542 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:29.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:29.550 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:29 smithi027 ceph-mon[195233]: pgmap v320: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 6.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 30747/293847 objects degraded (10.464%) 2023-09-23T15:09:29.551 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1441782851' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.551 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:29 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:29.622 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:09:29.622 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:09:29.623 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2023-09-23T15:09:29.623 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:09:29.623 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:09:29.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 config set filestore_inject_stall 3 2023-09-23T15:09:29.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:29.678 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:29.754 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:29.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:29.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:29.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:29.938 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:30.158 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:30.247 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:30.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:30.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:30.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:30.420 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[222878]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[222878]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[222878]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[215646]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[215646]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T15:09:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:30 smithi017 ceph-mon[215646]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:30 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:30 smithi027 ceph-mon[195233]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:30 smithi027 ceph-mon[195233]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T15:09:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:30 smithi027 ceph-mon[195233]: failed to encode map e139 with expected crc 2023-09-23T15:09:30.602 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:30.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:30.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:30.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:30.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:30.827 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:30.898 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:09:30.898 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:09:30.898 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:09:30.899 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:09:30.899 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:09:31.053 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:31.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:31.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:31.251 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:31.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:31.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:31.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:31.524 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:31.574 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:31.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:31 smithi027 ceph-mon[195233]: pgmap v322: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:31.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:31 smithi017 ceph-mon[215646]: pgmap v322: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:31.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:31 smithi017 ceph-mon[222878]: pgmap v322: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 961 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:31.711 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:31.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:31.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:31.753 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:09:31.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:31.870 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:32.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:09:32.273 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:32.302 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:32.323 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:09:32.341 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29780/285900 objects degraded (10.416%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/438017877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:32.375 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:32.407 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:32.448 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:32.495 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:09:32.514 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:32.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29780/285900 objects degraded (10.416%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:32.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/438017877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:32.517 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29780/285900 objects degraded (10.416%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:32.517 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/438017877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:32.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:32.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:32.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:32.689 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:32.891 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:32.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:33.001 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:33.061 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:33.180 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:33.180 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:33.194 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:33.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:33.301 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:33.328 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:33.369 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:33.371 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:33.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:33.374 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:33.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:33.494 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:09:33.494 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:09:33.494 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:09:33.532 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:33.757 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:33.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:33.764 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:33.778 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:33 smithi017 ceph-mon[222878]: pgmap v323: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:33.865 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:33.934 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:34.004 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:34.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:34.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:34.067 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:33 smithi017 ceph-mon[215646]: pgmap v323: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:34.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:33 smithi027 ceph-mon[195233]: pgmap v323: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 29780/285900 objects degraded (10.416%) 2023-09-23T15:09:34.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:34.152 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:34.512 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:34.513 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:34.850 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:09:34 smithi027 conmon[174777]: logger=cleanup t=2023-09-23T15:09:34.52753212Z level=info msg="Completed cleanup jobs" duration=111.365221ms 2023-09-23T15:09:34.986 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:34.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:35.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:35.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:35.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:09:35.400 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:35.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:09:35.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:35.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:35.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:35.596 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:35.686 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:35.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:35.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:35.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:35 smithi017 ceph-mon[215646]: pgmap v324: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29307/278868 objects degraded (10.509%) 2023-09-23T15:09:35.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:35.842 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:35 smithi027 ceph-mon[195233]: pgmap v324: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29307/278868 objects degraded (10.509%) 2023-09-23T15:09:35.850 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:35.915 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:35.949 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:35.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:35.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:36.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:36.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:35 smithi017 ceph-mon[222878]: pgmap v324: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 760 op/s; 29307/278868 objects degraded (10.509%) 2023-09-23T15:09:36.110 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:36.176 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:36.195 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:36.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:36.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:36.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:36.369 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:36.595 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:36.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:36.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:36.757 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:36.823 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:09:36.849 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:36.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:36.955 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:37.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:37.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:36 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29307/278868 objects degraded (10.509%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:37.022 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:37.180 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:37.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:37.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:36 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29307/278868 objects degraded (10.509%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:09:37.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:36 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29307/278868 objects degraded (10.509%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:37.292 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:37.360 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:37.410 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:37.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:37.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:37.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:37.601 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:37.729 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:37.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:37.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:37.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:37.894 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: pgmap v325: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 28634/274806 objects degraded (10.420%) 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:37 smithi027 ceph-mon[195233]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.115 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: pgmap v325: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 28634/274806 objects degraded (10.420%) 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.119 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[222878]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:38.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: pgmap v325: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 28634/274806 objects degraded (10.420%) 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3259282177' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:37 smithi017 ceph-mon[215646]: failed to encode map e140 with expected crc 2023-09-23T15:09:38.121 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:38] "GET /metrics HTTP/1.1" 200 36311 "" "Prometheus/2.43.0" 2023-09-23T15:09:38.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:38.250 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:38.696 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:38.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:38.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:38.805 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:38.870 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:39.045 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:39.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:39.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:39.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:39.228 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:39.280 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:39.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:39.332 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:39.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:39.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:39.359 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:09:39.359 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:136456 5.5 5:a5bc7366:::benchmark_data_smithi017_288048_object24687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:39.341479+0000", 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.014522264, 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014556575, 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:39.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 136456 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:39.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341479+0000", 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341479+0000", 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:39.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341478+0000", 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341483+0000", 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1200000000000004e-06 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:39.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341494+0000", 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1008e-05 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.341500+0000", 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.742e-06 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351047+0000", 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095461910000000007 2023-09-23T15:09:39.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351208+0000", 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000161218 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351351+0000", 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014284200000000001 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:39.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:136463 5.5 5:a5b0f885:::benchmark_data_smithi017_288048_object24694:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:39.351805+0000", 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0041963970000000001, 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.004315887, 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:39.367 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 136463 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351805+0000", 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351805+0000", 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351803+0000", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351811+0000", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3889999999999999e-06 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351828+0000", 2023-09-23T15:09:39.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7041000000000002e-05 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.351837+0000", 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6719999999999993e-06 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.355047+0000", 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032106359999999998 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.355145+0000", 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7411000000000006e-05 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:39.355243+0000", 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7952e-05 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:39.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:39.372 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:09:39.372 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2023-09-23T15:09:39.372 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:09:39.386 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:39.436 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:39.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:39.476 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:39.512 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.513 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:09:39.520 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:39.642 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:39.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:39.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:39.787 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:39.791 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:09:39.791 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:09:39.791 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T15:09:39.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:09:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:39 smithi027 ceph-mon[195233]: pgmap v327: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 28393/271233 objects degraded (10.468%) 2023-09-23T15:09:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/763792213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:39 smithi017 ceph-mon[222878]: pgmap v327: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 28393/271233 objects degraded (10.468%) 2023-09-23T15:09:40.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/763792213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:39 smithi017 ceph-mon[215646]: pgmap v327: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 551 op/s; 28393/271233 objects degraded (10.468%) 2023-09-23T15:09:40.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/763792213' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:40.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:09:40.842 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:40.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:40.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:40.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:09:40.927 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:40.928 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:40.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:41.008 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:41.064 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:41.097 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:41.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:41.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:41.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:41.274 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:41.377 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:41.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:41.490 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:41.552 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:41.573 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:41.702 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:41.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:41.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:41.817 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:41.846 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:41 smithi017 ceph-mon[215646]: pgmap v328: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:41.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:41 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27409/262785 objects degraded (10.430%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:41 smithi017 ceph-mon[222878]: pgmap v328: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:41 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27409/262785 objects degraded (10.430%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:42.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:41 smithi027 ceph-mon[195233]: pgmap v328: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:42.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:41 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27409/262785 objects degraded (10.430%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:42.222 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:42.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:42.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:42.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:42.409 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:42.502 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:42.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:42.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:42.656 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:42.667 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:42.870 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:42.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:42.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:42.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:09:42.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:43.046 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:43.562 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:43.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:43.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:43.647 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:09:43.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:43.674 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:09:43.749 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:43.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:43 smithi017 ceph-mon[215646]: pgmap v329: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:43.857 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:43.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:43 smithi017 ceph-mon[222878]: pgmap v329: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:43.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:44.192 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:44.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:44.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:44.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:43 smithi027 ceph-mon[195233]: pgmap v329: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 27409/262785 objects degraded (10.430%) 2023-09-23T15:09:44.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:44.359 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:44.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:44 smithi017 ceph-mon[215646]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:44.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:44 smithi017 ceph-mon[215646]: pgmap v330: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 26909/255183 objects degraded (10.545%) 2023-09-23T15:09:44.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:44 smithi017 ceph-mon[222878]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:44.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:44 smithi017 ceph-mon[222878]: pgmap v330: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 26909/255183 objects degraded (10.545%) 2023-09-23T15:09:45.118 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":329,"stamp":"2023-09-23T15:09:42.174382+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5743515595,"num_objects":87595,"num_object_clones":0,"num_object_copies":262785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27409,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87595,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":137113,"num_write_kb":7168166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133859,"ondisk_log_size":133859,"up":230,"acting":230,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17534108,"kb_used_data":16987036,"kb_used_omap":30,"kb_used_meta":546849,"kb_avail":732427108,"statfs":{"total":767960285184,"available":750005358592,"internally_reserved":0,"allocated":17394724864,"data_stored":17390168607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":559973976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-504954880,"num_objects":-7705,"num_object_clones":0,"num_object_copies":-23115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2371,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7705,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004527"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":124,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.315182+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:09:29.315182+0000","last_peered":"2023-09-23T15:09:29.315182+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:09:29.315182+0000","last_undegraded":"2023-09-23T15:09:29.315182+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":66,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321021+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:09:29.321021+0000","last_peered":"2023-09-23T15:09:29.321021+0000","last_clean":"2023-09-23T15:09:29.321021+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:09:29.321021+0000","last_undegraded":"2023-09-23T15:09:29.321021+0000","last_fullsized":"2023-09-23T15:09:29.321021+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":54,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.321525+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:09:29.321525+0000","last_peered":"2023-09-23T15:09:29.321525+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:09:29.321525+0000","last_undegraded":"2023-09-23T15:09:29.321525+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.323832+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:09:29.323832+0000","last_peered":"2023-09-23T15:09:29.323832+0000","last_clean":"2023-09-23T15:09:29.323832+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:09:29.323832+0000","last_undegraded":"2023-09-23T15:09:29.323832+0000","last_fullsized":"2023-09-23T15:09:29.323832+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.320656+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:09:29.320656+0000","last_peered":"2023-09-23T15:09:29.320656+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:09:29.320656+0000","last_undegraded":"2023-09-23T15:09:29.320656+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":104,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.334389+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:09:29.334389+0000","last_peered":"2023-09-23T15:09:29.334389+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:09:29.334389+0000","last_undegraded":"2023-09-23T15:09:29.334389+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.324453+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:09:29.324453+0000","last_peered":"2023-09-23T15:09:29.324453+0000","last_clean":"2023-09-23T15:09:29.324453+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:09:29.324453+0000","last_undegraded":"2023-09-23T15:09:29.324453+0000","last_fullsized":"2023-09-23T15:09:29.324453+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.320466+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:09:29.320466+0000","last_peered":"2023-09-23T15:09:29.320466+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:09:29.320466+0000","last_undegraded":"2023-09-23T15:09:29.320466+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":106,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.331008+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:09:29.331008+0000","last_peered":"2023-09-23T15:09:29.331008+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:09:29.331008+0000","last_undegraded":"2023-09-23T15:09:29.331008+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":51,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321466+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:09:29.321466+0000","last_peered":"2023-09-23T15:09:29.321466+0000","last_clean":"2023-09-23T15:09:29.321466+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:09:29.321466+0000","last_undegraded":"2023-09-23T15:09:29.321466+0000","last_fullsized":"2023-09-23T15:09:29.321466+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":101,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.333625+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:09:29.333625+0000","last_peered":"2023-09-23T15:09:29.333625+0000","last_clean":"2023-09-23T15:09:29.333625+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:09:29.333625+0000","last_undegraded":"2023-09-23T15:09:29.333625+0000","last_fullsized":"2023-09-23T15:09:29.333625+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":50,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.331077+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:09:29.331077+0000","last_peered":"2023-09-23T15:09:29.331077+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:09:29.331077+0000","last_undegraded":"2023-09-23T15:09:29.331077+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":104,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.315543+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:09:29.315543+0000","last_peered":"2023-09-23T15:09:29.315543+0000","last_clean":"2023-09-23T15:09:29.315543+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:09:29.315543+0000","last_undegraded":"2023-09-23T15:09:29.315543+0000","last_fullsized":"2023-09-23T15:09:29.315543+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":105,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.314475+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:09:29.314475+0000","last_peered":"2023-09-23T15:09:29.314475+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:09:29.314475+0000","last_undegraded":"2023-09-23T15:09:29.314475+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.333436+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:09:29.333436+0000","last_peered":"2023-09-23T15:09:29.333436+0000","last_clean":"2023-09-23T15:09:29.333436+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:09:29.333436+0000","last_undegraded":"2023-09-23T15:09:29.333436+0000","last_fullsized":"2023-09-23T15:09:29.333436+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.321474+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:09:29.321474+0000","last_peered":"2023-09-23T15:09:29.321474+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:09:29.321474+0000","last_undegraded":"2023-09-23T15:09:29.321474+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.325895+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:09:29.325895+0000","last_peered":"2023-09-23T15:09:29.325895+0000","last_clean":"2023-09-23T15:09:29.325895+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:09:29.325895+0000","last_undegraded":"2023-09-23T15:09:29.325895+0000","last_fullsized":"2023-09-23T15:09:29.325895+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.330468+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:09:29.330468+0000","last_peered":"2023-09-23T15:09:29.330468+0000","last_clean":"2023-09-23T15:09:29.330468+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:09:29.330468+0000","last_undegraded":"2023-09-23T15:09:29.330468+0000","last_fullsized":"2023-09-23T15:09:29.330468+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.341070+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:09:29.341070+0000","last_peered":"2023-09-23T15:09:29.341070+0000","last_clean":"2023-09-23T15:09:29.341070+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:09:29.341070+0000","last_undegraded":"2023-09-23T15:09:29.341070+0000","last_fullsized":"2023-09-23T15:09:29.341070+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.323759+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:09:29.323759+0000","last_peered":"2023-09-23T15:09:29.323759+0000","last_clean":"2023-09-23T15:09:29.323759+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:09:29.323759+0000","last_undegraded":"2023-09-23T15:09:29.323759+0000","last_fullsized":"2023-09-23T15:09:29.323759+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":106,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.330493+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:09:29.330493+0000","last_peered":"2023-09-23T15:09:29.330493+0000","last_clean":"2023-09-23T15:09:29.330493+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:09:29.330493+0000","last_undegraded":"2023-09-23T15:09:29.330493+0000","last_fullsized":"2023-09-23T15:09:29.330493+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.321295+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:09:29.321295+0000","last_peered":"2023-09-23T15:09:29.321295+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:09:29.321295+0000","last_undegraded":"2023-09-23T15:09:29.321295+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"140'8608","reported_seq":8639,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:39.115294+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:09:39.115294+0000","last_peered":"2023-09-23T15:09:39.115294+0000","last_clean":"2023-09-23T15:09:39.115294+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:09:39.115294+0000","last_undegraded":"2023-09-23T15:09:39.115294+0000","last_fullsized":"2023-09-23T15:09:39.115294+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8608,"log_dups_size":0,"ondisk_log_size":8608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358481920,"num_objects":5470,"num_object_clones":0,"num_object_copies":16410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8608,"num_write_kb":450496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.341202+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:09:29.341202+0000","last_peered":"2023-09-23T15:09:29.341202+0000","last_clean":"2023-09-23T15:09:29.341202+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:09:29.341202+0000","last_undegraded":"2023-09-23T15:09:29.341202+0000","last_fullsized":"2023-09-23T15:09:29.341202+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.329402+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:09:29.329402+0000","last_peered":"2023-09-23T15:09:29.329402+0000","last_clean":"2023-09-23T15:09:29.329402+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:09:29.329402+0000","last_undegraded":"2023-09-23T15:09:29.329402+0000","last_fullsized":"2023-09-23T15:09:29.329402+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"140'8429","reported_seq":8469,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:38.020315+0000","last_change":"2023-09-23T15:08:51.249550+0000","last_active":"2023-09-23T15:09:38.020315+0000","last_peered":"2023-09-23T15:09:38.020315+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:08:51.249550+0000","last_became_peered":"2023-09-23T15:08:51.249550+0000","last_unstale":"2023-09-23T15:09:38.020315+0000","last_undegraded":"2023-09-23T15:08:51.245609+0000","last_fullsized":"2023-09-23T15:08:51.245485+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8429,"log_dups_size":0,"ondisk_log_size":8429,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.928709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361955328,"num_objects":5523,"num_object_clones":0,"num_object_copies":16569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5523,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8429,"num_write_kb":446464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":5523}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":107,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.341254+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:09:29.341254+0000","last_peered":"2023-09-23T15:09:29.341254+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:09:29.341254+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.316478+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:09:29.316478+0000","last_peered":"2023-09-23T15:09:29.316478+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:09:29.316478+0000","last_undegraded":"2023-09-23T15:09:29.316478+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"140'8475","reported_seq":8509,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.033148+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:09:38.033148+0000","last_peered":"2023-09-23T15:09:38.033148+0000","last_clean":"2023-09-23T15:09:38.033148+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:09:38.033148+0000","last_undegraded":"2023-09-23T15:09:38.033148+0000","last_fullsized":"2023-09-23T15:09:38.033148+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8475,"log_dups_size":0,"ondisk_log_size":8475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363397120,"num_objects":5545,"num_object_clones":0,"num_object_copies":16635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8475,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.341327+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:09:29.341327+0000","last_peered":"2023-09-23T15:09:29.341327+0000","last_clean":"2023-09-23T15:09:29.341327+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:09:29.341327+0000","last_undegraded":"2023-09-23T15:09:29.341327+0000","last_fullsized":"2023-09-23T15:09:29.341327+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":104,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.331372+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:09:29.331372+0000","last_peered":"2023-09-23T15:09:29.331372+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:09:29.331372+0000","last_undegraded":"2023-09-23T15:09:29.331372+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"140'8409","reported_seq":8444,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.019239+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:09:38.019239+0000","last_peered":"2023-09-23T15:09:38.019239+0000","last_clean":"2023-09-23T15:09:38.019239+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:09:38.019239+0000","last_undegraded":"2023-09-23T15:09:38.019239+0000","last_fullsized":"2023-09-23T15:09:38.019239+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8409,"log_dups_size":0,"ondisk_log_size":8409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365494272,"num_objects":5577,"num_object_clones":0,"num_object_copies":16731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8409,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.329674+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:29.329674+0000","last_peered":"2023-09-23T15:09:29.329674+0000","last_clean":"2023-09-23T15:09:29.329674+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:09:29.329674+0000","last_undegraded":"2023-09-23T15:09:29.329674+0000","last_fullsized":"2023-09-23T15:09:29.329674+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":88,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.323803+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:09:29.323803+0000","last_peered":"2023-09-23T15:09:29.323803+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:09:29.323803+0000","last_undegraded":"2023-09-23T15:09:29.323803+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"140'8422","reported_seq":8458,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.029424+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:09:38.029424+0000","last_peered":"2023-09-23T15:09:38.029424+0000","last_clean":"2023-09-23T15:09:38.029424+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:09:38.029424+0000","last_undegraded":"2023-09-23T15:09:38.029424+0000","last_fullsized":"2023-09-23T15:09:38.029424+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8422,"log_dups_size":0,"ondisk_log_size":8422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359464984,"num_objects":5486,"num_object_clones":0,"num_object_copies":16458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5486,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8422,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321200+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:09:29.321200+0000","last_peered":"2023-09-23T15:09:29.321200+0000","last_clean":"2023-09-23T15:09:29.321200+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:09:29.321200+0000","last_undegraded":"2023-09-23T15:09:29.321200+0000","last_fullsized":"2023-09-23T15:09:29.321200+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":98,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.315623+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:09:29.315623+0000","last_peered":"2023-09-23T15:09:29.315623+0000","last_clean":"2023-09-23T15:09:29.315623+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:09:29.315623+0000","last_undegraded":"2023-09-23T15:09:29.315623+0000","last_fullsized":"2023-09-23T15:09:29.315623+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"140'8482","reported_seq":8522,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:39.153360+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:09:39.153360+0000","last_peered":"2023-09-23T15:09:39.153360+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:09:39.153360+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"119'1100","ondisk_log_start":"119'1100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359792640,"num_objects":5490,"num_object_clones":0,"num_object_copies":16470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5490,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8482,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":5490}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":105,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.331551+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:09:29.331551+0000","last_peered":"2023-09-23T15:09:29.331551+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:09:29.331551+0000","last_undegraded":"2023-09-23T15:09:29.331551+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.315868+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:09:29.315868+0000","last_peered":"2023-09-23T15:09:29.315868+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:09:29.315868+0000","last_undegraded":"2023-09-23T15:09:29.315868+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"140'8718","reported_seq":8752,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:39.192754+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:09:39.192754+0000","last_peered":"2023-09-23T15:09:39.192754+0000","last_clean":"2023-09-23T15:09:39.192754+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:09:39.192754+0000","last_undegraded":"2023-09-23T15:09:39.192754+0000","last_fullsized":"2023-09-23T15:09:39.192754+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8718,"log_dups_size":0,"ondisk_log_size":8718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360579072,"num_objects":5502,"num_object_clones":0,"num_object_copies":16506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8718,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.316314+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:09:29.316314+0000","last_peered":"2023-09-23T15:09:29.316314+0000","last_clean":"2023-09-23T15:09:29.316314+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:09:29.316314+0000","last_undegraded":"2023-09-23T15:09:29.316314+0000","last_fullsized":"2023-09-23T15:09:29.316314+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":105,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.316373+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:09:29.316373+0000","last_peered":"2023-09-23T15:09:29.316373+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:09:29.316373+0000","last_undegraded":"2023-09-23T15:09:29.316373+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"140'8441","reported_seq":8475,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:39.176182+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:09:39.176182+0000","last_peered":"2023-09-23T15:09:39.176182+0000","last_clean":"2023-09-23T15:09:39.176182+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:09:39.176182+0000","last_undegraded":"2023-09-23T15:09:39.176182+0000","last_fullsized":"2023-09-23T15:09:39.176182+0000","mapping_epoch":114,"log_start":"119'1100","ondisk_log_start":"119'1100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7341,"log_dups_size":0,"ondisk_log_size":7341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352649216,"num_objects":5381,"num_object_clones":0,"num_object_copies":16143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8441,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.320854+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:09:29.320854+0000","last_peered":"2023-09-23T15:09:29.320854+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:09:29.320854+0000","last_undegraded":"2023-09-23T15:09:29.320854+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.321518+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:09:29.321518+0000","last_peered":"2023-09-23T15:09:29.321518+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:09:29.321518+0000","last_undegraded":"2023-09-23T15:09:29.321518+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":331,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.324755+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:09:29.324755+0000","last_peered":"2023-09-23T15:09:29.324755+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:09:29.324755+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":116,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.330602+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:29.330602+0000","last_peered":"2023-09-23T15:09:29.330602+0000","last_clean":"2023-09-23T15:09:29.330602+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:09:29.330602+0000","last_undegraded":"2023-09-23T15:09:29.330602+0000","last_fullsized":"2023-09-23T15:09:29.330602+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.317799+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:09:29.317799+0000","last_peered":"2023-09-23T15:09:29.317799+0000","last_clean":"2023-09-23T15:09:29.317799+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:09:29.317799+0000","last_undegraded":"2023-09-23T15:09:29.317799+0000","last_fullsized":"2023-09-23T15:09:29.317799+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"140'8306","reported_seq":8339,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.986841+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:09:38.986841+0000","last_peered":"2023-09-23T15:09:38.986841+0000","last_clean":"2023-09-23T15:09:38.986841+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:09:38.986841+0000","last_undegraded":"2023-09-23T15:09:38.986841+0000","last_fullsized":"2023-09-23T15:09:38.986841+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8306,"log_dups_size":0,"ondisk_log_size":8306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346030080,"num_objects":5280,"num_object_clones":0,"num_object_copies":15840,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5280,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8306,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"140'8666","reported_seq":8707,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:39.002008+0000","last_change":"2023-09-23T15:08:51.256935+0000","last_active":"2023-09-23T15:09:39.002008+0000","last_peered":"2023-09-23T15:09:39.002008+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:08:51.256935+0000","last_became_peered":"2023-09-23T15:08:51.256935+0000","last_unstale":"2023-09-23T15:09:39.002008+0000","last_undegraded":"2023-09-23T15:08:51.248529+0000","last_fullsized":"2023-09-23T15:08:51.248430+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8666,"log_dups_size":0,"ondisk_log_size":8666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:33.609757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359137280,"num_objects":5480,"num_object_clones":0,"num_object_copies":16440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8666,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":5480}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":648,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.330098+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:09:29.330098+0000","last_peered":"2023-09-23T15:09:29.330098+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:09:29.330098+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":99,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.314384+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:09:29.314384+0000","last_peered":"2023-09-23T15:09:29.314384+0000","last_clean":"2023-09-23T15:09:29.314384+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:09:29.314384+0000","last_undegraded":"2023-09-23T15:09:29.314384+0000","last_fullsized":"2023-09-23T15:09:29.314384+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":69,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.317473+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:09:29.317473+0000","last_peered":"2023-09-23T15:09:29.317473+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:09:29.317473+0000","last_undegraded":"2023-09-23T15:09:29.317473+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.6","version":"140'8393","reported_seq":8428,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.040490+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:09:38.040490+0000","last_peered":"2023-09-23T15:09:38.040490+0000","last_clean":"2023-09-23T15:09:38.040490+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:09:38.040490+0000","last_undegraded":"2023-09-23T15:09:38.040490+0000","last_fullsized":"2023-09-23T15:09:38.040490+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8393,"log_dups_size":0,"ondisk_log_size":8393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363003904,"num_objects":5539,"num_object_clones":0,"num_object_copies":16617,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5539,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8393,"num_write_kb":445824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":749,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.317269+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:09:29.317269+0000","last_peered":"2023-09-23T15:09:29.317269+0000","last_clean":"2023-09-23T15:09:29.317269+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:09:29.317269+0000","last_undegraded":"2023-09-23T15:09:29.317269+0000","last_fullsized":"2023-09-23T15:09:29.317269+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":49,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.317533+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:09:29.317533+0000","last_peered":"2023-09-23T15:09:29.317533+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:09:29.317533+0000","last_undegraded":"2023-09-23T15:09:29.317533+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":74,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.330704+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:09:29.330704+0000","last_peered":"2023-09-23T15:09:29.330704+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:09:29.330704+0000","last_undegraded":"2023-09-23T15:09:29.330704+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":385,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.332449+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:09:29.332449+0000","last_peered":"2023-09-23T15:09:29.332449+0000","last_clean":"2023-09-23T15:09:29.332449+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:09:29.332449+0000","last_undegraded":"2023-09-23T15:09:29.332449+0000","last_fullsized":"2023-09-23T15:09:29.332449+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":479,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.333062+0000","last_change":"2023-09-23T15:04:40.700312+0000","last_active":"2023-09-23T15:09:29.333062+0000","last_peered":"2023-09-23T15:09:29.333062+0000","last_clean":"2023-09-23T15:09:29.333062+0000","last_became_active":"2023-09-23T15:04:40.699755+0000","last_became_peered":"2023-09-23T15:04:40.699755+0000","last_unstale":"2023-09-23T15:09:29.333062+0000","last_undegraded":"2023-09-23T15:09:29.333062+0000","last_fullsized":"2023-09-23T15:09:29.333062+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:12.069117+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":51,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321273+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:09:29.321273+0000","last_peered":"2023-09-23T15:09:29.321273+0000","last_clean":"2023-09-23T15:09:29.321273+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:09:29.321273+0000","last_undegraded":"2023-09-23T15:09:29.321273+0000","last_fullsized":"2023-09-23T15:09:29.321273+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.331252+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:09:29.331252+0000","last_peered":"2023-09-23T15:09:29.331252+0000","last_clean":"2023-09-23T15:09:29.331252+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:09:29.331252+0000","last_undegraded":"2023-09-23T15:09:29.331252+0000","last_fullsized":"2023-09-23T15:09:29.331252+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"140'8405","reported_seq":8438,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:38.995233+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:09:38.995233+0000","last_peered":"2023-09-23T15:09:38.995233+0000","last_clean":"2023-09-23T15:09:38.995233+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:09:38.995233+0000","last_undegraded":"2023-09-23T15:09:38.995233+0000","last_fullsized":"2023-09-23T15:09:38.995233+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8405,"log_dups_size":0,"ondisk_log_size":8405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354746368,"num_objects":5413,"num_object_clones":0,"num_object_copies":16239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8405,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"140'8580","reported_seq":8621,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:39.182383+0000","last_change":"2023-09-23T15:08:51.252737+0000","last_active":"2023-09-23T15:09:39.182383+0000","last_peered":"2023-09-23T15:09:39.182383+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:08:51.252737+0000","last_became_peered":"2023-09-23T15:08:51.252737+0000","last_unstale":"2023-09-23T15:09:39.182383+0000","last_undegraded":"2023-09-23T15:08:51.248004+0000","last_fullsized":"2023-09-23T15:08:51.247871+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8580,"log_dups_size":0,"ondisk_log_size":8580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.686400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362807296,"num_objects":5536,"num_object_clones":0,"num_object_copies":16608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5536,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8580,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":5536}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":351,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.333152+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:09:29.333152+0000","last_peered":"2023-09-23T15:09:29.333152+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:09:29.333152+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":82,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.323536+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:09:29.323536+0000","last_peered":"2023-09-23T15:09:29.323536+0000","last_clean":"2023-09-23T15:09:29.323536+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:09:29.323536+0000","last_undegraded":"2023-09-23T15:09:29.323536+0000","last_fullsized":"2023-09-23T15:09:29.323536+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.324051+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:09:29.324051+0000","last_peered":"2023-09-23T15:09:29.324051+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:09:29.324051+0000","last_undegraded":"2023-09-23T15:09:29.324051+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":809,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.333643+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:09:29.333643+0000","last_peered":"2023-09-23T15:09:29.333643+0000","last_clean":"2023-09-23T15:09:29.333643+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:09:29.333643+0000","last_undegraded":"2023-09-23T15:09:29.333643+0000","last_fullsized":"2023-09-23T15:09:29.333643+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":49,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.317622+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:09:29.317622+0000","last_peered":"2023-09-23T15:09:29.317622+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:09:29.317622+0000","last_undegraded":"2023-09-23T15:09:29.317622+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":67,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.324058+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:09:29.324058+0000","last_peered":"2023-09-23T15:09:29.324058+0000","last_clean":"2023-09-23T15:09:29.324058+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:09:29.324058+0000","last_undegraded":"2023-09-23T15:09:29.324058+0000","last_fullsized":"2023-09-23T15:09:29.324058+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"140'8687","reported_seq":8720,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:39.082694+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:09:39.082694+0000","last_peered":"2023-09-23T15:09:39.082694+0000","last_clean":"2023-09-23T15:09:39.082694+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:09:39.082694+0000","last_undegraded":"2023-09-23T15:09:39.082694+0000","last_fullsized":"2023-09-23T15:09:39.082694+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8687,"log_dups_size":0,"ondisk_log_size":8687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":362610688,"num_objects":5533,"num_object_clones":0,"num_object_copies":16599,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5533,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8687,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":100,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321421+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:09:29.321421+0000","last_peered":"2023-09-23T15:09:29.321421+0000","last_clean":"2023-09-23T15:09:29.321421+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:09:29.321421+0000","last_undegraded":"2023-09-23T15:09:29.321421+0000","last_fullsized":"2023-09-23T15:09:29.321421+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":634,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.331358+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:09:29.331358+0000","last_peered":"2023-09-23T15:09:29.331358+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:09:29.331358+0000","last_undegraded":"2023-09-23T15:09:29.331358+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":97,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.320762+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:09:29.320762+0000","last_peered":"2023-09-23T15:09:29.320762+0000","last_clean":"2023-09-23T15:09:29.320762+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:09:29.320762+0000","last_undegraded":"2023-09-23T15:09:29.320762+0000","last_fullsized":"2023-09-23T15:09:29.320762+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"140'8510","reported_seq":8544,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:39.146605+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:09:39.146605+0000","last_peered":"2023-09-23T15:09:39.146605+0000","last_clean":"2023-09-23T15:09:39.146605+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:09:39.146605+0000","last_undegraded":"2023-09-23T15:09:39.146605+0000","last_fullsized":"2023-09-23T15:09:39.146605+0000","mapping_epoch":114,"log_start":"119'1100","ondisk_log_start":"119'1100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7410,"log_dups_size":0,"ondisk_log_size":7410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356384768,"num_objects":5438,"num_object_clones":0,"num_object_copies":16314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8510,"num_write_kb":446336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":570,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.334299+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:09:29.334299+0000","last_peered":"2023-09-23T15:09:29.334299+0000","last_clean":"2023-09-23T15:09:29.334299+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:09:29.334299+0000","last_undegraded":"2023-09-23T15:09:29.334299+0000","last_fullsized":"2023-09-23T15:09:29.334299+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":91,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.323929+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:09:29.323929+0000","last_peered":"2023-09-23T15:09:29.323929+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:09:29.323929+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.316421+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:09:29.316421+0000","last_peered":"2023-09-23T15:09:29.316421+0000","last_clean":"2023-09-23T15:09:29.316421+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:09:29.316421+0000","last_undegraded":"2023-09-23T15:09:29.316421+0000","last_fullsized":"2023-09-23T15:09:29.316421+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"140'8436","reported_seq":8477,"reported_epoch":140,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:39.158407+0000","last_change":"2023-09-23T15:08:51.257194+0000","last_active":"2023-09-23T15:09:39.158407+0000","last_peered":"2023-09-23T15:09:39.158407+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:08:51.257194+0000","last_became_peered":"2023-09-23T15:08:51.257194+0000","last_unstale":"2023-09-23T15:09:39.158407+0000","last_undegraded":"2023-09-23T15:08:51.246208+0000","last_fullsized":"2023-09-23T15:08:51.246087+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8436,"log_dups_size":0,"ondisk_log_size":8436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.554621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5374,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8436,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":5374}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":70,"reported_epoch":140,"state":"active+clean","last_fresh":"2023-09-23T15:09:36.893753+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:09:36.893753+0000","last_peered":"2023-09-23T15:09:36.893753+0000","last_clean":"2023-09-23T15:09:36.893753+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:09:36.893753+0000","last_undegraded":"2023-09-23T15:09:36.893753+0000","last_fullsized":"2023-09-23T15:09:36.893753+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":128,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.317315+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:09:29.317315+0000","last_peered":"2023-09-23T15:09:29.317315+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:09:29.317315+0000","last_undegraded":"2023-09-23T15:09:29.317315+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":108,"reported_epoch":139,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:29.329776+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:09:29.329776+0000","last_peered":"2023-09-23T15:09:29.329776+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:09:29.329776+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":122,"reported_epoch":139,"state":"active+undersized","last_fresh":"2023-09-23T15:09:29.314659+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:09:29.314659+0000","last_peered":"2023-09-23T15:09:29.314659+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:09:29.314659+0000","last_undegraded":"2023-09-23T15:09:29.314659+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":84,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.330679+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:09:29.330679+0000","last_peered":"2023-09-23T15:09:29.330679+0000","last_clean":"2023-09-23T15:09:29.330679+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:09:29.330679+0000","last_undegraded":"2023-09-23T15:09:29.330679+0000","last_fullsized":"2023-09-23T15:09:29.330679+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":115,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.315083+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:09:29.315083+0000","last_peered":"2023-09-23T15:09:29.315083+0000","last_clean":"2023-09-23T15:09:29.315083+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:09:29.315083+0000","last_undegraded":"2023-09-23T15:09:29.315083+0000","last_fullsized":"2023-09-23T15:09:29.315083+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321077+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:09:29.321077+0000","last_peered":"2023-09-23T15:09:29.321077+0000","last_clean":"2023-09-23T15:09:29.321077+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:09:29.321077+0000","last_undegraded":"2023-09-23T15:09:29.321077+0000","last_fullsized":"2023-09-23T15:09:29.321077+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":123,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.321334+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:09:29.321334+0000","last_peered":"2023-09-23T15:09:29.321334+0000","last_clean":"2023-09-23T15:09:29.321334+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:09:29.321334+0000","last_undegraded":"2023-09-23T15:09:29.321334+0000","last_fullsized":"2023-09-23T15:09:29.321334+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":85,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.324366+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:09:29.324366+0000","last_peered":"2023-09-23T15:09:29.324366+0000","last_clean":"2023-09-23T15:09:29.324366+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:09:29.324366+0000","last_undegraded":"2023-09-23T15:09:29.324366+0000","last_fullsized":"2023-09-23T15:09:29.324366+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":108,"reported_epoch":139,"state":"active+clean","last_fresh":"2023-09-23T15:09:29.315121+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:09:29.315121+0000","last_peered":"2023-09-23T15:09:29.315121+0000","last_clean":"2023-09-23T15:09:29.315121+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:09:29.315121+0000","last_undegraded":"2023-09-23T15:09:29.315121+0000","last_fullsized":"2023-09-23T15:09:29.315121+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5738725400,"num_objects":87567,"num_object_clones":0,"num_object_copies":262701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27403,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":87567,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":135967,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17314033664,"data_stored":17314021448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132667,"ondisk_log_size":132667,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201459,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802016,"kb_used_data":1755864,"kb_used_omap":0,"kb_used_meta":46080,"kb_avail":91943136,"statfs":{"total":95995035648,"available":94149771264,"internally_reserved":0,"allocated":1798004736,"data_stored":1797461449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":47185920},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0920000000000001,"5min":0.94499999999999995,"15min":0.70699999999999996},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.2110000000000003,"5min":7.8799999999999999,"15min":7.8799999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.006,"15min":0.72599999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.0310000000000006,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":7.1870000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:08:50 2023","interfaces":[{"interface":"back","average":{"1min":11.904,"5min":2.8719999999999999,"15min":1.3680000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":170.63800000000001,"5min":170.63800000000001,"15min":170.63800000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":11.869,"5min":2.8300000000000001,"15min":1.3200000000000001},"min":{"1min":0.46000000000000002,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":170.47999999999999,"5min":170.47999999999999,"15min":170.47999999999999},"last":7.234}]},{"osd":3,"last update":"Sat Sep 23 15:09:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":1.181,"15min":0.77100000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8449999999999998,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":0.53500000000000003},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.1619999999999999,"15min":0.86399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.7960000000000003,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":0.65200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.82599999999999996,"15min":0.61599999999999999},"min":{"1min":0.35599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2919999999999998,"5min":5.2919999999999998,"15min":5.2919999999999998},"last":5.0439999999999996},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.014,"15min":0.69299999999999995},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.23999999999999999},"max":{"1min":6.766,"5min":6.766,"15min":6.766},"last":7.2960000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":0.92600000000000005,"15min":0.68400000000000005},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.26200000000000001},"max":{"1min":5.8970000000000002,"5min":5.8970000000000002,"15min":5.8970000000000002},"last":0.95999999999999996},{"interface":"front","average":{"1min":2.7869999999999999,"5min":1.1619999999999999,"15min":0.77400000000000002},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4779999999999998,"5min":8.4779999999999998,"15min":8.4779999999999998},"last":0.68400000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.82999999999999996,"15min":0.67200000000000004},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.1589999999999998,"5min":4.1589999999999998,"15min":4.1589999999999998},"last":0.628},{"interface":"front","average":{"1min":2.677,"5min":1.1259999999999999,"15min":0.77500000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.0190000000000001,"5min":7.0190000000000001,"15min":7.0190000000000001},"last":7.2690000000000001}]}]},{"osd":6,"up_from":47,"seq":201863463095,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446740,"kb_used_data":1400396,"kb_used_omap":0,"kb_used_meta":46336,"kb_avail":92298412,"statfs":{"total":95995035648,"available":94513573888,"internally_reserved":0,"allocated":1434005504,"data_stored":1433474354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":47448064},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":13.331,"5min":3.1190000000000002,"15min":1.403},"min":{"1min":0.26200000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":108.39400000000001,"5min":108.39400000000001,"15min":108.39400000000001},"last":5.4180000000000001},{"interface":"front","average":{"1min":13.425000000000001,"5min":3.1739999999999999,"15min":1.444},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":108.452,"5min":108.452,"15min":108.452},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":26.603999999999999,"5min":7.3860000000000001,"15min":2.8740000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":117.768,"5min":180.374,"15min":180.374},"last":0.54300000000000004},{"interface":"front","average":{"1min":27.042999999999999,"5min":7.5270000000000001,"15min":2.923},"min":{"1min":0.36499999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":119.129,"5min":181.685,"15min":181.685},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":0.998,"15min":0.73499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":17.084,"5min":17.084,"15min":17.084},"last":2.5310000000000001},{"interface":"front","average":{"1min":1.728,"5min":0.89300000000000002,"15min":0.70499999999999996},"min":{"1min":0.40600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4880000000000004,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.70399999999999996,"15min":0.56499999999999995},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":7.4539999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":5.3559999999999999},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.76300000000000001,"15min":0.59599999999999997},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.25600000000000001},"max":{"1min":3.8210000000000002,"5min":3.8210000000000002,"15min":3.8210000000000002},"last":0.88}]},{"osd":5,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":0.79900000000000004,"15min":0.61599999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9669999999999996,"5min":7.9669999999999996,"15min":7.9669999999999996},"last":5.3150000000000004},{"interface":"front","average":{"1min":1.0720000000000001,"5min":0.78600000000000003,"15min":0.627},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.9260000000000002,"5min":3.9260000000000002,"15min":3.9260000000000002},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":1.03,"15min":0.72899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":17,"5min":17,"15min":17},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.2290000000000001,"5min":0.89300000000000002,"15min":0.71199999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.4080000000000004,"5min":7.4080000000000004,"15min":7.4080000000000004},"last":5.3890000000000002}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893182,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2911236,"kb_used_data":2803260,"kb_used_omap":8,"kb_used_meta":107959,"kb_avail":90833916,"statfs":{"total":95995035648,"available":93013929984,"internally_reserved":0,"allocated":2870538240,"data_stored":2869905814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":110550612},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":4.9160000000000004,"5min":1.3640000000000001,"15min":0.72299999999999998},"min":{"1min":0.308,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":39.130000000000003,"5min":39.130000000000003,"15min":39.130000000000003},"last":0.26800000000000002},{"interface":"front","average":{"1min":3.9500000000000002,"5min":1.1319999999999999,"15min":0.65000000000000002},"min":{"1min":0.26700000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":39.034999999999997,"5min":39.034999999999997,"15min":39.034999999999997},"last":0.20599999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:09:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.0169999999999999,"15min":0.65100000000000002},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.1180000000000003,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":3.1720000000000002},{"interface":"front","average":{"1min":2.7519999999999998,"5min":1.4359999999999999,"15min":1.1499999999999999},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.5949999999999998,"5min":11.978,"15min":11.978},"last":3.2570000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":0.90000000000000002,"15min":0.73199999999999998},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":3.5800000000000001,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":1.377},{"interface":"front","average":{"1min":2.3700000000000001,"5min":1.01,"15min":0.76100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.0039999999999996,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.7149999999999999,"5min":1.1020000000000001,"15min":0.83199999999999996},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":13.49,"5min":13.49,"15min":13.49},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.198,"5min":0.94299999999999995,"15min":0.80200000000000005},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.181,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":1.1950000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":1.01,"15min":0.80200000000000005},"min":{"1min":0.39700000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.0540000000000003,"5min":9.0540000000000003,"15min":9.0540000000000003},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.194,"5min":0.97199999999999998,"15min":0.78000000000000003},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.4960000000000004,"5min":5.4960000000000004,"15min":5.4960000000000004},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.2679999999999998,"5min":0.99099999999999999,"15min":0.78100000000000003},"min":{"1min":0.48399999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.7919999999999998,"5min":5.7919999999999998,"15min":5.7919999999999998},"last":1.3999999999999999},{"interface":"front","average":{"1min":1.9299999999999999,"5min":1.0780000000000001,"15min":0.83499999999999996},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":8.1760000000000002,"5min":8.1760000000000002,"15min":8.1760000000000002},"last":1.3300000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566129,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2216436,"kb_used_data":2136428,"kb_used_omap":11,"kb_used_meta":79988,"kb_avail":91528716,"statfs":{"total":95995035648,"available":93725405184,"internally_reserved":0,"allocated":2187702272,"data_stored":2187083378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":81908496},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.504,"5min":0.61899999999999999,"15min":0.47299999999999998},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":6.6189999999999998,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.4059999999999999,"5min":0.59099999999999997,"15min":0.45300000000000001},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.7149999999999999,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":2.3370000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.3540000000000001,"15min":0.71099999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.7579999999999991,"5min":57.695,"15min":57.695},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.462,"15min":0.754},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7279999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":0.33900000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":0.84599999999999997,"15min":0.71599999999999997},"min":{"1min":0.38900000000000001,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.8250000000000002,"5min":4.8250000000000002,"15min":4.8250000000000002},"last":2.403},{"interface":"front","average":{"1min":1.611,"5min":0.83599999999999997,"15min":0.72099999999999997},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.0350000000000001,"5min":4.0350000000000001,"15min":4.0350000000000001},"last":5.2430000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6699999999999999,"5min":1.0289999999999999,"15min":0.76600000000000001},"min":{"1min":0.441,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.202,"5min":6.202,"15min":6.202},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.855,"5min":0.89300000000000002,"15min":0.753},"min":{"1min":0.48099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.1150000000000002,"5min":4.1150000000000002,"15min":4.1150000000000002},"last":5.6710000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":0.94499999999999995,"15min":0.80300000000000005},"min":{"1min":0.39300000000000002,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.9790000000000001,"5min":6.9790000000000001,"15min":6.9790000000000001},"last":5.0449999999999999},{"interface":"front","average":{"1min":2.552,"5min":1.0569999999999999,"15min":0.82499999999999996},"min":{"1min":0.47799999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":6.7720000000000002,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":6.2220000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":0.88700000000000001,"15min":0.76700000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.5439999999999996,"5min":4.5439999999999996,"15min":4.5439999999999996},"last":0.52100000000000002},{"interface":"front","average":{"1min":2.2349999999999999,"5min":0.98199999999999998,"15min":0.79200000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":4.9580000000000002,"5min":4.9580000000000002,"15min":4.9580000000000002},"last":4.8789999999999996}]}]},{"osd":3,"up_from":110,"seq":472446402603,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1108616,"kb_used_data":1050816,"kb_used_omap":3,"kb_used_meta":57788,"kb_avail":92636536,"statfs":{"total":95995035648,"available":94859812864,"internally_reserved":0,"allocated":1076035584,"data_stored":1075455642,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":59175081},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":7.1520000000000001,"5min":1.8400000000000001,"15min":0.86399999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":79.611000000000004,"5min":79.611000000000004,"15min":79.611000000000004},"last":1.304},{"interface":"front","average":{"1min":7.0190000000000001,"5min":1.8899999999999999,"15min":0.88800000000000001},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":79.656000000000006,"5min":79.656000000000006,"15min":79.656000000000006},"last":1.3200000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":15.308999999999999,"5min":3.6150000000000002,"15min":1.619},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":105.795,"5min":105.795,"15min":105.795},"last":1.9419999999999999},{"interface":"front","average":{"1min":16.312000000000001,"5min":3.8159999999999998,"15min":1.554},"min":{"1min":0.35699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":106.39,"5min":106.39,"15min":106.39},"last":1.3620000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.79900000000000004,"15min":0.66500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":1.986},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.1060000000000001,"15min":0.86199999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":8.3989999999999991}]},{"osd":5,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":0.92500000000000004,"15min":0.73399999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.3780000000000001,"5min":8.3780000000000001,"15min":8.3780000000000001},"last":1.502},{"interface":"front","average":{"1min":2.129,"5min":1.1579999999999999,"15min":0.92200000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":8.3309999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":0.94999999999999996,"15min":0.76600000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6020000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":1.3899999999999999},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.276,"15min":0.98199999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.2699999999999996,"5min":9.2699999999999996,"15min":9.2699999999999996},"last":8.4399999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.236,"15min":0.93999999999999995},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":8.4770000000000003},{"interface":"front","average":{"1min":1.276,"5min":0.85099999999999998,"15min":0.71799999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.738,"5min":4.1660000000000004,"15min":4.1660000000000004},"last":0.39900000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986368,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2888148,"kb_used_data":2806668,"kb_used_omap":0,"kb_used_meta":81472,"kb_avail":90857004,"statfs":{"total":95995035648,"available":93037572096,"internally_reserved":0,"allocated":2874028032,"data_stored":2873496395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":83427328},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.911,"5min":2.2469999999999999,"15min":1.107},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":7.7489999999999997,"5min":122.69199999999999,"15min":122.69199999999999},"last":2.5710000000000002},{"interface":"front","average":{"1min":2.992,"5min":2.48,"15min":1.1839999999999999},"min":{"1min":0.35099999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":10.265000000000001,"5min":122.745,"15min":122.745},"last":1.7689999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":5.101,"5min":1.4810000000000001,"15min":0.88800000000000001},"min":{"1min":0.434,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":36.884,"5min":36.884,"15min":36.884},"last":1.724},{"interface":"front","average":{"1min":5.8090000000000002,"5min":1.738,"15min":1.071},"min":{"1min":0.58099999999999996,"5min":0.433,"15min":0.433},"max":{"1min":44.177999999999997,"5min":44.177999999999997,"15min":44.177999999999997},"last":1.792}]},{"osd":3,"last update":"Sat Sep 23 15:09:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.1240000000000001,"15min":0.748},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":6.8920000000000003,"5min":6.8920000000000003,"15min":6.8920000000000003},"last":2.5339999999999998},{"interface":"front","average":{"1min":1.103,"5min":0.82299999999999995,"15min":0.68100000000000005},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.4889999999999999,"5min":4.4889999999999999,"15min":4.4889999999999999},"last":0.377}]},{"osd":5,"last update":"Sat Sep 23 15:09:32 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":1.6160000000000001,"15min":0.88},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":28.891999999999999,"5min":28.891999999999999,"15min":28.891999999999999},"last":6.3120000000000003},{"interface":"front","average":{"1min":3.847,"5min":1.427,"15min":0.82999999999999996},"min":{"1min":0.35999999999999999,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":32.100999999999999,"5min":32.100999999999999,"15min":32.100999999999999},"last":5.0209999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":0.93600000000000005,"15min":0.70499999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":2.6709999999999998},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.92000000000000004,"15min":0.68899999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33400000000000002},"max":{"1min":5.6470000000000002,"5min":5.6470000000000002,"15min":5.6470000000000002},"last":4.9829999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:09:10 2023","interfaces":[{"interface":"back","average":{"1min":3.524,"5min":1.292,"15min":0.82599999999999996},"min":{"1min":0.40300000000000002,"5min":0.376,"15min":0.28799999999999998},"max":{"1min":11.711,"5min":11.711,"15min":11.711},"last":7.3570000000000002},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.296,"15min":0.87},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":14.653,"5min":14.653,"15min":14.653},"last":2.6019999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724731,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3243340,"kb_used_data":3161668,"kb_used_omap":0,"kb_used_meta":81600,"kb_avail":90501812,"statfs":{"total":95995035648,"available":92673855488,"internally_reserved":0,"allocated":3237548032,"data_stored":3237020374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":83558400},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:46 2023","interfaces":[{"interface":"back","average":{"1min":2.8460000000000001,"5min":1.0249999999999999,"15min":0.70599999999999996},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":24.962,"5min":24.962,"15min":24.962},"last":0.23100000000000001},{"interface":"front","average":{"1min":2.589,"5min":0.97099999999999997,"15min":0.68100000000000005},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":25.317,"5min":25.317,"15min":25.317},"last":1.1200000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:08:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7749999999999999,"5min":1.0289999999999999,"15min":0.71699999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":19.236999999999998,"5min":19.236999999999998,"15min":19.236999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":3.036,"5min":1.1040000000000001,"15min":0.76000000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":19.699999999999999,"5min":19.699999999999999,"15min":19.699999999999999},"last":1.6419999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:09:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.1180000000000001,"15min":0.77700000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.265000000000001,"5min":15.265000000000001,"15min":15.265000000000001},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.599,"5min":1.05,"15min":0.76700000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.8390000000000004,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.88300000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":0.877,"15min":0.59999999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":6.8799999999999999,"5min":6.8799999999999999,"15min":6.8799999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.9550000000000001,"5min":0.94999999999999996,"15min":0.65900000000000003},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.6,"5min":13.6,"15min":13.6},"last":1.056}]},{"osd":6,"last update":"Sat Sep 23 15:08:54 2023","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":0.75,"15min":0.59299999999999997},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.4610000000000003,"5min":4.4610000000000003,"15min":4.4610000000000003},"last":10.082000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.61899999999999999},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.6659999999999999,"5min":2.1230000000000002,"15min":2.1230000000000002},"last":6}]},{"osd":7,"last update":"Sat Sep 23 15:09:26 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":1.2370000000000001,"15min":0.80300000000000005},"min":{"1min":0.41099999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":15.301,"5min":15.301,"15min":15.301},"last":0.74199999999999999},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.129,"15min":0.76300000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.361000000000001,"5min":13.361000000000001,"15min":13.361000000000001},"last":2.516}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2863857664,"data_stored":2863857664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2171277312,"data_stored":2171273240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1069481984,"data_stored":1069481984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2859204608,"data_stored":2859204608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3210481664,"data_stored":3210477592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1427898368,"data_stored":1427898368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1784680448,"data_stored":1784676376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:09:45.124 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:09:45.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T15:09:45.261 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:45.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:45.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:44 smithi027 ceph-mon[195233]: from='client.44852 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:09:45.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:44 smithi027 ceph-mon[195233]: pgmap v330: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 26909/255183 objects degraded (10.545%) 2023-09-23T15:09:45.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:45.418 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:45.460 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:45.523 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:45.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:45.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:45.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:45.697 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:45.966 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:45.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:45.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:46.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:46.074 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:46.132 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:46.231 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:09:46.242 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:46.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:46.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:46.257 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/886926442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.257 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:46 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/886926442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:46 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.331 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:09:46.331 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:09:46.331 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:09:46.331 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:09:46.331 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:09:46.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:46.425 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/886926442' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.425 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:46 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:09:46.426 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:46.564 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:46.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:46.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:09:46.722 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:47.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: pgmap v331: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26142/250677 objects degraded (10.429%) 2023-09-23T15:09:47.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:47.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26142/250677 objects degraded (10.429%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: pgmap v331: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26142/250677 objects degraded (10.429%) 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26142/250677 objects degraded (10.429%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: pgmap v331: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 26142/250677 objects degraded (10.429%) 2023-09-23T15:09:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:09:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:09:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T15:09:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: failed to encode map e141 with expected crc 2023-09-23T15:09:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26142/250677 objects degraded (10.429%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:48.063 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:09:48.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:48.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:09:48.152 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:48] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T15:09:48.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:48.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:09:48.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:48.350 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:48.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:48.358 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:48.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:48.521 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:48.830 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:48.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:48.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:48.837 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:48.880 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:48.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:48.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:48.891 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:48.928 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.967 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.968 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:09:48.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:49.044 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:49.115 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:49.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:49.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:49.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:49.315 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:49 smithi027 ceph-mon[195233]: pgmap v333: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 25880/246930 objects degraded (10.481%) 2023-09-23T15:09:49.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3871318381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:49.321 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:49.440 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:49.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:49.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:49.522 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:49 smithi017 ceph-mon[215646]: pgmap v333: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 25880/246930 objects degraded (10.481%) 2023-09-23T15:09:49.522 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3871318381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:49.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:49 smithi017 ceph-mon[222878]: pgmap v333: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 809 op/s; 25880/246930 objects degraded (10.481%) 2023-09-23T15:09:49.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3871318381' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:49.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:49.616 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:49.697 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:49.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:49.719 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:49.820 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:49.884 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:49.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:09:50.047 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:50.056 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:50.056 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:50.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:50.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:50.105 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:50.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:50.243 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:50.343 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:50.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:50.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:50.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:50.523 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:50.635 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:50.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:50.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:50.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:50.786 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:50.925 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:50.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:50.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:51.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:51.082 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:51.159 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:51.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:51.247 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 11 2023-09-23T15:09:51.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:51.324 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:51.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:51 smithi027 ceph-mon[195233]: pgmap v334: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:51.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3971754758' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:51.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[215646]: pgmap v334: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3971754758' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[222878]: pgmap v334: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3971754758' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T15:09:52.039 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:52.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:52.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:52.057 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:09:52.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:52.155 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:52.223 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:52.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:52.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:52.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.358 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.359 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24852/238623 objects degraded (10.415%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:52.359 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24852/238623 objects degraded (10.415%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:52.360 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:52 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T15:09:52.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: osdmap e142: 8 total, 7 up, 8 in 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: failed to encode map e142 with expected crc 2023-09-23T15:09:52.512 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24852/238623 objects degraded (10.415%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:09:52.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:52 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T15:09:52.513 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:52.656 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:52.699 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:09:52.699 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:09:52.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:52.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:52.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:52.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:52.869 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:53.867 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:53.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:53.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:53.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[215646]: pgmap v336: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[215646]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[215646]: failed to encode map e143 with expected crc 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[215646]: failed to encode map e143 with expected crc 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[222878]: pgmap v336: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[222878]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[222878]: failed to encode map e143 with expected crc 2023-09-23T15:09:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:53 smithi017 ceph-mon[222878]: failed to encode map e143 with expected crc 2023-09-23T15:09:53.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:54.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:53 smithi027 ceph-mon[195233]: pgmap v336: 89 pgs: 11 active+undersized+degraded, 26 active+undersized, 52 active+clean; 4.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 24852/238623 objects degraded (10.415%) 2023-09-23T15:09:54.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T15:09:54.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:53 smithi027 ceph-mon[195233]: osdmap e143: 8 total, 7 up, 8 in 2023-09-23T15:09:54.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:53 smithi027 ceph-mon[195233]: failed to encode map e143 with expected crc 2023-09-23T15:09:54.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:53 smithi027 ceph-mon[195233]: failed to encode map e143 with expected crc 2023-09-23T15:09:54.056 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:54.156 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:09:54.160 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:54.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:54.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:54.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:54.338 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:54.431 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:54.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:54.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:54.578 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:54.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[215646]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[222878]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:54 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:54.972 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:54.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:54.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:55.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:55.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:54 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:55.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:54 smithi027 ceph-mon[195233]: osdmap e144: 8 total, 7 up, 8 in 2023-09-23T15:09:55.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:54 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:55.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:54 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:55.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:54 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:55.143 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:55.269 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:55.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:55.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:55.387 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:55.446 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:55.626 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:55.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:55.632 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:55.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:55.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:55.930 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:55.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:55.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:56.028 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:09:56.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:56.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:09:56.089 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:09:56.090 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:55 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.090 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:55 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.090 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:55 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.090 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:55 smithi027 ceph-mon[195233]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.090 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:55 smithi027 ceph-mon[195233]: pgmap v339: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 24373/231402 objects degraded (10.533%) 2023-09-23T15:09:56.095 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:56.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[222878]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.098 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[222878]: pgmap v339: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 24373/231402 objects degraded (10.533%) 2023-09-23T15:09:56.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[215646]: failed to encode map e144 with expected crc 2023-09-23T15:09:56.099 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:55 smithi017 ceph-mon[215646]: pgmap v339: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 24373/231402 objects degraded (10.533%) 2023-09-23T15:09:56.119 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:09:56.152 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:09:56.184 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:56.188 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:09:56.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:56.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:56.229 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:09:56.229 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.229 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:09:56.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:56.403 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:56.489 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:56.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:56.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:56.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:56.695 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:56.819 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:56.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:56.857 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:09:56.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:56.986 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3787853980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:56 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24373/231402 objects degraded (10.533%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:09:57.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:57.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3787853980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:57.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24373/231402 objects degraded (10.533%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:09:57.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:09:57.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3787853980' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:09:57.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:56 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24373/231402 objects degraded (10.533%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:09:57.202 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:09:57.202 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:09:57.202 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-09-23T15:09:57.203 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:09:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:57.381 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:09:57.382 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:09:57.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:57.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:57.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:09:57.551 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:57.651 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:09:57.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:57.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:09:57.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:09:57.817 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:58.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[215646]: pgmap v340: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 23639/226854 objects degraded (10.420%) 2023-09-23T15:09:58.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:58.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[215646]: failed to encode map e145 with expected crc 2023-09-23T15:09:58.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:58.087 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[215646]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:09:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:09:58] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[222878]: pgmap v340: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 23639/226854 objects degraded (10.420%) 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[222878]: failed to encode map e145 with expected crc 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:58.088 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:57 smithi017 ceph-mon[222878]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T15:09:58.089 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:09:58.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:58.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:09:58.192 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:58.271 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:57 smithi027 ceph-mon[195233]: pgmap v340: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 23639/226854 objects degraded (10.420%) 2023-09-23T15:09:58.271 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:57 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:09:58.271 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:57 smithi027 ceph-mon[195233]: failed to encode map e145 with expected crc 2023-09-23T15:09:58.272 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/439095783' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:09:58.272 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:57 smithi027 ceph-mon[195233]: osdmap e145: 8 total, 7 up, 8 in 2023-09-23T15:09:58.277 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:58.314 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:09:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:09:58.370 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151828 5.d 5:b0ab6188:::benchmark_data_smithi017_288048_object40059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.357000+0000", 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012524970999999999, 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012556573999999999, 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151828 2023-09-23T15:09:58.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357000+0000", 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357000+0000", 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357000+0000", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357003+0000", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3280000000000002e-06 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357010+0000", 2023-09-23T15:09:58.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1400000000000002e-06 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357016+0000", 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.614e-06 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357066+0000", 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9929999999999998e-05 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357204+0000", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000138622 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357331+0000", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012646200000000001 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.362376+0000", 2023-09-23T15:09:58.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050447570000000004 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151829 5.b 5:dc77d3f3:::benchmark_data_smithi017_288048_object40060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.357031+0000", 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012493821, 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012585768000000001, 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.376 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151829 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357031+0000", 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357031+0000", 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357031+0000", 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357034+0000", 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.926e-06 2023-09-23T15:09:58.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357041+0000", 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0600000000000002e-06 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.357047+0000", 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6899999999999997e-06 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.362115+0000", 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050679089999999998 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.362262+0000", 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000147279 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.362380+0000", 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118158 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.366929+0000", 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045485600000000001 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368292+0000", 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013635609999999999 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151832 5.8 5:11f9534e:::benchmark_data_smithi017_288048_object40063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.363512+0000", 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0060128600000000001, 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061562020000000004, 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151832 2023-09-23T15:09:58.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363512+0000", 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363512+0000", 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363511+0000", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363518+0000", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.075e-06 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363535+0000", 2023-09-23T15:09:58.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7346999999999998e-05 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363544+0000", 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4419999999999998e-06 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363594+0000", 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0317e-05 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363718+0000", 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012424800000000001 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0", 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.363848+0000", 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012964399999999999 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:151828 5.d e145/114)", 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.368733+0000", 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00079211400000000001, 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00097665600000000001, 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368733+0000", 2023-09-23T15:09:58.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368733+0000", 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368733+0000", 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967296 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368735+0000", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1610000000000001e-06 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368740+0000", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.366e-06 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368744+0000", 2023-09-23T15:09:58.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9910000000000004e-06 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:151816 5.d e145/114)", 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.368763+0000", 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00076250000000000005, 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000972517, 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:09:58.390 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368763+0000", 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368763+0000", 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368762+0000", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368765+0000", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8279999999999999e-06 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368772+0000", 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6089999999999997e-06 2023-09-23T15:09:58.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368776+0000", 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5129999999999998e-06 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:151828 5.d e145/114)", 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.368820+0000", 2023-09-23T15:09:58.393 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00070564399999999995, 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00094746700000000001, 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368820+0000", 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368820+0000", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368819+0000", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368822+0000", 2023-09-23T15:09:58.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.976e-06 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368829+0000", 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6810000000000001e-06 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368834+0000", 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7700000000000001e-06 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.396 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151835 5.8 5:1eeaab24:::benchmark_data_smithi017_288048_object40066:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.368854+0000", 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00067142199999999997, 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001032318, 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151835 2023-09-23T15:09:58.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368854+0000", 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368854+0000", 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368853+0000", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368857+0000", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5020000000000003e-06 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368874+0000", 2023-09-23T15:09:58.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6427999999999999e-05 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368880+0000", 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3740000000000003e-06 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.368998+0000", 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011742 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369117+0000", 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000119146 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0", 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369221+0000", 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010476000000000001 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151839 5.6 5:6755fd28:::benchmark_data_smithi017_288048_object40070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.369113+0000", 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00041197400000000001, 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085204499999999995, 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151839 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369113+0000", 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369113+0000", 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369112+0000", 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369118+0000", 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0229999999999998e-06 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369129+0000", 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1099e-05 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369137+0000", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5780000000000001e-06 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369190+0000", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3041999999999998e-05 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369330+0000", 2023-09-23T15:09:58.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013987299999999999 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.369464+0000", 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013422900000000001 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:151816 5.d 5:b8f5b1bf:::benchmark_data_smithi017_288048_object40047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:58.336735+0000", 2023-09-23T15:09:58.406 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.032789829, 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033307550999999998, 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 151816 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336735+0000", 2023-09-23T15:09:58.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336735+0000", 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336734+0000", 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336741+0000", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4599999999999997e-06 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336756+0000", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4759999999999999e-05 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336764+0000", 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8450000000000003e-06 2023-09-23T15:09:58.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336814+0000", 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0010999999999999e-05 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.336963+0000", 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014940400000000001 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.337100+0000", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000136429 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.341454+0000", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043546180000000002 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:58.350498+0000", 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090433430000000006 2023-09-23T15:09:58.411 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 9 2023-09-23T15:09:58.412 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:09:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:58.413 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:58.413 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:58.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:58.561 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:58.634 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:09:58.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:58.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:09:58.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:09:58.767 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:58.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:09:58.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:58.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:09:58.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:09:58.983 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:59.099 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:09:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:59.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:59.109 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:09:59.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:09:59.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:09:59.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:59.342 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:09:59.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:09:59.388 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:09:59.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:09:59.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:09:59.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:09:59.658 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:09:59.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:09:59 smithi017 ceph-mon[215646]: pgmap v342: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 23390/222912 objects degraded (10.493%) 2023-09-23T15:09:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:09:59 smithi017 ceph-mon[222878]: pgmap v342: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 23390/222912 objects degraded (10.493%) 2023-09-23T15:10:00.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:10:00.135 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:00.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:00.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:00.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:00.345 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:09:59 smithi027 ceph-mon[195233]: pgmap v342: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 871 op/s; 23390/222912 objects degraded (10.493%) 2023-09-23T15:10:00.382 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:00.438 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:00.468 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:00.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:00.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:00.574 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:00.647 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:00.900 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:00.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:00.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:00.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T15:10:00.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: osd.1 (root=default,host=smithi017) is down 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: [WRN] PG_DEGRADED: Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 2.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,0] 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 2.2 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,6] 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 2.4 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,7] 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 2.6 is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.0 is stuck undersized for 66s, current state active+undersized, last acting [2,6] 2023-09-23T15:10:00.916 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.4 is stuck undersized for 66s, current state active+undersized, last acting [2,5] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.6 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.8 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.a is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.e is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,4] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.16 is stuck undersized for 66s, current state active+undersized, last acting [5,7] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.18 is stuck undersized for 66s, current state active+undersized, last acting [3,0] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.19 is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.1a is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 3.1c is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.4 is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:00.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.6 is stuck undersized for 66s, current state active+undersized, last acting [0,2] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.7 is stuck undersized for 66s, current state active+undersized, last acting [5,0] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.9 is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.a is stuck undersized for 66s, current state active+undersized, last acting [6,7] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.b is stuck undersized for 66s, current state active+undersized, last acting [0,4] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.d is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.e is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.f is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.10 is stuck undersized for 66s, current state active+undersized, last acting [3,6] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.12 is stuck undersized for 66s, current state active+undersized, last acting [6,2] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.13 is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:00.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.14 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.17 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.1b is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.1c is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 4.1f is stuck undersized for 66s, current state active+undersized, last acting [6,5] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 5.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [3,4] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 5.5 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 5.7 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 5.8 is stuck undersized for 66s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T15:10:00.919 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[215646]: pg 5.c is stuck undersized for 66s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T15:10:00.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:00.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:00.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T15:10:00.920 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: osd.1 (root=default,host=smithi017) is down 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: [WRN] PG_DEGRADED: Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 2.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,0] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 2.2 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,6] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 2.4 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,7] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 2.6 is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.0 is stuck undersized for 66s, current state active+undersized, last acting [2,6] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.4 is stuck undersized for 66s, current state active+undersized, last acting [2,5] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.6 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.8 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.a is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.e is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,4] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.16 is stuck undersized for 66s, current state active+undersized, last acting [5,7] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.18 is stuck undersized for 66s, current state active+undersized, last acting [3,0] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.19 is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.1a is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 3.1c is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.4 is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.6 is stuck undersized for 66s, current state active+undersized, last acting [0,2] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.7 is stuck undersized for 66s, current state active+undersized, last acting [5,0] 2023-09-23T15:10:00.922 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.9 is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.a is stuck undersized for 66s, current state active+undersized, last acting [6,7] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.b is stuck undersized for 66s, current state active+undersized, last acting [0,4] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.d is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.e is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.f is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.10 is stuck undersized for 66s, current state active+undersized, last acting [3,6] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.12 is stuck undersized for 66s, current state active+undersized, last acting [6,2] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.13 is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.14 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.923 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.17 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.1b is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.1c is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 4.1f is stuck undersized for 66s, current state active+undersized, last acting [6,5] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 5.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [3,4] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 5.5 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 5.7 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 5.8 is stuck undersized for 66s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T15:10:00.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:00 smithi017 ceph-mon[222878]: pg 5.c is stuck undersized for 66s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T15:10:01.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:01.071 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:01.179 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:01.184 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:01.184 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: osd.1 (root=default,host=smithi017) is down 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: [WRN] PG_DEGRADED: Degraded data redundancy: 23390/222912 objects degraded (10.493%), 11 pgs degraded, 37 pgs undersized 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 2.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,0] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 2.2 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,6] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 2.4 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,7] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 2.6 is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.0 is stuck undersized for 66s, current state active+undersized, last acting [2,6] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.4 is stuck undersized for 66s, current state active+undersized, last acting [2,5] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.6 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:01.185 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.8 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.a is stuck undersized for 66s, current state active+undersized, last acting [6,4] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.e is stuck undersized for 66s, current state active+undersized+degraded, last acting [7,4] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.16 is stuck undersized for 66s, current state active+undersized, last acting [5,7] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.18 is stuck undersized for 66s, current state active+undersized, last acting [3,0] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.19 is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.1a is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 3.1c is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,4] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.4 is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.6 is stuck undersized for 66s, current state active+undersized, last acting [0,2] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.7 is stuck undersized for 66s, current state active+undersized, last acting [5,0] 2023-09-23T15:10:01.186 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.9 is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.a is stuck undersized for 66s, current state active+undersized, last acting [6,7] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.b is stuck undersized for 66s, current state active+undersized, last acting [0,4] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.d is stuck undersized for 66s, current state active+undersized, last acting [4,2] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.e is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.f is stuck undersized for 66s, current state active+undersized, last acting [3,4] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.10 is stuck undersized for 66s, current state active+undersized, last acting [3,6] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.12 is stuck undersized for 66s, current state active+undersized, last acting [6,2] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.13 is stuck undersized for 66s, current state active+undersized, last acting [4,6] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.14 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.17 is stuck undersized for 66s, current state active+undersized, last acting [3,7] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.1b is stuck undersized for 66s, current state active+undersized, last acting [4,3] 2023-09-23T15:10:01.187 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.1c is stuck undersized for 66s, current state active+undersized, last acting [2,3] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 4.1f is stuck undersized for 66s, current state active+undersized, last acting [6,5] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 5.0 is stuck undersized for 66s, current state active+undersized+degraded, last acting [3,4] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 5.5 is stuck undersized for 66s, current state active+undersized+degraded, last acting [0,4] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 5.7 is stuck undersized for 66s, current state active+undersized+degraded, last acting [5,0] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 5.8 is stuck undersized for 66s, current state active+undersized+degraded, last acting [2,0] 2023-09-23T15:10:01.188 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:00 smithi027 ceph-mon[195233]: pg 5.c is stuck undersized for 66s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T15:10:01.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:01.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:01.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:01.352 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:10:01.354 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:10:01.357 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:01.940 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:02.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[215646]: pgmap v343: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:02.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[215646]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:02.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22329/214254 objects degraded (10.422%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:02.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[222878]: pgmap v343: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:02.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[222878]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:02.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:01 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22329/214254 objects degraded (10.422%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:02.200 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:02.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:02.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:02.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:02.344 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:01 smithi027 ceph-mon[195233]: pgmap v343: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:01 smithi027 ceph-mon[195233]: from='client.34953 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:02.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:01 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22329/214254 objects degraded (10.422%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:03.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:02 smithi017 ceph-mon[215646]: pgmap v344: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:03.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:02 smithi017 ceph-mon[222878]: pgmap v344: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:03.229 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":343,"stamp":"2023-09-23T15:10:00.194454+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4683339723,"num_objects":71418,"num_object_clones":0,"num_object_copies":214254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22329,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71418,"num_whiteouts":0,"num_read":6496,"num_read_kb":14137,"num_write":153290,"num_write_kb":7168166,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":134750,"ondisk_log_size":134750,"up":260,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14773088,"kb_used_data":14194848,"kb_used_omap":30,"kb_used_meta":577889,"kb_avail":735188128,"statfs":{"total":767960285184,"available":752832643072,"internally_reserved":0,"allocated":14535524352,"data_stored":14530923970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":591758936},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,4,5,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-532348928,"num_objects":-8123,"num_object_clones":0,"num_object_copies":-24369,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2523,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8123,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8123,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.401843"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":129,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.819514+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:09:53.819514+0000","last_peered":"2023-09-23T15:09:53.819514+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:09:53.819514+0000","last_undegraded":"2023-09-23T15:09:53.819514+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.827177+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:09:53.827177+0000","last_peered":"2023-09-23T15:09:53.827177+0000","last_clean":"2023-09-23T15:09:53.827177+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:09:53.827177+0000","last_undegraded":"2023-09-23T15:09:53.827177+0000","last_fullsized":"2023-09-23T15:09:53.827177+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":59,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.824579+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:09:53.824579+0000","last_peered":"2023-09-23T15:09:53.824579+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:09:53.824579+0000","last_undegraded":"2023-09-23T15:09:53.824579+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.812509+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:09:53.812509+0000","last_peered":"2023-09-23T15:09:53.812509+0000","last_clean":"2023-09-23T15:09:53.812509+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:09:53.812509+0000","last_undegraded":"2023-09-23T15:09:53.812509+0000","last_fullsized":"2023-09-23T15:09:53.812509+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.823970+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:09:53.823970+0000","last_peered":"2023-09-23T15:09:53.823970+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:09:53.823970+0000","last_undegraded":"2023-09-23T15:09:53.823970+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.840203+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:09:53.840203+0000","last_peered":"2023-09-23T15:09:53.840203+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:09:53.840203+0000","last_undegraded":"2023-09-23T15:09:53.840203+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.812053+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:09:53.812053+0000","last_peered":"2023-09-23T15:09:53.812053+0000","last_clean":"2023-09-23T15:09:53.812053+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:09:53.812053+0000","last_undegraded":"2023-09-23T15:09:53.812053+0000","last_fullsized":"2023-09-23T15:09:53.812053+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.823838+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:09:53.823838+0000","last_peered":"2023-09-23T15:09:53.823838+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:09:53.823838+0000","last_undegraded":"2023-09-23T15:09:53.823838+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":111,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.817835+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:09:53.817835+0000","last_peered":"2023-09-23T15:09:53.817835+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:09:53.817835+0000","last_undegraded":"2023-09-23T15:09:53.817835+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":56,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824490+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:09:53.824490+0000","last_peered":"2023-09-23T15:09:53.824490+0000","last_clean":"2023-09-23T15:09:53.824490+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:09:53.824490+0000","last_undegraded":"2023-09-23T15:09:53.824490+0000","last_fullsized":"2023-09-23T15:09:53.824490+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":106,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824419+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:09:53.824419+0000","last_peered":"2023-09-23T15:09:53.824419+0000","last_clean":"2023-09-23T15:09:53.824419+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:09:53.824419+0000","last_undegraded":"2023-09-23T15:09:53.824419+0000","last_fullsized":"2023-09-23T15:09:53.824419+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.841769+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:09:53.841769+0000","last_peered":"2023-09-23T15:09:53.841769+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:09:53.841769+0000","last_undegraded":"2023-09-23T15:09:53.841769+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":109,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.818789+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:09:53.818789+0000","last_peered":"2023-09-23T15:09:53.818789+0000","last_clean":"2023-09-23T15:09:53.818789+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:09:53.818789+0000","last_undegraded":"2023-09-23T15:09:53.818789+0000","last_fullsized":"2023-09-23T15:09:53.818789+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.818572+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:09:53.818572+0000","last_peered":"2023-09-23T15:09:53.818572+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:09:53.818572+0000","last_undegraded":"2023-09-23T15:09:53.818572+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.828747+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:09:53.828747+0000","last_peered":"2023-09-23T15:09:53.828747+0000","last_clean":"2023-09-23T15:09:53.828747+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:09:53.828747+0000","last_undegraded":"2023-09-23T15:09:53.828747+0000","last_fullsized":"2023-09-23T15:09:53.828747+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.824074+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:09:53.824074+0000","last_peered":"2023-09-23T15:09:53.824074+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:09:53.824074+0000","last_undegraded":"2023-09-23T15:09:53.824074+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.811547+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:09:53.811547+0000","last_peered":"2023-09-23T15:09:53.811547+0000","last_clean":"2023-09-23T15:09:53.811547+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:09:53.811547+0000","last_undegraded":"2023-09-23T15:09:53.811547+0000","last_fullsized":"2023-09-23T15:09:53.811547+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":103,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817167+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:09:53.817167+0000","last_peered":"2023-09-23T15:09:53.817167+0000","last_clean":"2023-09-23T15:09:53.817167+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:09:53.817167+0000","last_undegraded":"2023-09-23T15:09:53.817167+0000","last_fullsized":"2023-09-23T15:09:53.817167+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":86,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824122+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:09:53.824122+0000","last_peered":"2023-09-23T15:09:53.824122+0000","last_clean":"2023-09-23T15:09:53.824122+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:09:53.824122+0000","last_undegraded":"2023-09-23T15:09:53.824122+0000","last_fullsized":"2023-09-23T15:09:53.824122+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.812442+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:09:53.812442+0000","last_peered":"2023-09-23T15:09:53.812442+0000","last_clean":"2023-09-23T15:09:53.812442+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:09:53.812442+0000","last_undegraded":"2023-09-23T15:09:53.812442+0000","last_fullsized":"2023-09-23T15:09:53.812442+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":111,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.828593+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:09:53.828593+0000","last_peered":"2023-09-23T15:09:53.828593+0000","last_clean":"2023-09-23T15:09:53.828593+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:09:53.828593+0000","last_undegraded":"2023-09-23T15:09:53.828593+0000","last_fullsized":"2023-09-23T15:09:53.828593+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.826800+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:09:53.826800+0000","last_peered":"2023-09-23T15:09:53.826800+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:09:53.826800+0000","last_undegraded":"2023-09-23T15:09:53.826800+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"145'9607","reported_seq":9645,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.102597+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:09:59.102597+0000","last_peered":"2023-09-23T15:09:59.102597+0000","last_clean":"2023-09-23T15:09:59.102597+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:09:59.102597+0000","last_undegraded":"2023-09-23T15:09:59.102597+0000","last_fullsized":"2023-09-23T15:09:59.102597+0000","mapping_epoch":114,"log_start":"122'2906","ondisk_log_start":"122'2906","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6701,"log_dups_size":0,"ondisk_log_size":6701,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9607,"num_write_kb":450496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.826809+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:09:53.826809+0000","last_peered":"2023-09-23T15:09:53.826809+0000","last_clean":"2023-09-23T15:09:53.826809+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:09:53.826809+0000","last_undegraded":"2023-09-23T15:09:53.826809+0000","last_fullsized":"2023-09-23T15:09:53.826809+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":86,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817080+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:09:53.817080+0000","last_peered":"2023-09-23T15:09:53.817080+0000","last_clean":"2023-09-23T15:09:53.817080+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:09:53.817080+0000","last_undegraded":"2023-09-23T15:09:53.817080+0000","last_fullsized":"2023-09-23T15:09:53.817080+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"145'9420","reported_seq":9466,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:58.081959+0000","last_change":"2023-09-23T15:08:51.249550+0000","last_active":"2023-09-23T15:09:58.081959+0000","last_peered":"2023-09-23T15:09:58.081959+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:08:51.249550+0000","last_became_peered":"2023-09-23T15:08:51.249550+0000","last_unstale":"2023-09-23T15:09:58.081959+0000","last_undegraded":"2023-09-23T15:08:51.245609+0000","last_fullsized":"2023-09-23T15:08:51.245485+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9420,"log_dups_size":0,"ondisk_log_size":9420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.928709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297009152,"num_objects":4532,"num_object_clones":0,"num_object_copies":13596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4532,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9420,"num_write_kb":446464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":4532}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":112,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.825205+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:09:53.825205+0000","last_peered":"2023-09-23T15:09:53.825205+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:09:53.825205+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":127,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.819484+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:09:53.819484+0000","last_peered":"2023-09-23T15:09:53.819484+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:09:53.819484+0000","last_undegraded":"2023-09-23T15:09:53.819484+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"145'9501","reported_seq":9542,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:58.038142+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:09:58.038142+0000","last_peered":"2023-09-23T15:09:58.038142+0000","last_clean":"2023-09-23T15:09:58.038142+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:09:58.038142+0000","last_undegraded":"2023-09-23T15:09:58.038142+0000","last_fullsized":"2023-09-23T15:09:58.038142+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9501,"log_dups_size":0,"ondisk_log_size":9501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296157184,"num_objects":4519,"num_object_clones":0,"num_object_copies":13557,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4519,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9501,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824249+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:09:53.824249+0000","last_peered":"2023-09-23T15:09:53.824249+0000","last_clean":"2023-09-23T15:09:53.824249+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:09:53.824249+0000","last_undegraded":"2023-09-23T15:09:53.824249+0000","last_fullsized":"2023-09-23T15:09:53.824249+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":109,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.841522+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:09:53.841522+0000","last_peered":"2023-09-23T15:09:53.841522+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:09:53.841522+0000","last_undegraded":"2023-09-23T15:09:53.841522+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"145'9409","reported_seq":9450,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:58.076522+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:09:58.076522+0000","last_peered":"2023-09-23T15:09:58.076522+0000","last_clean":"2023-09-23T15:09:58.076522+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:09:58.076522+0000","last_undegraded":"2023-09-23T15:09:58.076522+0000","last_fullsized":"2023-09-23T15:09:58.076522+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9409,"log_dups_size":0,"ondisk_log_size":9409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299958272,"num_objects":4577,"num_object_clones":0,"num_object_copies":13731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9409,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":104,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817442+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:53.817442+0000","last_peered":"2023-09-23T15:09:53.817442+0000","last_clean":"2023-09-23T15:09:53.817442+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:09:53.817442+0000","last_undegraded":"2023-09-23T15:09:53.817442+0000","last_fullsized":"2023-09-23T15:09:53.817442+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":93,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.814719+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:09:53.814719+0000","last_peered":"2023-09-23T15:09:53.814719+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:09:53.814719+0000","last_undegraded":"2023-09-23T15:09:53.814719+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.871353+0000","last_change":"2023-09-23T15:09:53.860607+0000","last_active":"2023-09-23T15:09:53.871353+0000","last_peered":"2023-09-23T15:09:53.871353+0000","last_clean":"2023-09-23T15:09:53.871353+0000","last_became_active":"2023-09-23T15:09:53.859972+0000","last_became_peered":"2023-09-23T15:09:53.859972+0000","last_unstale":"2023-09-23T15:09:53.871353+0000","last_undegraded":"2023-09-23T15:09:53.871353+0000","last_fullsized":"2023-09-23T15:09:53.871353+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"145'9407","reported_seq":9449,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:58.101460+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:09:58.101460+0000","last_peered":"2023-09-23T15:09:58.101460+0000","last_clean":"2023-09-23T15:09:58.101460+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:09:58.101460+0000","last_undegraded":"2023-09-23T15:09:58.101460+0000","last_fullsized":"2023-09-23T15:09:58.101460+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294912024,"num_objects":4501,"num_object_clones":0,"num_object_copies":13503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4501,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9407,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.825692+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:09:53.825692+0000","last_peered":"2023-09-23T15:09:53.825692+0000","last_clean":"2023-09-23T15:09:53.825692+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:09:53.825692+0000","last_undegraded":"2023-09-23T15:09:53.825692+0000","last_fullsized":"2023-09-23T15:09:53.825692+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":103,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.818840+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:09:53.818840+0000","last_peered":"2023-09-23T15:09:53.818840+0000","last_clean":"2023-09-23T15:09:53.818840+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:09:53.818840+0000","last_undegraded":"2023-09-23T15:09:53.818840+0000","last_fullsized":"2023-09-23T15:09:53.818840+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.876083+0000","last_change":"2023-09-23T15:09:53.869033+0000","last_active":"2023-09-23T15:09:53.876083+0000","last_peered":"2023-09-23T15:09:53.876083+0000","last_clean":"2023-09-23T15:09:53.876083+0000","last_became_active":"2023-09-23T15:09:53.868759+0000","last_became_peered":"2023-09-23T15:09:53.868759+0000","last_unstale":"2023-09-23T15:09:53.876083+0000","last_undegraded":"2023-09-23T15:09:53.876083+0000","last_fullsized":"2023-09-23T15:09:53.876083+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"145'9515","reported_seq":9560,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:59.191106+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:09:59.191106+0000","last_peered":"2023-09-23T15:09:59.191106+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:09:59.191106+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"121'2100","ondisk_log_start":"121'2100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4457,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9515,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":4457}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.841352+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:09:53.841352+0000","last_peered":"2023-09-23T15:09:53.841352+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:09:53.841352+0000","last_undegraded":"2023-09-23T15:09:53.841352+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":127,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.818770+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:09:53.818770+0000","last_peered":"2023-09-23T15:09:53.818770+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:09:53.818770+0000","last_undegraded":"2023-09-23T15:09:53.818770+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"145'9762","reported_seq":9800,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.216767+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:09:59.216767+0000","last_peered":"2023-09-23T15:09:59.216767+0000","last_clean":"2023-09-23T15:09:59.216767+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:09:59.216767+0000","last_undegraded":"2023-09-23T15:09:59.216767+0000","last_fullsized":"2023-09-23T15:09:59.216767+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9762,"log_dups_size":0,"ondisk_log_size":9762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292159488,"num_objects":4458,"num_object_clones":0,"num_object_copies":13374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9762,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":104,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.819267+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:09:53.819267+0000","last_peered":"2023-09-23T15:09:53.819267+0000","last_clean":"2023-09-23T15:09:53.819267+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:09:53.819267+0000","last_undegraded":"2023-09-23T15:09:53.819267+0000","last_fullsized":"2023-09-23T15:09:53.819267+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.825824+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:09:53.825824+0000","last_peered":"2023-09-23T15:09:53.825824+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:09:53.825824+0000","last_undegraded":"2023-09-23T15:09:53.825824+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.880211+0000","last_change":"2023-09-23T15:09:53.870148+0000","last_active":"2023-09-23T15:09:53.880211+0000","last_peered":"2023-09-23T15:09:53.880211+0000","last_clean":"2023-09-23T15:09:53.880211+0000","last_became_active":"2023-09-23T15:09:53.869639+0000","last_became_peered":"2023-09-23T15:09:53.869639+0000","last_unstale":"2023-09-23T15:09:53.880211+0000","last_undegraded":"2023-09-23T15:09:53.880211+0000","last_fullsized":"2023-09-23T15:09:53.880211+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"145'9430","reported_seq":9469,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.182456+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:09:59.182456+0000","last_peered":"2023-09-23T15:09:59.182456+0000","last_clean":"2023-09-23T15:09:59.182456+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:09:59.182456+0000","last_undegraded":"2023-09-23T15:09:59.182456+0000","last_fullsized":"2023-09-23T15:09:59.182456+0000","mapping_epoch":114,"log_start":"121'2100","ondisk_log_start":"121'2100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7330,"log_dups_size":0,"ondisk_log_size":7330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287834112,"num_objects":4392,"num_object_clones":0,"num_object_copies":13176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9430,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.824174+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:09:53.824174+0000","last_peered":"2023-09-23T15:09:53.824174+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:09:53.824174+0000","last_undegraded":"2023-09-23T15:09:53.824174+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":59,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.826954+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:09:53.826954+0000","last_peered":"2023-09-23T15:09:53.826954+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:09:53.826954+0000","last_undegraded":"2023-09-23T15:09:53.826954+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.877928+0000","last_change":"2023-09-23T15:09:53.869836+0000","last_active":"2023-09-23T15:09:53.877928+0000","last_peered":"2023-09-23T15:09:53.877928+0000","last_clean":"2023-09-23T15:09:53.877928+0000","last_became_active":"2023-09-23T15:09:53.869503+0000","last_became_peered":"2023-09-23T15:09:53.869503+0000","last_unstale":"2023-09-23T15:09:53.877928+0000","last_undegraded":"2023-09-23T15:09:53.877928+0000","last_fullsized":"2023-09-23T15:09:53.877928+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":336,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.813246+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:09:53.813246+0000","last_peered":"2023-09-23T15:09:53.813246+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:09:53.813246+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":121,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817279+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:09:53.817279+0000","last_peered":"2023-09-23T15:09:53.817279+0000","last_clean":"2023-09-23T15:09:53.817279+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:09:53.817279+0000","last_undegraded":"2023-09-23T15:09:53.817279+0000","last_fullsized":"2023-09-23T15:09:53.817279+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":64,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.809034+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:09:53.809034+0000","last_peered":"2023-09-23T15:09:53.809034+0000","last_clean":"2023-09-23T15:09:53.809034+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:09:53.809034+0000","last_undegraded":"2023-09-23T15:09:53.809034+0000","last_fullsized":"2023-09-23T15:09:53.809034+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"145'9282","reported_seq":9322,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:58.978306+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:09:58.978306+0000","last_peered":"2023-09-23T15:09:58.978306+0000","last_clean":"2023-09-23T15:09:58.978306+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:09:58.978306+0000","last_undegraded":"2023-09-23T15:09:58.978306+0000","last_fullsized":"2023-09-23T15:09:58.978306+0000","mapping_epoch":114,"log_start":"122'2447","ondisk_log_start":"122'2447","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6835,"log_dups_size":0,"ondisk_log_size":6835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282066944,"num_objects":4304,"num_object_clones":0,"num_object_copies":12912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9282,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":493,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.871333+0000","last_change":"2023-09-23T15:09:53.860391+0000","last_active":"2023-09-23T15:09:53.871333+0000","last_peered":"2023-09-23T15:09:53.871333+0000","last_clean":"2023-09-23T15:09:53.871333+0000","last_became_active":"2023-09-23T15:09:53.859571+0000","last_became_peered":"2023-09-23T15:09:53.859571+0000","last_unstale":"2023-09-23T15:09:53.871333+0000","last_undegraded":"2023-09-23T15:09:53.871333+0000","last_fullsized":"2023-09-23T15:09:53.871333+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"145'9686","reported_seq":9731,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:59.024219+0000","last_change":"2023-09-23T15:08:51.256935+0000","last_active":"2023-09-23T15:09:59.024219+0000","last_peered":"2023-09-23T15:09:59.024219+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:08:51.256935+0000","last_became_peered":"2023-09-23T15:08:51.256935+0000","last_unstale":"2023-09-23T15:09:59.024219+0000","last_undegraded":"2023-09-23T15:08:51.248529+0000","last_fullsized":"2023-09-23T15:08:51.248430+0000","mapping_epoch":132,"log_start":"122'2842","ondisk_log_start":"122'2842","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6844,"log_dups_size":0,"ondisk_log_size":6844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:33.609757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292290560,"num_objects":4460,"num_object_clones":0,"num_object_copies":13380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4460,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9686,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":4460}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":653,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.817867+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:09:53.817867+0000","last_peered":"2023-09-23T15:09:53.817867+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:09:53.817867+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":104,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.818561+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:09:53.818561+0000","last_peered":"2023-09-23T15:09:53.818561+0000","last_clean":"2023-09-23T15:09:53.818561+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:09:53.818561+0000","last_undegraded":"2023-09-23T15:09:53.818561+0000","last_fullsized":"2023-09-23T15:09:53.818561+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":74,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.809185+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:09:53.809185+0000","last_peered":"2023-09-23T15:09:53.809185+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:09:53.809185+0000","last_undegraded":"2023-09-23T15:09:53.809185+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.875988+0000","last_change":"2023-09-23T15:09:53.863434+0000","last_active":"2023-09-23T15:09:53.875988+0000","last_peered":"2023-09-23T15:09:53.875988+0000","last_clean":"2023-09-23T15:09:53.875988+0000","last_became_active":"2023-09-23T15:09:53.863114+0000","last_became_peered":"2023-09-23T15:09:53.863114+0000","last_unstale":"2023-09-23T15:09:53.875988+0000","last_undegraded":"2023-09-23T15:09:53.875988+0000","last_fullsized":"2023-09-23T15:09:53.875988+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"145'9416","reported_seq":9457,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:58.070662+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:09:58.070662+0000","last_peered":"2023-09-23T15:09:58.070662+0000","last_clean":"2023-09-23T15:09:58.070662+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:09:58.070662+0000","last_undegraded":"2023-09-23T15:09:58.070662+0000","last_fullsized":"2023-09-23T15:09:58.070662+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9416,"log_dups_size":0,"ondisk_log_size":9416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295960576,"num_objects":4516,"num_object_clones":0,"num_object_copies":13548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9416,"num_write_kb":445824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":755,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.808765+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:09:53.808765+0000","last_peered":"2023-09-23T15:09:53.808765+0000","last_clean":"2023-09-23T15:09:53.808765+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:09:53.808765+0000","last_undegraded":"2023-09-23T15:09:53.808765+0000","last_fullsized":"2023-09-23T15:09:53.808765+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":54,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.808685+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:09:53.808685+0000","last_peered":"2023-09-23T15:09:53.808685+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:09:53.808685+0000","last_undegraded":"2023-09-23T15:09:53.808685+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":79,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.817604+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:09:53.817604+0000","last_peered":"2023-09-23T15:09:53.817604+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:09:53.817604+0000","last_undegraded":"2023-09-23T15:09:53.817604+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":390,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817536+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:09:53.817536+0000","last_peered":"2023-09-23T15:09:53.817536+0000","last_clean":"2023-09-23T15:09:53.817536+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:09:53.817536+0000","last_undegraded":"2023-09-23T15:09:53.817536+0000","last_fullsized":"2023-09-23T15:09:53.817536+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":495,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.869427+0000","last_change":"2023-09-23T15:09:53.855531+0000","last_active":"2023-09-23T15:09:53.869427+0000","last_peered":"2023-09-23T15:09:53.869427+0000","last_clean":"2023-09-23T15:09:53.869427+0000","last_became_active":"2023-09-23T15:09:53.855178+0000","last_became_peered":"2023-09-23T15:09:53.855178+0000","last_unstale":"2023-09-23T15:09:53.869427+0000","last_undegraded":"2023-09-23T15:09:53.869427+0000","last_fullsized":"2023-09-23T15:09:53.869427+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":56,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.826631+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:09:53.826631+0000","last_peered":"2023-09-23T15:09:53.826631+0000","last_clean":"2023-09-23T15:09:53.826631+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:09:53.826631+0000","last_undegraded":"2023-09-23T15:09:53.826631+0000","last_fullsized":"2023-09-23T15:09:53.826631+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":85,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.840137+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:09:53.840137+0000","last_peered":"2023-09-23T15:09:53.840137+0000","last_clean":"2023-09-23T15:09:53.840137+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:09:53.840137+0000","last_undegraded":"2023-09-23T15:09:53.840137+0000","last_fullsized":"2023-09-23T15:09:53.840137+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"145'9428","reported_seq":9466,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.113324+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:09:59.113324+0000","last_peered":"2023-09-23T15:09:59.113324+0000","last_clean":"2023-09-23T15:09:59.113324+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:09:59.113324+0000","last_undegraded":"2023-09-23T15:09:59.113324+0000","last_fullsized":"2023-09-23T15:09:59.113324+0000","mapping_epoch":114,"log_start":"122'2741","ondisk_log_start":"122'2741","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6687,"log_dups_size":0,"ondisk_log_size":6687,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287703040,"num_objects":4390,"num_object_clones":0,"num_object_copies":13170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4390,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9428,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.880179+0000","last_change":"2023-09-23T15:09:53.869987+0000","last_active":"2023-09-23T15:09:53.880179+0000","last_peered":"2023-09-23T15:09:53.880179+0000","last_clean":"2023-09-23T15:09:53.880179+0000","last_became_active":"2023-09-23T15:09:53.869669+0000","last_became_peered":"2023-09-23T15:09:53.869669+0000","last_unstale":"2023-09-23T15:09:53.880179+0000","last_undegraded":"2023-09-23T15:09:53.880179+0000","last_fullsized":"2023-09-23T15:09:53.880179+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"145'9652","reported_seq":9697,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:59.227927+0000","last_change":"2023-09-23T15:08:51.252737+0000","last_active":"2023-09-23T15:09:59.227927+0000","last_peered":"2023-09-23T15:09:59.227927+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:08:51.252737+0000","last_became_peered":"2023-09-23T15:08:51.252737+0000","last_unstale":"2023-09-23T15:09:59.227927+0000","last_undegraded":"2023-09-23T15:08:51.248004+0000","last_fullsized":"2023-09-23T15:08:51.247871+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9652,"log_dups_size":0,"ondisk_log_size":9652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.686400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4464,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9652,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":4464}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":356,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.827112+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:09:53.827112+0000","last_peered":"2023-09-23T15:09:53.827112+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:09:53.827112+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.811113+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:09:53.811113+0000","last_peered":"2023-09-23T15:09:53.811113+0000","last_clean":"2023-09-23T15:09:53.811113+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:09:53.811113+0000","last_undegraded":"2023-09-23T15:09:53.811113+0000","last_fullsized":"2023-09-23T15:09:53.811113+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":108,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.812736+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:09:53.812736+0000","last_peered":"2023-09-23T15:09:53.812736+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:09:53.812736+0000","last_undegraded":"2023-09-23T15:09:53.812736+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.877344+0000","last_change":"2023-09-23T15:09:53.870560+0000","last_active":"2023-09-23T15:09:53.877344+0000","last_peered":"2023-09-23T15:09:53.877344+0000","last_clean":"2023-09-23T15:09:53.877344+0000","last_became_active":"2023-09-23T15:09:53.870279+0000","last_became_peered":"2023-09-23T15:09:53.870279+0000","last_unstale":"2023-09-23T15:09:53.877344+0000","last_undegraded":"2023-09-23T15:09:53.877344+0000","last_fullsized":"2023-09-23T15:09:53.877344+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":814,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824338+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:09:53.824338+0000","last_peered":"2023-09-23T15:09:53.824338+0000","last_clean":"2023-09-23T15:09:53.824338+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:09:53.824338+0000","last_undegraded":"2023-09-23T15:09:53.824338+0000","last_fullsized":"2023-09-23T15:09:53.824338+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":54,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.809308+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:09:53.809308+0000","last_peered":"2023-09-23T15:09:53.809308+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:09:53.809308+0000","last_undegraded":"2023-09-23T15:09:53.809308+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":72,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.814818+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:09:53.814818+0000","last_peered":"2023-09-23T15:09:53.814818+0000","last_clean":"2023-09-23T15:09:53.814818+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:09:53.814818+0000","last_undegraded":"2023-09-23T15:09:53.814818+0000","last_fullsized":"2023-09-23T15:09:53.814818+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"145'9682","reported_seq":9720,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.020061+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:09:59.020061+0000","last_peered":"2023-09-23T15:09:59.020061+0000","last_clean":"2023-09-23T15:09:59.020061+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:09:59.020061+0000","last_undegraded":"2023-09-23T15:09:59.020061+0000","last_fullsized":"2023-09-23T15:09:59.020061+0000","mapping_epoch":114,"log_start":"121'2360","ondisk_log_start":"121'2360","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7322,"log_dups_size":0,"ondisk_log_size":7322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402368,"num_objects":4538,"num_object_clones":0,"num_object_copies":13614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9682,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.869922+0000","last_change":"2023-09-23T15:09:53.857751+0000","last_active":"2023-09-23T15:09:53.869922+0000","last_peered":"2023-09-23T15:09:53.869922+0000","last_clean":"2023-09-23T15:09:53.869922+0000","last_became_active":"2023-09-23T15:09:53.857395+0000","last_became_peered":"2023-09-23T15:09:53.857395+0000","last_unstale":"2023-09-23T15:09:53.869922+0000","last_undegraded":"2023-09-23T15:09:53.869922+0000","last_fullsized":"2023-09-23T15:09:53.869922+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.826887+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:09:53.826887+0000","last_peered":"2023-09-23T15:09:53.826887+0000","last_clean":"2023-09-23T15:09:53.826887+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:09:53.826887+0000","last_undegraded":"2023-09-23T15:09:53.826887+0000","last_fullsized":"2023-09-23T15:09:53.826887+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":639,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.840316+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:09:53.840316+0000","last_peered":"2023-09-23T15:09:53.840316+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:09:53.840316+0000","last_undegraded":"2023-09-23T15:09:53.840316+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":102,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824085+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:09:53.824085+0000","last_peered":"2023-09-23T15:09:53.824085+0000","last_clean":"2023-09-23T15:09:53.824085+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:09:53.824085+0000","last_undegraded":"2023-09-23T15:09:53.824085+0000","last_fullsized":"2023-09-23T15:09:53.824085+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"145'9547","reported_seq":9586,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T15:09:59.172344+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:09:59.172344+0000","last_peered":"2023-09-23T15:09:59.172344+0000","last_clean":"2023-09-23T15:09:59.172344+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:09:59.172344+0000","last_undegraded":"2023-09-23T15:09:59.172344+0000","last_fullsized":"2023-09-23T15:09:59.172344+0000","mapping_epoch":114,"log_start":"121'2200","ondisk_log_start":"121'2200","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7347,"log_dups_size":0,"ondisk_log_size":7347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288423936,"num_objects":4401,"num_object_clones":0,"num_object_copies":13203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9547,"num_write_kb":446336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":494,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.877710+0000","last_change":"2023-09-23T15:09:53.870050+0000","last_active":"2023-09-23T15:09:53.877710+0000","last_peered":"2023-09-23T15:09:53.877710+0000","last_clean":"2023-09-23T15:09:53.877710+0000","last_became_active":"2023-09-23T15:09:53.869710+0000","last_became_peered":"2023-09-23T15:09:53.869710+0000","last_unstale":"2023-09-23T15:09:53.877710+0000","last_undegraded":"2023-09-23T15:09:53.877710+0000","last_fullsized":"2023-09-23T15:09:53.877710+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":575,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.840079+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:09:53.840079+0000","last_peered":"2023-09-23T15:09:53.840079+0000","last_clean":"2023-09-23T15:09:53.840079+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:09:53.840079+0000","last_undegraded":"2023-09-23T15:09:53.840079+0000","last_fullsized":"2023-09-23T15:09:53.840079+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":96,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.812599+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:09:53.812599+0000","last_peered":"2023-09-23T15:09:53.812599+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:09:53.812599+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":86,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.819398+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:09:53.819398+0000","last_peered":"2023-09-23T15:09:53.819398+0000","last_clean":"2023-09-23T15:09:53.819398+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:09:53.819398+0000","last_undegraded":"2023-09-23T15:09:53.819398+0000","last_fullsized":"2023-09-23T15:09:53.819398+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"145'9400","reported_seq":9446,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:59.182528+0000","last_change":"2023-09-23T15:08:51.257194+0000","last_active":"2023-09-23T15:09:59.182528+0000","last_peered":"2023-09-23T15:09:59.182528+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:08:51.257194+0000","last_became_peered":"2023-09-23T15:08:51.257194+0000","last_unstale":"2023-09-23T15:09:59.182528+0000","last_undegraded":"2023-09-23T15:08:51.246208+0000","last_fullsized":"2023-09-23T15:08:51.246087+0000","mapping_epoch":132,"log_start":"119'500","ondisk_log_start":"119'500","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8900,"log_dups_size":0,"ondisk_log_size":8900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.554621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4410,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9400,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":4410}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":75,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.814801+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:09:53.814801+0000","last_peered":"2023-09-23T15:09:53.814801+0000","last_clean":"2023-09-23T15:09:53.814801+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:09:53.814801+0000","last_undegraded":"2023-09-23T15:09:53.814801+0000","last_fullsized":"2023-09-23T15:09:53.814801+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":133,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.809050+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:09:53.809050+0000","last_peered":"2023-09-23T15:09:53.809050+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:09:53.809050+0000","last_undegraded":"2023-09-23T15:09:53.809050+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":113,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:09:53.817565+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:09:53.817565+0000","last_peered":"2023-09-23T15:09:53.817565+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:09:53.817565+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":127,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T15:09:53.818708+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:09:53.818708+0000","last_peered":"2023-09-23T15:09:53.818708+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:09:53.818708+0000","last_undegraded":"2023-09-23T15:09:53.818708+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":89,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.817315+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:09:53.817315+0000","last_peered":"2023-09-23T15:09:53.817315+0000","last_clean":"2023-09-23T15:09:53.817315+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:09:53.817315+0000","last_undegraded":"2023-09-23T15:09:53.817315+0000","last_fullsized":"2023-09-23T15:09:53.817315+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.819424+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:09:53.819424+0000","last_peered":"2023-09-23T15:09:53.819424+0000","last_clean":"2023-09-23T15:09:53.819424+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:09:53.819424+0000","last_undegraded":"2023-09-23T15:09:53.819424+0000","last_fullsized":"2023-09-23T15:09:53.819424+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":113,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.825496+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:09:53.825496+0000","last_peered":"2023-09-23T15:09:53.825496+0000","last_clean":"2023-09-23T15:09:53.825496+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:09:53.825496+0000","last_undegraded":"2023-09-23T15:09:53.825496+0000","last_fullsized":"2023-09-23T15:09:53.825496+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":128,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.824343+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:09:53.824343+0000","last_peered":"2023-09-23T15:09:53.824343+0000","last_clean":"2023-09-23T15:09:53.824343+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:09:53.824343+0000","last_undegraded":"2023-09-23T15:09:53.824343+0000","last_fullsized":"2023-09-23T15:09:53.824343+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":90,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.812133+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:09:53.812133+0000","last_peered":"2023-09-23T15:09:53.812133+0000","last_clean":"2023-09-23T15:09:53.812133+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:09:53.812133+0000","last_undegraded":"2023-09-23T15:09:53.812133+0000","last_fullsized":"2023-09-23T15:09:53.812133+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T15:09:53.818394+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:09:53.818394+0000","last_peered":"2023-09-23T15:09:53.818394+0000","last_clean":"2023-09-23T15:09:53.818394+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:09:53.818394+0000","last_undegraded":"2023-09-23T15:09:53.818394+0000","last_fullsized":"2023-09-23T15:09:53.818394+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4678549528,"num_objects":71390,"num_object_clones":0,"num_object_copies":214170,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22323,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71390,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":152144,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14472720384,"data_stored":14472708168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131948,"ondisk_log_size":131948,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201463,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1477512,"kb_used_data":1428032,"kb_used_omap":0,"kb_used_meta":49472,"kb_avail":92267640,"statfs":{"total":95995035648,"available":94482063360,"internally_reserved":0,"allocated":1462304768,"data_stored":1461760879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":50659328},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":4.29,"5min":1.6850000000000001,"15min":0.95399999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":46.814,"5min":46.814,"15min":46.814},"last":0.22500000000000001},{"interface":"front","average":{"1min":6.2210000000000001,"5min":2.133,"15min":1.1020000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":46.966999999999999,"5min":46.966999999999999,"15min":46.966999999999999},"last":15.935}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":10.708,"5min":4.8899999999999997,"15min":2.0409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":89.450999999999993,"5min":170.63800000000001,"15min":170.63800000000001},"last":0.26900000000000002},{"interface":"front","average":{"1min":12.073,"5min":5.1319999999999997,"15min":2.0880000000000001},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":93.789000000000001,"5min":170.47999999999999,"15min":170.47999999999999},"last":15.978999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:09:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5710000000000002,"5min":1.181,"15min":0.77100000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":6.8449999999999998,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":0.67200000000000004},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.1619999999999999,"15min":0.86399999999999999},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":6.7960000000000003,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.82599999999999996,"15min":0.61599999999999999},"min":{"1min":0.35599999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.2919999999999998,"5min":5.2919999999999998,"15min":5.2919999999999998},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.014,"15min":0.69299999999999995},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.23999999999999999},"max":{"1min":6.766,"5min":6.766,"15min":6.766},"last":18.132999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.871,"5min":0.92600000000000005,"15min":0.68400000000000005},"min":{"1min":0.31900000000000001,"5min":0.30599999999999999,"15min":0.26200000000000001},"max":{"1min":5.8970000000000002,"5min":5.8970000000000002,"15min":5.8970000000000002},"last":8.2050000000000001},{"interface":"front","average":{"1min":2.7869999999999999,"5min":1.1619999999999999,"15min":0.77400000000000002},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.4779999999999998,"5min":8.4779999999999998,"15min":8.4779999999999998},"last":0.57999999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":0.82999999999999996,"15min":0.67200000000000004},"min":{"1min":0.33700000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.1589999999999998,"5min":4.1589999999999998,"15min":4.1589999999999998},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.677,"5min":1.1259999999999999,"15min":0.77500000000000002},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":7.0190000000000001,"5min":7.0190000000000001,"15min":7.0190000000000001},"last":16.007999999999999}]}]},{"osd":6,"up_from":47,"seq":201863463099,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190460,"kb_used_data":1141300,"kb_used_omap":0,"kb_used_meta":49152,"kb_avail":92554692,"statfs":{"total":95995035648,"available":94776004608,"internally_reserved":0,"allocated":1168691200,"data_stored":1168159448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":50331648},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:51 2023","interfaces":[{"interface":"back","average":{"1min":9.3209999999999997,"5min":4.8739999999999997,"15min":1.988},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.721000000000004,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.72799999999999998},{"interface":"front","average":{"1min":10.013,"5min":5.0609999999999999,"15min":2.073},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":96.019000000000005,"5min":108.452,"15min":108.452},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:09:51 2023","interfaces":[{"interface":"back","average":{"1min":13.82,"5min":10.026,"15min":3.754},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":139.17500000000001,"5min":180.374,"15min":180.374},"last":5.9340000000000002},{"interface":"front","average":{"1min":13.638999999999999,"5min":10.130000000000001,"15min":3.79},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":146.999,"5min":181.685,"15min":181.685},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":2.1579999999999999,"5min":0.998,"15min":0.73499999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":17.084,"5min":17.084,"15min":17.084},"last":0.622},{"interface":"front","average":{"1min":1.728,"5min":0.89300000000000002,"15min":0.70499999999999996},"min":{"1min":0.40600000000000003,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.4880000000000004,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.48899999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.151,"5min":0.70399999999999996,"15min":0.56499999999999995},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.23599999999999999},"max":{"1min":7.4539999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":6.0359999999999996},{"interface":"front","average":{"1min":1.1160000000000001,"5min":0.76300000000000001,"15min":0.59599999999999997},"min":{"1min":0.34200000000000003,"5min":0.33800000000000002,"15min":0.25600000000000001},"max":{"1min":3.8210000000000002,"5min":3.8210000000000002,"15min":3.8210000000000002},"last":0.56699999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:09:03 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":0.79900000000000004,"15min":0.61599999999999999},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.9669999999999996,"5min":7.9669999999999996,"15min":7.9669999999999996},"last":6.0019999999999998},{"interface":"front","average":{"1min":1.0720000000000001,"5min":0.78600000000000003,"15min":0.627},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":3.9260000000000002,"5min":3.9260000000000002,"15min":3.9260000000000002},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:09:19 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":1.03,"15min":0.72899999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":17,"5min":17,"15min":17},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.2290000000000001,"5min":0.89300000000000002,"15min":0.71199999999999997},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.4080000000000004,"5min":7.4080000000000004,"15min":7.4080000000000004},"last":0.68700000000000006}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893186,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2391992,"kb_used_data":2288112,"kb_used_omap":8,"kb_used_meta":103863,"kb_avail":91353160,"statfs":{"total":95995035648,"available":93545635840,"internally_reserved":0,"allocated":2343026688,"data_stored":2342382743,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":106356308},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":1.6759999999999999,"15min":0.82699999999999996},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":11.285,"5min":39.130000000000003,"15min":39.130000000000003},"last":0.311},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.3069999999999999,"15min":0.70899999999999996},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.798,"5min":39.034999999999997,"15min":39.034999999999997},"last":0.23000000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:09:17 2023","interfaces":[{"interface":"back","average":{"1min":2.3679999999999999,"5min":1.0169999999999999,"15min":0.65100000000000002},"min":{"1min":0.30099999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":9.1180000000000003,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":0.372},{"interface":"front","average":{"1min":2.7519999999999998,"5min":1.4359999999999999,"15min":1.1499999999999999},"min":{"1min":0.31900000000000001,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.5949999999999998,"5min":11.978,"15min":11.978},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.206,"15min":0.83399999999999996},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":5.5140000000000002,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":4.6829999999999998},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.141,"15min":0.80500000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.218,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.48099999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.304,"15min":0.89900000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.7130000000000001,"5min":13.49,"15min":13.49},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.153,"15min":0.872},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.0030000000000001,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":7.585}]},{"osd":6,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.24,"15min":0.879},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":11.641999999999999,"5min":11.641999999999999,"15min":11.641999999999999},"last":4.6600000000000001},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.155,"15min":0.84099999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.8360000000000003,"5min":5.8360000000000003,"15min":5.8360000000000003},"last":3.4740000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.2649999999999999,"15min":0.872},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2989999999999995,"5min":8.2989999999999995,"15min":8.2989999999999995},"last":2.9689999999999999},{"interface":"front","average":{"1min":1.9970000000000001,"5min":1.335,"15min":0.92100000000000004},"min":{"1min":0.45000000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":6.7240000000000002,"5min":8.1760000000000002,"15min":8.1760000000000002},"last":7.3780000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566133,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1836204,"kb_used_data":1750436,"kb_used_omap":11,"kb_used_meta":85748,"kb_avail":91908948,"statfs":{"total":95995035648,"available":94114762752,"internally_reserved":0,"allocated":1792446464,"data_stored":1791812286,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":87806736},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.76200000000000001,"15min":0.52100000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.7309999999999999,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":0.20000000000000001},{"interface":"front","average":{"1min":1.141,"5min":0.74199999999999999,"15min":0.504},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.7729999999999997,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.32000000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":1.3540000000000001,"15min":0.71099999999999997},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":9.7579999999999991,"5min":57.695,"15min":57.695},"last":0.25700000000000001},{"interface":"front","average":{"1min":1.9039999999999999,"5min":1.462,"15min":0.754},"min":{"1min":0.29699999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.7279999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.97699999999999998,"15min":0.75900000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.0300000000000002,"5min":7.0300000000000002,"15min":7.0300000000000002},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.7480000000000002,"5min":1.2529999999999999,"15min":0.85999999999999999},"min":{"1min":0.46100000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.8360000000000003,"5min":5.8360000000000003,"15min":5.8360000000000003},"last":1.6040000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":1.3540000000000001,"15min":0.874},"min":{"1min":0.41199999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":6.6310000000000002},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.379,"5min":1.232,"15min":0.86599999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.9359999999999999,"5min":6.9359999999999999,"15min":6.9359999999999999},"last":4.6050000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.2050000000000001,"15min":0.88900000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.8209999999999997,"5min":6.9790000000000001,"15min":6.9790000000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":6.2220000000000004,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":4.1879999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":1.296,"15min":0.90400000000000003},"min":{"1min":0.45100000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.2680000000000007,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.744,"5min":1.1910000000000001,"15min":0.86199999999999999},"min":{"1min":0.41899999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.8239999999999998,"5min":5.8239999999999998,"15min":5.8239999999999998},"last":0.60799999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402607,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":930688,"kb_used_data":858936,"kb_used_omap":3,"kb_used_meta":71740,"kb_avail":92814464,"statfs":{"total":95995035648,"available":95042011136,"internally_reserved":0,"allocated":879550464,"data_stored":878955608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":73461929},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":7.1520000000000001,"5min":1.8400000000000001,"15min":0.86399999999999999},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":79.611000000000004,"5min":79.611000000000004,"15min":79.611000000000004},"last":3.343},{"interface":"front","average":{"1min":7.0190000000000001,"5min":1.8899999999999999,"15min":0.88800000000000001},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":79.656000000000006,"5min":79.656000000000006,"15min":79.656000000000006},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":15.308999999999999,"5min":3.6150000000000002,"15min":1.619},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":105.795,"5min":105.795,"15min":105.795},"last":0.42799999999999999},{"interface":"front","average":{"1min":16.312000000000001,"5min":3.8159999999999998,"15min":1.554},"min":{"1min":0.35699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":106.39,"5min":106.39,"15min":106.39},"last":0.68600000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.79900000000000004,"15min":0.66500000000000004},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.835,"5min":6.835,"15min":6.835},"last":0.63},{"interface":"front","average":{"1min":1.8859999999999999,"5min":1.1060000000000001,"15min":0.86199999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.9009999999999998,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":6.1050000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.6579999999999999,"5min":0.92500000000000004,"15min":0.73399999999999999},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.3780000000000001,"5min":8.3780000000000001,"15min":8.3780000000000001},"last":0.875},{"interface":"front","average":{"1min":2.129,"5min":1.1579999999999999,"15min":0.92200000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.9789999999999992,"5min":8.9789999999999992,"15min":8.9789999999999992},"last":6.0439999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":1.903,"5min":0.94999999999999996,"15min":0.76600000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.6020000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":0.66000000000000003},{"interface":"front","average":{"1min":2.6640000000000001,"5min":1.276,"15min":0.98199999999999998},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":9.2699999999999996,"5min":9.2699999999999996,"15min":9.2699999999999996},"last":6.133}]},{"osd":7,"last update":"Sat Sep 23 15:09:20 2023","interfaces":[{"interface":"back","average":{"1min":2.327,"5min":1.236,"15min":0.93999999999999995},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.0269999999999992,"5min":9.0269999999999992,"15min":9.0269999999999992},"last":6.0839999999999996},{"interface":"front","average":{"1min":1.276,"5min":0.85099999999999998,"15min":0.71799999999999997},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":3.738,"5min":4.1660000000000004,"15min":4.1660000000000004},"last":0.46700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986372,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2370364,"kb_used_data":2286068,"kb_used_omap":0,"kb_used_meta":84288,"kb_avail":91374788,"statfs":{"total":95995035648,"available":93567782912,"internally_reserved":0,"allocated":2340933632,"data_stored":2340401393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":86310912},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:49 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":3.6749999999999998,"15min":1.583},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":108.148,"5min":122.69199999999999,"15min":122.69199999999999},"last":5.3639999999999999},{"interface":"front","average":{"1min":8.0969999999999995,"5min":3.992,"15min":1.6879999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":108.124,"5min":122.745,"15min":122.745},"last":0.38700000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:09:49 2023","interfaces":[{"interface":"back","average":{"1min":10.913,"5min":3.5449999999999999,"15min":1.5760000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":109.541,"5min":109.541,"15min":109.541},"last":0.41199999999999998},{"interface":"front","average":{"1min":11.193,"5min":3.8290000000000002,"15min":1.768},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.619,"5min":109.619,"15min":109.619},"last":0.433}]},{"osd":3,"last update":"Sat Sep 23 15:09:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.1240000000000001,"15min":0.748},"min":{"1min":0.40300000000000002,"5min":0.317,"15min":0.317},"max":{"1min":6.8920000000000003,"5min":6.8920000000000003,"15min":6.8920000000000003},"last":5.407},{"interface":"front","average":{"1min":1.103,"5min":0.82299999999999995,"15min":0.68100000000000005},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.4889999999999999,"5min":4.4889999999999999,"15min":4.4889999999999999},"last":3.645}]},{"osd":5,"last update":"Sat Sep 23 15:09:32 2023","interfaces":[{"interface":"back","average":{"1min":4.9950000000000001,"5min":1.6160000000000001,"15min":0.88},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":28.891999999999999,"5min":28.891999999999999,"15min":28.891999999999999},"last":5.4640000000000004},{"interface":"front","average":{"1min":3.847,"5min":1.427,"15min":0.82999999999999996},"min":{"1min":0.35999999999999999,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":32.100999999999999,"5min":32.100999999999999,"15min":32.100999999999999},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:09:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9930000000000001,"5min":0.93600000000000005,"15min":0.70499999999999996},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":7.4509999999999996,"5min":7.4509999999999996,"15min":7.4509999999999996},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.92000000000000004,"15min":0.68899999999999995},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33400000000000002},"max":{"1min":5.6470000000000002,"5min":5.6470000000000002,"15min":5.6470000000000002},"last":0.67700000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:09:10 2023","interfaces":[{"interface":"back","average":{"1min":3.524,"5min":1.292,"15min":0.82599999999999996},"min":{"1min":0.40300000000000002,"5min":0.376,"15min":0.28799999999999998},"max":{"1min":11.711,"5min":11.711,"15min":11.711},"last":18.23},{"interface":"front","average":{"1min":3.3479999999999999,"5min":1.296,"15min":0.87},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":14.653,"5min":14.653,"15min":14.653},"last":0.71499999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724735,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2658292,"kb_used_data":2570028,"kb_used_omap":0,"kb_used_meta":88000,"kb_avail":91086860,"statfs":{"total":95995035648,"available":93272944640,"internally_reserved":0,"allocated":2631708672,"data_stored":2631180412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":90112000},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1479999999999999,"15min":0.747},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":6.9569999999999999,"5min":24.962,"15min":24.962},"last":3.6890000000000001},{"interface":"front","average":{"1min":1.879,"5min":1.24,"15min":0.77000000000000002},"min":{"1min":0.29199999999999998,"5min":0.222,"15min":0.222},"max":{"1min":13.788,"5min":25.317,"15min":25.317},"last":0.29999999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3610000000000002,"5min":1.589,"15min":0.90400000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.228999999999999,"5min":19.236999999999998,"15min":19.236999999999998},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.4259999999999999,"15min":0.86799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.1980000000000004,"5min":19.699999999999999,"15min":19.699999999999999},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:09:26 2023","interfaces":[{"interface":"back","average":{"1min":2.5950000000000002,"5min":1.1180000000000001,"15min":0.77700000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":15.265000000000001,"5min":15.265000000000001,"15min":15.265000000000001},"last":5.5369999999999999},{"interface":"front","average":{"1min":1.599,"5min":1.05,"15min":0.76700000000000002},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.8390000000000004,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":3.7149999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":0.877,"15min":0.59999999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":6.8799999999999999,"5min":6.8799999999999999,"15min":6.8799999999999999},"last":3.6600000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":0.94999999999999996,"15min":0.65900000000000003},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.6,"5min":13.6,"15min":13.6},"last":5.3099999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:09:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4159999999999999,"5min":1.3149999999999999,"15min":0.78900000000000003},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":15.196,"5min":15.196,"15min":15.196},"last":3.7320000000000002},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.028,"15min":0.74399999999999999},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4689999999999994,"5min":9.4689999999999994,"15min":9.4689999999999994},"last":5.2729999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:09:26 2023","interfaces":[{"interface":"back","average":{"1min":2.976,"5min":1.2370000000000001,"15min":0.80300000000000005},"min":{"1min":0.41099999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":15.301,"5min":15.301,"15min":15.301},"last":5.2990000000000004},{"interface":"front","average":{"1min":2.2469999999999999,"5min":1.129,"15min":0.76300000000000001},"min":{"1min":0.33800000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.361000000000001,"5min":13.361000000000001,"15min":13.361000000000001},"last":13.095000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2334195712,"data_stored":2334195712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1773408256,"data_stored":1773404184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":874840064,"data_stored":874840064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2331246592,"data_stored":2331246592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2613645312,"data_stored":2613641240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1164836864,"data_stored":1164836864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1453395968,"data_stored":1453391896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:10:03.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-09-23T15:10:03.310 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:03.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:03.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:03.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:02 smithi027 ceph-mon[195233]: pgmap v344: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 829 op/s; 22329/214254 objects degraded (10.422%) 2023-09-23T15:10:03.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:03.554 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:03.799 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:03.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:03.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:03.830 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:03.881 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:03.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:03.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:03.954 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:04.003 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:04.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:04.064 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:04.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:04.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:04.119 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:04.120 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:04.126 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.127 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:10:04.145 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3733009405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:04.145 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3733009405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:04.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:04.234 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:04.291 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:04.317 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:04.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:04.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:04.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3733009405' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:04.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:04.460 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:04.581 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:04.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:04.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:04.767 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:04.851 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:04.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:04.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:04.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:05.011 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:04 smithi027 ceph-mon[195233]: pgmap v345: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 21854/206922 objects degraded (10.561%) 2023-09-23T15:10:05.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:04 smithi017 ceph-mon[215646]: pgmap v345: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 21854/206922 objects degraded (10.561%) 2023-09-23T15:10:05.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:04 smithi017 ceph-mon[222878]: pgmap v345: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 815 op/s; 21854/206922 objects degraded (10.561%) 2023-09-23T15:10:05.853 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:05.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:05.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:06.010 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:06.061 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:06.112 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:06.115 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:06.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:06.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:10:06.284 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:06.395 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:06.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:06.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:06.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:06.558 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:06.674 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:06.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:06.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:06.846 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:06.963 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:06.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:06.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:07.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:07.151 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:07.239 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:07.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:07.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:07.313 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:07.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:07.432 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:07.508 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:07 smithi017 ceph-mon[215646]: pgmap v346: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 21134/202719 objects degraded (10.425%) 2023-09-23T15:10:07.508 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21134/202719 objects degraded (10.425%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:07.509 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:07 smithi017 ceph-mon[222878]: pgmap v346: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 21134/202719 objects degraded (10.425%) 2023-09-23T15:10:07.509 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21134/202719 objects degraded (10.425%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:07.543 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:07.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:07.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:07.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:07 smithi027 ceph-mon[195233]: pgmap v346: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 21134/202719 objects degraded (10.425%) 2023-09-23T15:10:07.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21134/202719 objects degraded (10.425%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:07.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:07.745 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:07.869 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:07.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:07.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:07.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:08.023 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:08] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:10:08.059 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:08.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:10:08.161 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:08.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:08.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:08.286 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/28453206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1454103846' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:08.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:08.309 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:10:08.332 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 5 pgp_num to 16 2023-09-23T15:10:08.371 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/28453206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1454103846' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:08.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/28453206' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:08.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1454103846' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:08.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:10:09.094 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:09.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:09.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:09.227 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:09.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: pgmap v347: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 20868/198969 objects degraded (10.488%) 2023-09-23T15:10:09.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:09.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:09 smithi027 ceph-mon[195233]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: pgmap v347: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 20868/198969 objects degraded (10.488%) 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:09.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[215646]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: pgmap v347: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 818 op/s; 20868/198969 objects degraded (10.488%) 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: osdmap e146: 8 total, 7 up, 8 in 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:09 smithi017 ceph-mon[222878]: failed to encode map e146 with expected crc 2023-09-23T15:10:09.696 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:09.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:09.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:09.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:09.881 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:09.923 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:09.982 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:09.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:09.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:10.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:10.176 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:10.312 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:10.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:10.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:10.429 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:10.445 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:10.445 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:10.492 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:10.615 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:10.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:10.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:10.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:10.785 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:11.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:11 smithi017 ceph-mon[222878]: pgmap v349: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:11.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:10:11.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:11 smithi017 ceph-mon[215646]: pgmap v349: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:11.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:10:11.612 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:11 smithi027 ceph-mon[195233]: pgmap v349: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:11.612 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:10:11.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:10:11.728 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:11.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:11.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:11.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:11.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:10:11.930 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:12.137 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:12.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:12.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:12.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:12.330 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:12.434 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:12.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 19878/190839 objects degraded (10.416%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:12.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:12.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:12.545 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:12.627 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:12.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 19878/190839 objects degraded (10.416%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:12.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 19878/190839 objects degraded (10.416%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:12.652 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:12.663 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:12.714 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:12.729 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:12.753 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:12.788 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:12.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:12.797 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:12.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:12.841 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:12.879 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:12.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:12.920 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:10:12.962 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:13.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:13 smithi027 ceph-mon[195233]: pgmap v350: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:13.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/587772881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:13.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:13.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[215646]: pgmap v350: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/587772881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:13.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[222878]: pgmap v350: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 780 op/s; 19878/190839 objects degraded (10.416%) 2023-09-23T15:10:13.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/587772881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:13.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:13.741 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:13.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:13.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:13.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:13.906 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:14.196 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:14.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:14.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:14.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:14.318 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:14.377 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:14.512 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:14.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:14.516 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:14.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:14.690 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:14.766 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:10:14.767 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:10:14.767 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T15:10:14.767 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:10:14.767 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:10:14.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T15:10:14.978 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:14.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:14.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:15.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:15.130 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:15.335 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:15 smithi027 ceph-mon[195233]: pgmap v351: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 19385/183429 objects degraded (10.568%) 2023-09-23T15:10:15.349 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:15.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:15.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:15.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:15.545 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:15.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:15 smithi017 ceph-mon[222878]: pgmap v351: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 19385/183429 objects degraded (10.568%) 2023-09-23T15:10:15.581 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:15 smithi017 ceph-mon[215646]: pgmap v351: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 19385/183429 objects degraded (10.568%) 2023-09-23T15:10:15.671 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:15.697 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:15.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:15.706 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:15.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:15.863 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:16.089 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:16.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:16.095 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:16.180 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:10:16.180 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:10:16.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:165613 5.0 5:0da79dff:::benchmark_data_smithi017_288048_object53844:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:16.173557+0000", 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0064692100000000004, 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0065051429999999997, 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165613 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.181 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173557+0000", 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173557+0000", 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173556+0000", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173564+0000", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173582+0000", 2023-09-23T15:10:16.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7278999999999998e-05 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173592+0000", 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0465000000000001e-05 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173650+0000", 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8171999999999998e-05 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173800+0000", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014997200000000001 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.173927+0000", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126964 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:16.178686+0000", 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047590230000000002 2023-09-23T15:10:16.185 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 1 2023-09-23T15:10:16.186 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:10:16.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:16.264 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:16.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:10:16.602 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:16.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:16.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:16.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:16.752 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:17.107 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:17.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:10:17.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:17.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:17.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:17.266 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:10:17.281 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:17.286 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: pgmap v352: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 18647/179091 objects degraded (10.412%) 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2752212169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 18647/179091 objects degraded (10.412%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: pgmap v352: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 18647/179091 objects degraded (10.412%) 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2752212169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 18647/179091 objects degraded (10.412%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: pgmap v352: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 18647/179091 objects degraded (10.412%) 2023-09-23T15:10:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2752212169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 18647/179091 objects degraded (10.412%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:18.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:18.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:18.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[215646]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[215646]: failed to encode map e147 with expected crc 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:18] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[222878]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T15:10:18.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:18 smithi017 ceph-mon[222878]: failed to encode map e147 with expected crc 2023-09-23T15:10:18.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:18 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:18.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:18.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:18 smithi027 ceph-mon[195233]: osdmap e147: 8 total, 7 up, 8 in 2023-09-23T15:10:18.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:18 smithi027 ceph-mon[195233]: failed to encode map e147 with expected crc 2023-09-23T15:10:18.608 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:18.719 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:18.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:18.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:18.825 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:18.915 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:19.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:19.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:19.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:19.187 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:19.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:19 smithi027 ceph-mon[195233]: failed to encode map e147 with expected crc 2023-09-23T15:10:19.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:19 smithi027 ceph-mon[195233]: pgmap v354: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 18429/175635 objects degraded (10.493%) 2023-09-23T15:10:19.505 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:19.506 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:10:19.506 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:10:19.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:19 smithi017 ceph-mon[222878]: failed to encode map e147 with expected crc 2023-09-23T15:10:19.605 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:19 smithi017 ceph-mon[222878]: pgmap v354: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 18429/175635 objects degraded (10.493%) 2023-09-23T15:10:19.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:19 smithi017 ceph-mon[215646]: failed to encode map e147 with expected crc 2023-09-23T15:10:19.613 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:19 smithi017 ceph-mon[215646]: pgmap v354: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 18429/175635 objects degraded (10.493%) 2023-09-23T15:10:19.639 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:19.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:19.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:19.822 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:20.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:10:20.618 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:20.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:20.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:20.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:20.814 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:20.979 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:20.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:20.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:21.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:21.192 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:21.217 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:21.340 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:21.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:21.352 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:21.389 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:21.431 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:21.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:21.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:21.517 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:21.518 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:21.561 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:21 smithi027 ceph-mon[195233]: pgmap v355: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:21.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:21 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[222878]: pgmap v355: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[215646]: pgmap v355: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.609 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:21 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:21.612 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:21.614 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:21.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:21.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:10:21.696 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:21.696 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:21.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:21.771 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:21.866 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:21.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:21.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:21.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:22.049 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:22.170 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:22.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:22.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:22.328 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: 1.0 scrub starts 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: 1.0 scrub ok 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3659846953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:22.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 17556/168687 objects degraded (10.407%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: 1.0 scrub starts 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: 1.0 scrub ok 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3659846953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:22.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 17556/168687 objects degraded (10.407%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: 1.0 scrub starts 2023-09-23T15:10:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: 1.0 scrub ok 2023-09-23T15:10:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3659846953' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:10:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:10:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 17556/168687 objects degraded (10.407%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:23.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:10:23.265 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:23.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:23.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:23.276 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:23.277 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:23.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:23.458 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:23.501 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:23.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:23.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:23.608 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:23.691 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:23.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:23 smithi027 ceph-mon[195233]: pgmap v356: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:23.818 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:23.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:23.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:23.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:23.979 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:23.996 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:24.115 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:24.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:23 smithi017 ceph-mon[215646]: pgmap v356: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:24.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:24.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:23 smithi017 ceph-mon[222878]: pgmap v356: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.4 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 738 op/s; 17556/168687 objects degraded (10.407%) 2023-09-23T15:10:24.221 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:24.284 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:24.422 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:24.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:24.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:24.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:24.594 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:24.678 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:24.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:24.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:24.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:24.839 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:24.971 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:24.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:24.984 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:25.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:25.169 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:25.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:10:25.290 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:25.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:25.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:25.490 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:25.540 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:10:25.540 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:10:25.540 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T15:10:25.540 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:10:25.540 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:10:25.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T15:10:25.604 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:25.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:25.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:25.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:25.832 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:25.840 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:10:25.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:25 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:10:25.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:25 smithi027 ceph-mon[195233]: 1.1 scrub ok 2023-09-23T15:10:25.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:25 smithi027 ceph-mon[195233]: pgmap v357: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 17101/161958 objects degraded (10.559%) 2023-09-23T15:10:25.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/958008327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:10:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[215646]: 1.1 scrub ok 2023-09-23T15:10:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[215646]: pgmap v357: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 17101/161958 objects degraded (10.559%) 2023-09-23T15:10:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/958008327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:10:25.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[222878]: 1.1 scrub ok 2023-09-23T15:10:25.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[222878]: pgmap v357: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 711 op/s; 17101/161958 objects degraded (10.559%) 2023-09-23T15:10:25.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/958008327' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.867 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:25.963 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:25.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:25.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:26.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:26.133 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:26.133 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:26.145 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:26.311 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:26.333 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:26.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:26.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:26.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:26.490 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:26.670 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:26.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:26.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:26.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:26.886 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: 1.9 scrub starts 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: 1.9 scrub ok 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: failed to encode map e148 with expected crc 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: failed to encode map e148 with expected crc 2023-09-23T15:10:26.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:26 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 17101/161958 objects degraded (10.559%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:27.046 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:27.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:27.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:27.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:27.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: 1.9 scrub starts 2023-09-23T15:10:27.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: 1.9 scrub ok 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: failed to encode map e148 with expected crc 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: failed to encode map e148 with expected crc 2023-09-23T15:10:27.187 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 17101/161958 objects degraded (10.559%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:27.189 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: 1.9 scrub starts 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: 1.9 scrub ok 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: osdmap e148: 8 total, 7 up, 8 in 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: failed to encode map e148 with expected crc 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: failed to encode map e148 with expected crc 2023-09-23T15:10:27.190 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:26 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 17101/161958 objects degraded (10.559%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:27.209 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:27.319 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:27.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:27.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:27.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:10:27.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:27.497 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:27.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:27.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:27.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:27.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:27.781 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:27.964 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:27.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:27.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:28.068 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:28.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:27 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:10:28.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:27 smithi027 ceph-mon[195233]: 1.5 scrub ok 2023-09-23T15:10:28.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:27 smithi027 ceph-mon[195233]: pgmap v359: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 16368/157677 objects degraded (10.381%) 2023-09-23T15:10:28.119 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:10:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[215646]: 1.5 scrub ok 2023-09-23T15:10:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[215646]: pgmap v359: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 16368/157677 objects degraded (10.381%) 2023-09-23T15:10:28.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:28] "GET /metrics HTTP/1.1" 200 36309 "" "Prometheus/2.43.0" 2023-09-23T15:10:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:10:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[222878]: 1.5 scrub ok 2023-09-23T15:10:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:27 smithi017 ceph-mon[222878]: pgmap v359: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 792 op/s; 16368/157677 objects degraded (10.381%) 2023-09-23T15:10:28.572 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:28.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:28.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:28.681 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:28.727 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:28.814 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:28.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:28.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:28.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:28.964 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:29.018 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:29.131 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:29.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:29.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:29.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[215646]: pgmap v360: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16139/154038 objects degraded (10.477%) 2023-09-23T15:10:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:29.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:29 smithi017 ceph-mon[222878]: pgmap v360: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16139/154038 objects degraded (10.477%) 2023-09-23T15:10:29.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:10:29.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:29.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:29.268 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:29.288 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:29.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:29.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:29.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:29 smithi027 ceph-mon[195233]: pgmap v360: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 3.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 16139/154038 objects degraded (10.477%) 2023-09-23T15:10:29.354 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:29.387 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:29.389 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:29.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:29.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.428 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.429 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:10:29.452 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:29.452 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:10:29.452 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:10:29.493 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:29.582 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:30.268 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:30.269 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3034993862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:30 smithi027 ceph-mon[195233]: 1.4 scrub starts 2023-09-23T15:10:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:30 smithi027 ceph-mon[195233]: 1.4 scrub ok 2023-09-23T15:10:30.389 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:30.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3034993862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[222878]: 1.4 scrub starts 2023-09-23T15:10:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[222878]: 1.4 scrub ok 2023-09-23T15:10:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3034993862' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[215646]: 1.4 scrub starts 2023-09-23T15:10:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:30 smithi017 ceph-mon[215646]: 1.4 scrub ok 2023-09-23T15:10:31.087 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:31.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:31.192 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:31 smithi027 ceph-mon[195233]: 1.8 scrub starts 2023-09-23T15:10:31.192 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:31 smithi027 ceph-mon[195233]: 1.8 scrub ok 2023-09-23T15:10:31.192 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:31 smithi027 ceph-mon[195233]: pgmap v361: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:31.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:31.232 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:31.286 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:31.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[215646]: 1.8 scrub starts 2023-09-23T15:10:31.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[215646]: 1.8 scrub ok 2023-09-23T15:10:31.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[215646]: pgmap v361: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:31.291 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[222878]: 1.8 scrub starts 2023-09-23T15:10:31.291 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[222878]: 1.8 scrub ok 2023-09-23T15:10:31.291 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:31 smithi017 ceph-mon[222878]: pgmap v361: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:31.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:31.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:31.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:31.470 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:31.578 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:31.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:31.699 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:31.752 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:31.831 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:176993 5.e 5:7083203a:::benchmark_data_smithi017_288048_object65224:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:31.813668+0000", 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.017400372000000001, 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017432335, 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:10:31.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 176993 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813668+0000", 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813668+0000", 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813667+0000", 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813671+0000", 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8240000000000001e-06 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813678+0000", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6499999999999999e-06 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.813681+0000", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.066e-06 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.817127+0000", 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034463330000000002 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.817230+0000", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103504 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.817350+0000", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011908699999999999 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.826607+0000", 2023-09-23T15:10:31.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092569890000000002 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:176990 5.a e148/114 5:5c3f7f1d:::benchmark_data_smithi017_288048_object65221:head v 148'11020, mlcod=148'11017)", 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:31.828209+0000", 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0028592829999999998, 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002991694, 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828209+0000", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828209+0000", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828208+0000", 2023-09-23T15:10:31.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828215+0000", 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0509999999999999e-06 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828231+0000", 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5970000000000001e-05 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828237+0000", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.251e-06 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828290+0000", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3251999999999998e-05 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:31.828308+0000", 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7569000000000001e-05 2023-09-23T15:10:31.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 2 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:10:31.841 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:31.845 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:31.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:32.036 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:32.139 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:32 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:10:32.139 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:32 smithi027 ceph-mon[195233]: 1.6 scrub ok 2023-09-23T15:10:32.139 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 15155/146031 objects degraded (10.378%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:32.177 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:32.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:32.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:32.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:10:32.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:32.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:10:32.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[215646]: 1.6 scrub ok 2023-09-23T15:10:32.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 15155/146031 objects degraded (10.378%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:32.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:10:32.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[222878]: 1.6 scrub ok 2023-09-23T15:10:32.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 15155/146031 objects degraded (10.378%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:32.382 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:32.519 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:32.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:32.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:32.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:32.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:32.779 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:32.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:32.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:32.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:10:32.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:32.908 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:33.003 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:33.163 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:10:33.177 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:33.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:33.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:33.366 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:33.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:33 smithi017 ceph-mon[215646]: pgmap v362: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:33.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:10:33.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:33 smithi017 ceph-mon[222878]: pgmap v362: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:33.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:10:33.485 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:33.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:33 smithi027 ceph-mon[195233]: pgmap v362: 99 pgs: 1 active+clean+scrubbing, 11 active+undersized+degraded, 26 active+undersized, 61 active+clean; 3.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 746 op/s; 15155/146031 objects degraded (10.378%) 2023-09-23T15:10:33.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:10:33.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:33.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:33.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:33.612 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:33.641 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.234 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:34 smithi027 ceph-mon[195233]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.238 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:34.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:34.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:34.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:34.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:10:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T15:10:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[215646]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2896529626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: osdmap e149: 8 total, 7 up, 8 in 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:34 smithi017 ceph-mon[222878]: failed to encode map e149 with expected crc 2023-09-23T15:10:34.445 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:34.578 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:34.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:34.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:34.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:34.736 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:34.843 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:34.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:34.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:34.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:35.052 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:35.288 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:35.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:35.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:35.303 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:35 smithi027 ceph-mon[195233]: pgmap v364: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 14675/139008 objects degraded (10.557%) 2023-09-23T15:10:35.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:35.354 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:35.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:35.402 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:35.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:35.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:35 smithi017 ceph-mon[215646]: pgmap v364: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 14675/139008 objects degraded (10.557%) 2023-09-23T15:10:35.441 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:35 smithi017 ceph-mon[222878]: pgmap v364: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s; 14675/139008 objects degraded (10.557%) 2023-09-23T15:10:35.445 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:35.474 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:35.508 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:35.526 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:35.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:35.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:10:35.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:35.701 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:35.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:35.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:35.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:35.948 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:35.948 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:35.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:10:35.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:10:35.964 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T15:10:35.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:10:35.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:36.058 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:36.174 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:36.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:36.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:36.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:36.334 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1734244218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:36.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1734244218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:36.336 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:36.368 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:36.369 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:36.421 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:36.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:36.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:36.525 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:36.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:36.568 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1734244218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:36.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:10:36.768 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:36.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:36.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:36.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:36.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:10:36.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:37.105 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:37.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:37.201 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:37 smithi027 ceph-mon[195233]: pgmap v365: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 13976/134847 objects degraded (10.364%) 2023-09-23T15:10:37.202 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 14675/139008 objects degraded (10.557%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:37.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:37.264 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:37.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:37 smithi017 ceph-mon[222878]: pgmap v365: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 13976/134847 objects degraded (10.364%) 2023-09-23T15:10:37.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 14675/139008 objects degraded (10.557%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:37.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:37 smithi017 ceph-mon[215646]: pgmap v365: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 749 op/s; 13976/134847 objects degraded (10.364%) 2023-09-23T15:10:37.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 14675/139008 objects degraded (10.557%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:37.546 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:37.738 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:37.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:37.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:37.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:37.894 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:37.958 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:37.987 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:37.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:37.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:38.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:38.153 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:38.344 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:38] "GET /metrics HTTP/1.1" 200 36312 "" "Prometheus/2.43.0" 2023-09-23T15:10:38.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:10:38.394 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:38.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:38.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:38.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:38.545 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:38.605 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:38.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:38.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:38.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:38.773 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:10:38.779 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:10:38.830 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:39.051 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:39.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:39.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:39.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:39.257 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:39.446 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:39.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:39.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:39.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:39.590 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:39 smithi027 ceph-mon[195233]: pgmap v366: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 13761/131328 objects degraded (10.478%) 2023-09-23T15:10:39.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:39 smithi027 ceph-mon[195233]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:39 smithi017 ceph-mon[215646]: pgmap v366: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 13761/131328 objects degraded (10.478%) 2023-09-23T15:10:39.591 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:39 smithi017 ceph-mon[215646]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:39 smithi017 ceph-mon[222878]: pgmap v366: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.7 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 745 op/s; 13761/131328 objects degraded (10.478%) 2023-09-23T15:10:39.592 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:39 smithi017 ceph-mon[222878]: from='client.44921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:39.603 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:39.794 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:39.794 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:40.070 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:40.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:40.076 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:40.088 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":366,"stamp":"2023-09-23T15:10:38.214705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2871942754,"num_objects":43776,"num_object_clones":0,"num_object_copies":131328,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13761,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43776,"num_whiteouts":0,"num_read":6178,"num_read_kb":13549,"num_write":180781,"num_write_kb":7166629,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":644038,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138493,"ondisk_log_size":138493,"up":260,"acting":260,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10257236,"kb_used_data":9579732,"kb_used_omap":30,"kb_used_meta":677281,"kb_avail":739703980,"statfs":{"total":767960285184,"available":757456875520,"internally_reserved":0,"allocated":9809645568,"data_stored":9805023965,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":693536344},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,5,4,7,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-495916574,"num_objects":-7570,"num_object_clones":0,"num_object_copies":-22710,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2378,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7570,"num_whiteouts":0,"num_read":-227,"num_read_kb":-420,"num_write":7463,"num_write_kb":-1098,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3,"num_bytes_recovered":-805047,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007918"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":133,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.841810+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:10:25.841810+0000","last_peered":"2023-09-23T15:10:25.841810+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:10:25.841810+0000","last_undegraded":"2023-09-23T15:10:25.841810+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":75,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839675+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:10:25.839675+0000","last_peered":"2023-09-23T15:10:25.839675+0000","last_clean":"2023-09-23T15:10:25.839675+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:10:25.839675+0000","last_undegraded":"2023-09-23T15:10:25.839675+0000","last_fullsized":"2023-09-23T15:10:25.839675+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":63,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.839085+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:10:25.839085+0000","last_peered":"2023-09-23T15:10:25.839085+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:10:25.839085+0000","last_undegraded":"2023-09-23T15:10:25.839085+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.845719+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:10:25.845719+0000","last_peered":"2023-09-23T15:10:25.845719+0000","last_clean":"2023-09-23T15:10:25.845719+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:10:25.845719+0000","last_undegraded":"2023-09-23T15:10:25.845719+0000","last_fullsized":"2023-09-23T15:10:25.845719+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":135,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.839039+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:10:25.839039+0000","last_peered":"2023-09-23T15:10:25.839039+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:10:25.839039+0000","last_undegraded":"2023-09-23T15:10:25.839039+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.861628+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:10:25.861628+0000","last_peered":"2023-09-23T15:10:25.861628+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:10:25.861628+0000","last_undegraded":"2023-09-23T15:10:25.861628+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.843967+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:10:25.843967+0000","last_peered":"2023-09-23T15:10:25.843967+0000","last_clean":"2023-09-23T15:10:25.843967+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:10:25.843967+0000","last_undegraded":"2023-09-23T15:10:25.843967+0000","last_fullsized":"2023-09-23T15:10:25.843967+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.838888+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:10:25.838888+0000","last_peered":"2023-09-23T15:10:25.838888+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:10:25.838888+0000","last_undegraded":"2023-09-23T15:10:25.838888+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":115,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.834740+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:10:25.834740+0000","last_peered":"2023-09-23T15:10:25.834740+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:10:25.834740+0000","last_undegraded":"2023-09-23T15:10:25.834740+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":60,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839030+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:10:25.839030+0000","last_peered":"2023-09-23T15:10:25.839030+0000","last_clean":"2023-09-23T15:10:25.839030+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:10:25.839030+0000","last_undegraded":"2023-09-23T15:10:25.839030+0000","last_fullsized":"2023-09-23T15:10:25.839030+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":110,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828843+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:10:25.828843+0000","last_peered":"2023-09-23T15:10:25.828843+0000","last_clean":"2023-09-23T15:10:25.828843+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:10:25.828843+0000","last_undegraded":"2023-09-23T15:10:25.828843+0000","last_fullsized":"2023-09-23T15:10:25.828843+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":59,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.863650+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:10:25.863650+0000","last_peered":"2023-09-23T15:10:25.863650+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:10:25.863650+0000","last_undegraded":"2023-09-23T15:10:25.863650+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":113,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.841505+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:10:25.841505+0000","last_peered":"2023-09-23T15:10:25.841505+0000","last_clean":"2023-09-23T15:10:25.841505+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:10:25.841505+0000","last_undegraded":"2023-09-23T15:10:25.841505+0000","last_fullsized":"2023-09-23T15:10:25.841505+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.840441+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:10:25.840441+0000","last_peered":"2023-09-23T15:10:25.840441+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:10:25.840441+0000","last_undegraded":"2023-09-23T15:10:25.840441+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":109,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828144+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:10:25.828144+0000","last_peered":"2023-09-23T15:10:25.828144+0000","last_clean":"2023-09-23T15:10:25.828144+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:10:25.828144+0000","last_undegraded":"2023-09-23T15:10:25.828144+0000","last_fullsized":"2023-09-23T15:10:25.828144+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.839486+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:10:25.839486+0000","last_peered":"2023-09-23T15:10:25.839486+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:10:25.839486+0000","last_undegraded":"2023-09-23T15:10:25.839486+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.843742+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:10:25.843742+0000","last_peered":"2023-09-23T15:10:25.843742+0000","last_clean":"2023-09-23T15:10:25.843742+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:10:25.843742+0000","last_undegraded":"2023-09-23T15:10:25.843742+0000","last_fullsized":"2023-09-23T15:10:25.843742+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":107,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.829400+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:10:25.829400+0000","last_peered":"2023-09-23T15:10:25.829400+0000","last_clean":"2023-09-23T15:10:25.829400+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:10:25.829400+0000","last_undegraded":"2023-09-23T15:10:25.829400+0000","last_fullsized":"2023-09-23T15:10:25.829400+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":90,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.832748+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:10:25.832748+0000","last_peered":"2023-09-23T15:10:25.832748+0000","last_clean":"2023-09-23T15:10:25.832748+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:10:25.832748+0000","last_undegraded":"2023-09-23T15:10:25.832748+0000","last_fullsized":"2023-09-23T15:10:25.832748+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.845515+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:10:25.845515+0000","last_peered":"2023-09-23T15:10:25.845515+0000","last_clean":"2023-09-23T15:10:25.845515+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:10:25.845515+0000","last_undegraded":"2023-09-23T15:10:25.845515+0000","last_fullsized":"2023-09-23T15:10:25.845515+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":115,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.860438+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:10:25.860438+0000","last_peered":"2023-09-23T15:10:25.860438+0000","last_clean":"2023-09-23T15:10:25.860438+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:10:25.860438+0000","last_undegraded":"2023-09-23T15:10:25.860438+0000","last_fullsized":"2023-09-23T15:10:25.860438+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.839223+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:10:25.839223+0000","last_peered":"2023-09-23T15:10:25.839223+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:10:25.839223+0000","last_undegraded":"2023-09-23T15:10:25.839223+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"149'11296","reported_seq":11338,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.153994+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:10:34.153994+0000","last_peered":"2023-09-23T15:10:34.153994+0000","last_clean":"2023-09-23T15:10:34.153994+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:10:34.153994+0000","last_undegraded":"2023-09-23T15:10:34.153994+0000","last_fullsized":"2023-09-23T15:10:34.153994+0000","mapping_epoch":114,"log_start":"127'4606","ondisk_log_start":"127'4606","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182321152,"num_objects":2782,"num_object_clones":0,"num_object_copies":8346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11296,"num_write_kb":450496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828488+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:10:25.828488+0000","last_peered":"2023-09-23T15:10:25.828488+0000","last_clean":"2023-09-23T15:10:25.828488+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:10:25.828488+0000","last_undegraded":"2023-09-23T15:10:25.828488+0000","last_fullsized":"2023-09-23T15:10:25.828488+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":90,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828616+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:10:25.828616+0000","last_peered":"2023-09-23T15:10:25.828616+0000","last_clean":"2023-09-23T15:10:25.828616+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:10:25.828616+0000","last_undegraded":"2023-09-23T15:10:25.828616+0000","last_fullsized":"2023-09-23T15:10:25.828616+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"149'11317","reported_seq":11367,"reported_epoch":149,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:38.149347+0000","last_change":"2023-09-23T15:08:51.249550+0000","last_active":"2023-09-23T15:10:38.149347+0000","last_peered":"2023-09-23T15:10:38.149347+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:08:51.249550+0000","last_became_peered":"2023-09-23T15:08:51.249550+0000","last_unstale":"2023-09-23T15:10:38.149347+0000","last_undegraded":"2023-09-23T15:08:51.245609+0000","last_fullsized":"2023-09-23T15:08:51.245485+0000","mapping_epoch":132,"log_start":"120'1300","ondisk_log_start":"120'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10017,"log_dups_size":0,"ondisk_log_size":10017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.928709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172687360,"num_objects":2635,"num_object_clones":0,"num_object_copies":7905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2635,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11317,"num_write_kb":446464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":2635}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":116,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.829441+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:10:25.829441+0000","last_peered":"2023-09-23T15:10:25.829441+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:10:25.829441+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.843726+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:10:25.843726+0000","last_peered":"2023-09-23T15:10:25.843726+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:10:25.843726+0000","last_undegraded":"2023-09-23T15:10:25.843726+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"149'11401","reported_seq":11446,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:38.129265+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:10:38.129265+0000","last_peered":"2023-09-23T15:10:38.129265+0000","last_clean":"2023-09-23T15:10:38.129265+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:10:38.129265+0000","last_undegraded":"2023-09-23T15:10:38.129265+0000","last_fullsized":"2023-09-23T15:10:38.129265+0000","mapping_epoch":114,"log_start":"120'1400","ondisk_log_start":"120'1400","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10001,"log_dups_size":0,"ondisk_log_size":10001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171638784,"num_objects":2619,"num_object_clones":0,"num_object_copies":7857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11401,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828677+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:10:25.828677+0000","last_peered":"2023-09-23T15:10:25.828677+0000","last_clean":"2023-09-23T15:10:25.828677+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:10:25.828677+0000","last_undegraded":"2023-09-23T15:10:25.828677+0000","last_fullsized":"2023-09-23T15:10:25.828677+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":113,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.861179+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:10:25.861179+0000","last_peered":"2023-09-23T15:10:25.861179+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:10:25.861179+0000","last_undegraded":"2023-09-23T15:10:25.861179+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"149'11327","reported_seq":11372,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:38.119963+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:10:38.119963+0000","last_peered":"2023-09-23T15:10:38.119963+0000","last_clean":"2023-09-23T15:10:38.119963+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:10:38.119963+0000","last_undegraded":"2023-09-23T15:10:38.119963+0000","last_fullsized":"2023-09-23T15:10:38.119963+0000","mapping_epoch":114,"log_start":"120'1300","ondisk_log_start":"120'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174260224,"num_objects":2659,"num_object_clones":0,"num_object_copies":7977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.832302+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:10:25.832302+0000","last_peered":"2023-09-23T15:10:25.832302+0000","last_clean":"2023-09-23T15:10:25.832302+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:10:25.832302+0000","last_undegraded":"2023-09-23T15:10:25.832302+0000","last_fullsized":"2023-09-23T15:10:25.832302+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":97,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.844553+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:10:25.844553+0000","last_peered":"2023-09-23T15:10:25.844553+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:10:25.844553+0000","last_undegraded":"2023-09-23T15:10:25.844553+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":508,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.832651+0000","last_change":"2023-09-23T15:10:25.167704+0000","last_active":"2023-09-23T15:10:25.832651+0000","last_peered":"2023-09-23T15:10:25.832651+0000","last_clean":"2023-09-23T15:10:25.832651+0000","last_became_active":"2023-09-23T15:09:53.859972+0000","last_became_peered":"2023-09-23T15:09:53.859972+0000","last_unstale":"2023-09-23T15:10:25.832651+0000","last_undegraded":"2023-09-23T15:10:25.832651+0000","last_fullsized":"2023-09-23T15:10:25.832651+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:25.167633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:25.167633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:36.115620+0000","scrub_duration":0.022723641999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"149'11331","reported_seq":11375,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:38.145432+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:10:38.145432+0000","last_peered":"2023-09-23T15:10:38.145432+0000","last_clean":"2023-09-23T15:10:38.145432+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:10:38.145432+0000","last_undegraded":"2023-09-23T15:10:38.145432+0000","last_fullsized":"2023-09-23T15:10:38.145432+0000","mapping_epoch":114,"log_start":"120'1300","ondisk_log_start":"120'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10031,"log_dups_size":0,"ondisk_log_size":10031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168820760,"num_objects":2577,"num_object_clones":0,"num_object_copies":7731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2577,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11331,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":109,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839149+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:10:25.839149+0000","last_peered":"2023-09-23T15:10:25.839149+0000","last_clean":"2023-09-23T15:10:25.839149+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:10:25.839149+0000","last_undegraded":"2023-09-23T15:10:25.839149+0000","last_fullsized":"2023-09-23T15:10:25.839149+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":107,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.841573+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:10:25.841573+0000","last_peered":"2023-09-23T15:10:25.841573+0000","last_clean":"2023-09-23T15:10:25.841573+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:10:25.841573+0000","last_undegraded":"2023-09-23T15:10:25.841573+0000","last_fullsized":"2023-09-23T15:10:25.841573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"91'161","reported_seq":508,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:30.199000+0000","last_change":"2023-09-23T15:10:30.198919+0000","last_active":"2023-09-23T15:10:30.199000+0000","last_peered":"2023-09-23T15:10:30.199000+0000","last_clean":"2023-09-23T15:10:30.199000+0000","last_became_active":"2023-09-23T15:09:53.868759+0000","last_became_peered":"2023-09-23T15:09:53.868759+0000","last_unstale":"2023-09-23T15:10:30.199000+0000","last_undegraded":"2023-09-23T15:10:30.199000+0000","last_fullsized":"2023-09-23T15:10:30.199000+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:30.198818+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:30.198818+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:52.349088+0000","scrub_duration":0.0099226060000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"149'11182","reported_seq":11231,"reported_epoch":149,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:34.213530+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:10:34.213530+0000","last_peered":"2023-09-23T15:10:34.213530+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:10:34.213530+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"124'3800","ondisk_log_start":"124'3800","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7382,"log_dups_size":0,"ondisk_log_size":7382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182845440,"num_objects":2790,"num_object_clones":0,"num_object_copies":8370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11182,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2790}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.860964+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:10:25.860964+0000","last_peered":"2023-09-23T15:10:25.860964+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:10:25.860964+0000","last_undegraded":"2023-09-23T15:10:25.860964+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.840962+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:10:25.840962+0000","last_peered":"2023-09-23T15:10:25.840962+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:10:25.840962+0000","last_undegraded":"2023-09-23T15:10:25.840962+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"149'11354","reported_seq":11396,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.206343+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:10:34.206343+0000","last_peered":"2023-09-23T15:10:34.206343+0000","last_clean":"2023-09-23T15:10:34.206343+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:10:34.206343+0000","last_undegraded":"2023-09-23T15:10:34.206343+0000","last_fullsized":"2023-09-23T15:10:34.206343+0000","mapping_epoch":114,"log_start":"119'1300","ondisk_log_start":"119'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187826176,"num_objects":2866,"num_object_clones":0,"num_object_copies":8598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11354,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.843528+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:10:25.843528+0000","last_peered":"2023-09-23T15:10:25.843528+0000","last_clean":"2023-09-23T15:10:25.843528+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:10:25.843528+0000","last_undegraded":"2023-09-23T15:10:25.843528+0000","last_fullsized":"2023-09-23T15:10:25.843528+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":114,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.844433+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:10:25.844433+0000","last_peered":"2023-09-23T15:10:25.844433+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:10:25.844433+0000","last_undegraded":"2023-09-23T15:10:25.844433+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":498,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.827595+0000","last_change":"2023-09-23T15:09:53.870148+0000","last_active":"2023-09-23T15:10:25.827595+0000","last_peered":"2023-09-23T15:10:25.827595+0000","last_clean":"2023-09-23T15:10:25.827595+0000","last_became_active":"2023-09-23T15:09:53.869639+0000","last_became_peered":"2023-09-23T15:09:53.869639+0000","last_unstale":"2023-09-23T15:10:25.827595+0000","last_undegraded":"2023-09-23T15:10:25.827595+0000","last_fullsized":"2023-09-23T15:10:25.827595+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"149'11065","reported_seq":11108,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.230415+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:10:34.230415+0000","last_peered":"2023-09-23T15:10:34.230415+0000","last_clean":"2023-09-23T15:10:34.230415+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:10:34.230415+0000","last_undegraded":"2023-09-23T15:10:34.230415+0000","last_fullsized":"2023-09-23T15:10:34.230415+0000","mapping_epoch":114,"log_start":"124'3700","ondisk_log_start":"124'3700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7365,"log_dups_size":0,"ondisk_log_size":7365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180682752,"num_objects":2757,"num_object_clones":0,"num_object_copies":8271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11065,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":119,"reported_epoch":149,"state":"active+undersized","last_fresh":"2023-09-23T15:10:33.172986+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:10:33.172986+0000","last_peered":"2023-09-23T15:10:33.172986+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:10:33.172986+0000","last_undegraded":"2023-09-23T15:10:33.172986+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.839433+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:10:25.839433+0000","last_peered":"2023-09-23T15:10:25.839433+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:10:25.839433+0000","last_undegraded":"2023-09-23T15:10:25.839433+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":498,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828243+0000","last_change":"2023-09-23T15:09:53.869836+0000","last_active":"2023-09-23T15:10:25.828243+0000","last_peered":"2023-09-23T15:10:25.828243+0000","last_clean":"2023-09-23T15:10:25.828243+0000","last_became_active":"2023-09-23T15:09:53.869503+0000","last_became_peered":"2023-09-23T15:09:53.869503+0000","last_unstale":"2023-09-23T15:10:25.828243+0000","last_undegraded":"2023-09-23T15:10:25.828243+0000","last_fullsized":"2023-09-23T15:10:25.828243+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":45,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":340,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.843694+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:10:25.843694+0000","last_peered":"2023-09-23T15:10:25.843694+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:10:25.843694+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":125,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.829528+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:10:25.829528+0000","last_peered":"2023-09-23T15:10:25.829528+0000","last_clean":"2023-09-23T15:10:25.829528+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:10:25.829528+0000","last_undegraded":"2023-09-23T15:10:25.829528+0000","last_fullsized":"2023-09-23T15:10:25.829528+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":68,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.836089+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:10:25.836089+0000","last_peered":"2023-09-23T15:10:25.836089+0000","last_clean":"2023-09-23T15:10:25.836089+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:10:25.836089+0000","last_undegraded":"2023-09-23T15:10:25.836089+0000","last_fullsized":"2023-09-23T15:10:25.836089+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"149'10900","reported_seq":10944,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.026891+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:10:34.026891+0000","last_peered":"2023-09-23T15:10:34.026891+0000","last_clean":"2023-09-23T15:10:34.026891+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:10:34.026891+0000","last_undegraded":"2023-09-23T15:10:34.026891+0000","last_fullsized":"2023-09-23T15:10:34.026891+0000","mapping_epoch":114,"log_start":"126'4047","ondisk_log_start":"126'4047","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176029696,"num_objects":2686,"num_object_clones":0,"num_object_copies":8058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10900,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":507,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.832901+0000","last_change":"2023-09-23T15:10:24.157411+0000","last_active":"2023-09-23T15:10:25.832901+0000","last_peered":"2023-09-23T15:10:25.832901+0000","last_clean":"2023-09-23T15:10:25.832901+0000","last_became_active":"2023-09-23T15:09:53.859571+0000","last_became_peered":"2023-09-23T15:09:53.859571+0000","last_unstale":"2023-09-23T15:10:25.832901+0000","last_undegraded":"2023-09-23T15:10:25.832901+0000","last_fullsized":"2023-09-23T15:10:25.832901+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:24.157332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:24.157332+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:54.719068+0000","scrub_duration":0.0084559189999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"149'11333","reported_seq":11383,"reported_epoch":149,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:34.036013+0000","last_change":"2023-09-23T15:08:51.256935+0000","last_active":"2023-09-23T15:10:34.036013+0000","last_peered":"2023-09-23T15:10:34.036013+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:08:51.256935+0000","last_became_peered":"2023-09-23T15:08:51.256935+0000","last_unstale":"2023-09-23T15:10:34.036013+0000","last_undegraded":"2023-09-23T15:08:51.248529+0000","last_fullsized":"2023-09-23T15:08:51.248430+0000","mapping_epoch":132,"log_start":"126'4442","ondisk_log_start":"126'4442","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6891,"log_dups_size":0,"ondisk_log_size":6891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:33.609757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184352768,"num_objects":2813,"num_object_clones":0,"num_object_copies":8439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2813,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11333,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2813}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":657,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.828915+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:10:25.828915+0000","last_peered":"2023-09-23T15:10:25.828915+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:10:25.828915+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":108,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.840358+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:10:25.840358+0000","last_peered":"2023-09-23T15:10:25.840358+0000","last_clean":"2023-09-23T15:10:25.840358+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:10:25.840358+0000","last_undegraded":"2023-09-23T15:10:25.840358+0000","last_fullsized":"2023-09-23T15:10:25.840358+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":78,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.837270+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:10:25.837270+0000","last_peered":"2023-09-23T15:10:25.837270+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:10:25.837270+0000","last_undegraded":"2023-09-23T15:10:25.837270+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":498,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.827671+0000","last_change":"2023-09-23T15:09:53.863434+0000","last_active":"2023-09-23T15:10:25.827671+0000","last_peered":"2023-09-23T15:10:25.827671+0000","last_clean":"2023-09-23T15:10:25.827671+0000","last_became_active":"2023-09-23T15:09:53.863114+0000","last_became_peered":"2023-09-23T15:09:53.863114+0000","last_unstale":"2023-09-23T15:10:25.827671+0000","last_undegraded":"2023-09-23T15:10:25.827671+0000","last_fullsized":"2023-09-23T15:10:25.827671+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"149'11331","reported_seq":11375,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:38.150245+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:10:38.150245+0000","last_peered":"2023-09-23T15:10:38.150245+0000","last_clean":"2023-09-23T15:10:38.150245+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:10:38.150245+0000","last_undegraded":"2023-09-23T15:10:38.150245+0000","last_fullsized":"2023-09-23T15:10:38.150245+0000","mapping_epoch":114,"log_start":"120'1300","ondisk_log_start":"120'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10031,"log_dups_size":0,"ondisk_log_size":10031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11331,"num_write_kb":445824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":759,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.835984+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:10:25.835984+0000","last_peered":"2023-09-23T15:10:25.835984+0000","last_clean":"2023-09-23T15:10:25.835984+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:10:25.835984+0000","last_undegraded":"2023-09-23T15:10:25.835984+0000","last_fullsized":"2023-09-23T15:10:25.835984+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":58,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.835936+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:10:25.835936+0000","last_peered":"2023-09-23T15:10:25.835936+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:10:25.835936+0000","last_undegraded":"2023-09-23T15:10:25.835936+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":83,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.829688+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:10:25.829688+0000","last_peered":"2023-09-23T15:10:25.829688+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:10:25.829688+0000","last_undegraded":"2023-09-23T15:10:25.829688+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":394,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.829174+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:10:25.829174+0000","last_peered":"2023-09-23T15:10:25.829174+0000","last_clean":"2023-09-23T15:10:25.829174+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:10:25.829174+0000","last_undegraded":"2023-09-23T15:10:25.829174+0000","last_fullsized":"2023-09-23T15:10:25.829174+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":509,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.827250+0000","last_change":"2023-09-23T15:10:21.135723+0000","last_active":"2023-09-23T15:10:25.827250+0000","last_peered":"2023-09-23T15:10:25.827250+0000","last_clean":"2023-09-23T15:10:25.827250+0000","last_became_active":"2023-09-23T15:09:53.855178+0000","last_became_peered":"2023-09-23T15:09:53.855178+0000","last_unstale":"2023-09-23T15:10:25.827250+0000","last_undegraded":"2023-09-23T15:10:25.827250+0000","last_fullsized":"2023-09-23T15:10:25.827250+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:06.813630+0000","scrub_duration":0.0070307169999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":61,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839269+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:10:25.839269+0000","last_peered":"2023-09-23T15:10:25.839269+0000","last_clean":"2023-09-23T15:10:25.839269+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:10:25.839269+0000","last_undegraded":"2023-09-23T15:10:25.839269+0000","last_fullsized":"2023-09-23T15:10:25.839269+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.863783+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:10:25.863783+0000","last_peered":"2023-09-23T15:10:25.863783+0000","last_clean":"2023-09-23T15:10:25.863783+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:10:25.863783+0000","last_undegraded":"2023-09-23T15:10:25.863783+0000","last_fullsized":"2023-09-23T15:10:25.863783+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"149'11031","reported_seq":11073,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.144240+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:10:34.144240+0000","last_peered":"2023-09-23T15:10:34.144240+0000","last_clean":"2023-09-23T15:10:34.144240+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:10:34.144240+0000","last_undegraded":"2023-09-23T15:10:34.144240+0000","last_fullsized":"2023-09-23T15:10:34.144240+0000","mapping_epoch":114,"log_start":"126'4341","ondisk_log_start":"126'4341","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6690,"log_dups_size":0,"ondisk_log_size":6690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182648832,"num_objects":2787,"num_object_clones":0,"num_object_copies":8361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11031,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":498,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828056+0000","last_change":"2023-09-23T15:09:53.869987+0000","last_active":"2023-09-23T15:10:25.828056+0000","last_peered":"2023-09-23T15:10:25.828056+0000","last_clean":"2023-09-23T15:10:25.828056+0000","last_became_active":"2023-09-23T15:09:53.869669+0000","last_became_peered":"2023-09-23T15:09:53.869669+0000","last_unstale":"2023-09-23T15:10:25.828056+0000","last_undegraded":"2023-09-23T15:10:25.828056+0000","last_fullsized":"2023-09-23T15:10:25.828056+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":46,"num_read_kb":84,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"149'11357","reported_seq":11407,"reported_epoch":149,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:34.217770+0000","last_change":"2023-09-23T15:08:51.252737+0000","last_active":"2023-09-23T15:10:34.217770+0000","last_peered":"2023-09-23T15:10:34.217770+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:08:51.252737+0000","last_became_peered":"2023-09-23T15:08:51.252737+0000","last_unstale":"2023-09-23T15:10:34.217770+0000","last_undegraded":"2023-09-23T15:08:51.248004+0000","last_fullsized":"2023-09-23T15:08:51.247871+0000","mapping_epoch":132,"log_start":"120'1300","ondisk_log_start":"120'1300","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10057,"log_dups_size":0,"ondisk_log_size":10057,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.686400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2759,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11357,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":2759}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":360,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.827420+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:10:25.827420+0000","last_peered":"2023-09-23T15:10:25.827420+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:10:25.827420+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":91,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.843467+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:10:25.843467+0000","last_peered":"2023-09-23T15:10:25.843467+0000","last_clean":"2023-09-23T15:10:25.843467+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:10:25.843467+0000","last_undegraded":"2023-09-23T15:10:25.843467+0000","last_fullsized":"2023-09-23T15:10:25.843467+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":112,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.845795+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:10:25.845795+0000","last_peered":"2023-09-23T15:10:25.845795+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:10:25.845795+0000","last_undegraded":"2023-09-23T15:10:25.845795+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":508,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:31.275273+0000","last_change":"2023-09-23T15:10:31.275188+0000","last_active":"2023-09-23T15:10:31.275273+0000","last_peered":"2023-09-23T15:10:31.275273+0000","last_clean":"2023-09-23T15:10:31.275273+0000","last_became_active":"2023-09-23T15:09:53.870279+0000","last_became_peered":"2023-09-23T15:09:53.870279+0000","last_unstale":"2023-09-23T15:10:31.275273+0000","last_undegraded":"2023-09-23T15:10:31.275273+0000","last_fullsized":"2023-09-23T15:10:31.275273+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:09.695872+0000","scrub_duration":0.115976862,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":818,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.828753+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:10:25.828753+0000","last_peered":"2023-09-23T15:10:25.828753+0000","last_clean":"2023-09-23T15:10:25.828753+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:10:25.828753+0000","last_undegraded":"2023-09-23T15:10:25.828753+0000","last_fullsized":"2023-09-23T15:10:25.828753+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":58,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.837354+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:10:25.837354+0000","last_peered":"2023-09-23T15:10:25.837354+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:10:25.837354+0000","last_undegraded":"2023-09-23T15:10:25.837354+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":76,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.844696+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:10:25.844696+0000","last_peered":"2023-09-23T15:10:25.844696+0000","last_clean":"2023-09-23T15:10:25.844696+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:10:25.844696+0000","last_undegraded":"2023-09-23T15:10:25.844696+0000","last_fullsized":"2023-09-23T15:10:25.844696+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"149'11395","reported_seq":11437,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.096432+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:10:34.096432+0000","last_peered":"2023-09-23T15:10:34.096432+0000","last_clean":"2023-09-23T15:10:34.096432+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:10:34.096432+0000","last_undegraded":"2023-09-23T15:10:34.096432+0000","last_fullsized":"2023-09-23T15:10:34.096432+0000","mapping_epoch":114,"log_start":"126'4060","ondisk_log_start":"126'4060","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7335,"log_dups_size":0,"ondisk_log_size":7335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185139200,"num_objects":2825,"num_object_clones":0,"num_object_copies":8475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11395,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":508,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:26.187190+0000","last_change":"2023-09-23T15:10:26.187120+0000","last_active":"2023-09-23T15:10:26.187190+0000","last_peered":"2023-09-23T15:10:26.187190+0000","last_clean":"2023-09-23T15:10:26.187190+0000","last_became_active":"2023-09-23T15:09:53.857395+0000","last_became_peered":"2023-09-23T15:09:53.857395+0000","last_unstale":"2023-09-23T15:10:26.187190+0000","last_undegraded":"2023-09-23T15:10:26.187190+0000","last_fullsized":"2023-09-23T15:10:26.187190+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:05.570057+0000","scrub_duration":0.0046530740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:33.173660+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:10:33.173660+0000","last_peered":"2023-09-23T15:10:33.173660+0000","last_clean":"2023-09-23T15:10:33.173660+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:10:33.173660+0000","last_undegraded":"2023-09-23T15:10:33.173660+0000","last_fullsized":"2023-09-23T15:10:33.173660+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":643,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.860213+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:10:25.860213+0000","last_peered":"2023-09-23T15:10:25.860213+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:10:25.860213+0000","last_undegraded":"2023-09-23T15:10:25.860213+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839045+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:10:25.839045+0000","last_peered":"2023-09-23T15:10:25.839045+0000","last_clean":"2023-09-23T15:10:25.839045+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:10:25.839045+0000","last_undegraded":"2023-09-23T15:10:25.839045+0000","last_fullsized":"2023-09-23T15:10:25.839045+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"149'11115","reported_seq":11158,"reported_epoch":149,"state":"active+clean","last_fresh":"2023-09-23T15:10:34.209246+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:10:34.209246+0000","last_peered":"2023-09-23T15:10:34.209246+0000","last_clean":"2023-09-23T15:10:34.209246+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:10:34.209246+0000","last_undegraded":"2023-09-23T15:10:34.209246+0000","last_fullsized":"2023-09-23T15:10:34.209246+0000","mapping_epoch":114,"log_start":"124'3700","ondisk_log_start":"124'3700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7415,"log_dups_size":0,"ondisk_log_size":7415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185663488,"num_objects":2833,"num_object_clones":0,"num_object_copies":8499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11115,"num_write_kb":446336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":508,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:29.174058+0000","last_change":"2023-09-23T15:10:29.173994+0000","last_active":"2023-09-23T15:10:29.174058+0000","last_peered":"2023-09-23T15:10:29.174058+0000","last_clean":"2023-09-23T15:10:29.174058+0000","last_became_active":"2023-09-23T15:09:53.869710+0000","last_became_peered":"2023-09-23T15:09:53.869710+0000","last_unstale":"2023-09-23T15:10:29.174058+0000","last_undegraded":"2023-09-23T15:10:29.174058+0000","last_fullsized":"2023-09-23T15:10:29.174058+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:49.304513+0000","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":579,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.861507+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:10:25.861507+0000","last_peered":"2023-09-23T15:10:25.861507+0000","last_clean":"2023-09-23T15:10:25.861507+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:10:25.861507+0000","last_undegraded":"2023-09-23T15:10:25.861507+0000","last_fullsized":"2023-09-23T15:10:25.861507+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":100,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.845642+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:10:25.845642+0000","last_peered":"2023-09-23T15:10:25.845642+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:10:25.845642+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":90,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.843649+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:10:25.843649+0000","last_peered":"2023-09-23T15:10:25.843649+0000","last_clean":"2023-09-23T15:10:25.843649+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:10:25.843649+0000","last_undegraded":"2023-09-23T15:10:25.843649+0000","last_fullsized":"2023-09-23T15:10:25.843649+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"149'11052","reported_seq":11100,"reported_epoch":149,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:34.222918+0000","last_change":"2023-09-23T15:08:51.257194+0000","last_active":"2023-09-23T15:10:34.222918+0000","last_peered":"2023-09-23T15:10:34.222918+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:08:51.257194+0000","last_became_peered":"2023-09-23T15:08:51.257194+0000","last_unstale":"2023-09-23T15:10:34.222918+0000","last_undegraded":"2023-09-23T15:08:51.246208+0000","last_fullsized":"2023-09-23T15:08:51.246087+0000","mapping_epoch":132,"log_start":"121'2200","ondisk_log_start":"121'2200","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8852,"log_dups_size":0,"ondisk_log_size":8852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.554621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180748288,"num_objects":2758,"num_object_clones":0,"num_object_copies":8274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2758,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11052,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":2758}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":79,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.844656+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:10:25.844656+0000","last_peered":"2023-09-23T15:10:25.844656+0000","last_clean":"2023-09-23T15:10:25.844656+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:10:25.844656+0000","last_undegraded":"2023-09-23T15:10:25.844656+0000","last_fullsized":"2023-09-23T15:10:25.844656+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":137,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.837215+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:10:25.837215+0000","last_peered":"2023-09-23T15:10:25.837215+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:10:25.837215+0000","last_undegraded":"2023-09-23T15:10:25.837215+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":117,"reported_epoch":148,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:25.832410+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:10:25.832410+0000","last_peered":"2023-09-23T15:10:25.832410+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:10:25.832410+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":131,"reported_epoch":148,"state":"active+undersized","last_fresh":"2023-09-23T15:10:25.840612+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:10:25.840612+0000","last_peered":"2023-09-23T15:10:25.840612+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:10:25.840612+0000","last_undegraded":"2023-09-23T15:10:25.840612+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":93,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.829626+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:10:25.829626+0000","last_peered":"2023-09-23T15:10:25.829626+0000","last_clean":"2023-09-23T15:10:25.829626+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:10:25.829626+0000","last_undegraded":"2023-09-23T15:10:25.829626+0000","last_fullsized":"2023-09-23T15:10:25.829626+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.841723+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:10:25.841723+0000","last_peered":"2023-09-23T15:10:25.841723+0000","last_clean":"2023-09-23T15:10:25.841723+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:10:25.841723+0000","last_undegraded":"2023-09-23T15:10:25.841723+0000","last_fullsized":"2023-09-23T15:10:25.841723+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":118,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.839651+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:10:25.839651+0000","last_peered":"2023-09-23T15:10:25.839651+0000","last_clean":"2023-09-23T15:10:25.839651+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:10:25.839651+0000","last_undegraded":"2023-09-23T15:10:25.839651+0000","last_fullsized":"2023-09-23T15:10:25.839651+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.838944+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:10:25.838944+0000","last_peered":"2023-09-23T15:10:25.838944+0000","last_clean":"2023-09-23T15:10:25.838944+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:10:25.838944+0000","last_undegraded":"2023-09-23T15:10:25.838944+0000","last_fullsized":"2023-09-23T15:10:25.838944+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":94,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.844033+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:10:25.844033+0000","last_peered":"2023-09-23T15:10:25.844033+0000","last_clean":"2023-09-23T15:10:25.844033+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:10:25.844033+0000","last_undegraded":"2023-09-23T15:10:25.844033+0000","last_fullsized":"2023-09-23T15:10:25.844033+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":117,"reported_epoch":148,"state":"active+clean","last_fresh":"2023-09-23T15:10:25.840052+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:10:25.840052+0000","last_peered":"2023-09-23T15:10:25.840052+0000","last_clean":"2023-09-23T15:10:25.840052+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:10:25.840052+0000","last_undegraded":"2023-09-23T15:10:25.840052+0000","last_fullsized":"2023-09-23T15:10:25.840052+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2866937880,"num_objects":43747,"num_object_clones":0,"num_object_copies":131241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13755,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43747,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":179787,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9724178432,"data_stored":9724166216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":135691,"ondisk_log_size":135691,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":805047,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":182,"num_read_kb":336,"num_write":87,"num_write_kb":879,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":644038,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201470,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":980200,"kb_used_data":913312,"kb_used_omap":0,"kb_used_meta":66880,"kb_avail":92764952,"statfs":{"total":95995035648,"available":94991310848,"internally_reserved":0,"allocated":935231488,"data_stored":934687111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":68485120},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,4,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":4.29,"5min":1.6850000000000001,"15min":0.95399999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":46.814,"5min":46.814,"15min":46.814},"last":2.5720000000000001},{"interface":"front","average":{"1min":6.2210000000000001,"5min":2.133,"15min":1.1020000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":46.966999999999999,"5min":46.966999999999999,"15min":46.966999999999999},"last":4.0620000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":10.708,"5min":4.8899999999999997,"15min":2.0409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":89.450999999999993,"5min":170.63800000000001,"15min":170.63800000000001},"last":2.6339999999999999},{"interface":"front","average":{"1min":12.073,"5min":5.1319999999999997,"15min":2.0880000000000001},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":93.789000000000001,"5min":170.47999999999999,"15min":170.47999999999999},"last":4.1420000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:10:29 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.5069999999999999,"15min":0.879},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5999999999999996,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.1150000000000002,"5min":1.4419999999999999,"15min":0.95699999999999996},"min":{"1min":0.42199999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0979999999999999,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":2.6589999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":12.598000000000001,"5min":3.2360000000000002,"15min":1.425},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":150.119,"5min":150.119,"15min":150.119},"last":2.6080000000000001},{"interface":"front","average":{"1min":13.884,"5min":3.6749999999999998,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.23999999999999999},"max":{"1min":150.596,"5min":150.596,"15min":150.596},"last":9.3699999999999992}]},{"osd":5,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":5.4809999999999999,"5min":1.9039999999999999,"15min":1.012},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":33.115000000000002,"5min":33.115000000000002,"15min":33.115000000000002},"last":0.93799999999999994},{"interface":"front","average":{"1min":4.2210000000000001,"5min":1.8819999999999999,"15min":1.0189999999999999},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":33.314,"5min":33.314,"15min":33.314},"last":2.6930000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":1.246,"15min":0.81599999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":10.063000000000001,"5min":10.063000000000001,"15min":10.063000000000001},"last":5.9850000000000003},{"interface":"front","average":{"1min":4.3819999999999997,"5min":1.873,"15min":1.0289999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":16.007999999999999,"5min":16.007999999999999,"15min":16.007999999999999},"last":4.1100000000000003}]}]},{"osd":6,"up_from":47,"seq":201863463106,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782300,"kb_used_data":720724,"kb_used_omap":0,"kb_used_meta":61568,"kb_avail":92962852,"statfs":{"total":95995035648,"available":95193960448,"internally_reserved":0,"allocated":738021376,"data_stored":737489136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":63045632},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:51 2023","interfaces":[{"interface":"back","average":{"1min":9.3209999999999997,"5min":4.8739999999999997,"15min":1.988},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":95.721000000000004,"5min":108.39400000000001,"15min":108.39400000000001},"last":1.012},{"interface":"front","average":{"1min":10.013,"5min":5.0609999999999999,"15min":2.073},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":96.019000000000005,"5min":108.452,"15min":108.452},"last":0.68000000000000005}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:09:51 2023","interfaces":[{"interface":"back","average":{"1min":13.82,"5min":10.026,"15min":3.754},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":139.17500000000001,"5min":180.374,"15min":180.374},"last":0.59199999999999997},{"interface":"front","average":{"1min":13.638999999999999,"5min":10.130000000000001,"15min":3.79},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":146.999,"5min":181.685,"15min":181.685},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.405,"15min":0.871},"min":{"1min":0.47799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7330000000000005,"5min":17.084,"15min":17.084},"last":0.71499999999999997},{"interface":"front","average":{"1min":2.044,"5min":1.179,"15min":0.80000000000000004},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.9509999999999996,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.0840000000000001,"15min":0.69999999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6879999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":5.2859999999999996},{"interface":"front","average":{"1min":2.7679999999999998,"5min":1.1910000000000001,"15min":0.751},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.25600000000000001},"max":{"1min":17.120000000000001,"5min":17.120000000000001,"15min":17.120000000000001},"last":5.2119999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":1.135,"15min":0.73699999999999999},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.6459999999999999,"5min":7.9669999999999996,"15min":7.9669999999999996},"last":5.0060000000000002},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.232,"15min":0.78600000000000003},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":17.224,"5min":17.224,"15min":17.224},"last":0.747}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":1.194,"15min":0.79400000000000004},"min":{"1min":0.33700000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":4.2610000000000001,"5min":17,"15min":17},"last":0.63900000000000001},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.135,"15min":0.80400000000000005},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2489999999999997,"5min":7.4080000000000004,"15min":7.4080000000000004},"last":5.0720000000000001}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893193,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1557664,"kb_used_data":1435736,"kb_used_omap":8,"kb_used_meta":121911,"kb_avail":92187488,"statfs":{"total":95995035648,"available":94399987712,"internally_reserved":0,"allocated":1470193664,"data_stored":1469542041,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":124837460},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9590000000000001,"5min":1.6759999999999999,"15min":0.82699999999999996},"min":{"1min":0.188,"5min":0.188,"15min":0.188},"max":{"1min":11.285,"5min":39.130000000000003,"15min":39.130000000000003},"last":0.311},{"interface":"front","average":{"1min":1.2829999999999999,"5min":1.3069999999999999,"15min":0.70899999999999996},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.798,"5min":39.034999999999997,"15min":39.034999999999997},"last":1.923}]},{"osd":3,"last update":"Sat Sep 23 15:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.242,"15min":0.72599999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.165,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":2.2959999999999998},{"interface":"front","average":{"1min":1.923,"5min":1.619,"15min":1.2110000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.3819999999999997,"5min":11.978,"15min":11.978},"last":7.883}]},{"osd":4,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.177,"5min":1.206,"15min":0.83399999999999996},"min":{"1min":0.35899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":5.5140000000000002,"5min":8.1050000000000004,"15min":8.1050000000000004},"last":2.27},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.141,"15min":0.80500000000000005},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.218,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.34799999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":1.304,"15min":0.89900000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.7130000000000001,"5min":13.49,"15min":13.49},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.7809999999999999,"5min":1.153,"15min":0.872},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.0030000000000001,"5min":8.1859999999999999,"15min":8.1859999999999999},"last":0.36099999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.849,"5min":1.24,"15min":0.879},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":11.641999999999999,"5min":11.641999999999999,"15min":11.641999999999999},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.6040000000000001,"5min":1.155,"15min":0.84099999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":5.8360000000000003,"5min":5.8360000000000003,"15min":5.8360000000000003},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.2649999999999999,"15min":0.872},"min":{"1min":0.40000000000000002,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2989999999999995,"5min":8.2989999999999995,"15min":8.2989999999999995},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.9970000000000001,"5min":1.335,"15min":0.92100000000000004},"min":{"1min":0.45000000000000001,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":6.7240000000000002,"5min":8.1760000000000002,"15min":8.1760000000000002},"last":0.39500000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566141,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1130196,"kb_used_data":1018188,"kb_used_omap":11,"kb_used_meta":111988,"kb_avail":92614956,"statfs":{"total":95995035648,"available":94837714944,"internally_reserved":0,"allocated":1042624512,"data_stored":1041982656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":114676496},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.76200000000000001,"15min":0.52100000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.7309999999999999,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":4.7460000000000004},{"interface":"front","average":{"1min":1.141,"5min":0.74199999999999999,"15min":0.504},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.7729999999999997,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":1.597}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.4330000000000001,"15min":0.73699999999999999},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.556,"5min":57.695,"15min":57.695},"last":4.7080000000000002},{"interface":"front","average":{"1min":1.258,"5min":1.6339999999999999,"15min":0.81200000000000006},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.0330000000000004,"5min":58.387999999999998,"15min":58.387999999999998},"last":1.7829999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.97699999999999998,"15min":0.75900000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.0300000000000002,"5min":7.0300000000000002,"15min":7.0300000000000002},"last":0.85199999999999998},{"interface":"front","average":{"1min":2.7480000000000002,"5min":1.2529999999999999,"15min":0.85999999999999999},"min":{"1min":0.46100000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.8360000000000003,"5min":5.8360000000000003,"15min":5.8360000000000003},"last":5.0590000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":1.3540000000000001,"15min":0.874},"min":{"1min":0.41199999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":6.6310000000000002},"last":3.4319999999999999},{"interface":"front","average":{"1min":2.379,"5min":1.232,"15min":0.86599999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.9359999999999999,"5min":6.9359999999999999,"15min":6.9359999999999999},"last":4.4560000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.2050000000000001,"15min":0.88900000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.8209999999999997,"5min":6.9790000000000001,"15min":6.9790000000000001},"last":4.6299999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":6.2220000000000004,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":4.7779999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":1.296,"15min":0.90400000000000003},"min":{"1min":0.45100000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.2680000000000007,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":2.9129999999999998},{"interface":"front","average":{"1min":1.744,"5min":1.1910000000000001,"15min":0.86199999999999999},"min":{"1min":0.41899999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.8239999999999998,"5min":5.8239999999999998,"15min":5.8239999999999998},"last":1.071}]}]},{"osd":3,"up_from":110,"seq":472446402614,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":617508,"kb_used_data":545756,"kb_used_omap":3,"kb_used_meta":71740,"kb_avail":93127644,"statfs":{"total":95995035648,"available":95362707456,"internally_reserved":0,"allocated":558854144,"data_stored":558255375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":73461929},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":2.347,"15min":1.0329999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":40.578000000000003,"5min":79.611000000000004,"15min":79.611000000000004},"last":0.26900000000000002},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.4740000000000002,"15min":1.083},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.326000000000001,"5min":79.656000000000006,"15min":79.656000000000006},"last":0.46999999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":11.795,"5min":5.8499999999999996,"15min":2.3639999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":109.077,"5min":109.077,"15min":109.077},"last":0.56699999999999995},{"interface":"front","average":{"1min":12.923,"5min":6.3159999999999998,"15min":2.387},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":119.72799999999999,"5min":119.72799999999999,"15min":119.72799999999999},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.085,"15min":0.76100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.9700000000000006,"5min":8.9700000000000006,"15min":8.9700000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.5310000000000001,"5min":1.464,"15min":0.98199999999999998},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.3989999999999991,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":1.1819999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":4.4409999999999998,"5min":1.6850000000000001,"15min":0.98699999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.960999999999999,"5min":56.960999999999999,"15min":56.960999999999999},"last":2.1699999999999999},{"interface":"front","average":{"1min":5.3239999999999998,"5min":2.0619999999999998,"15min":1.2230000000000001},"min":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":56.981999999999999,"5min":56.981999999999999,"15min":56.981999999999999},"last":2.1989999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.2310000000000001,"15min":0.85999999999999999},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.238,"5min":11.238,"15min":11.238},"last":1.008},{"interface":"front","average":{"1min":3.0670000000000002,"5min":1.7230000000000001,"15min":1.131},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.254,"5min":11.254,"15min":11.254},"last":1.2609999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.8280000000000001,"15min":1.137},"min":{"1min":0.42399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.2270000000000001},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.2210000000000001,"15min":0.84099999999999997},"min":{"1min":0.38100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.403,"5min":13.403,"15min":13.403},"last":0.53400000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986379,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1561244,"kb_used_data":1457748,"kb_used_omap":0,"kb_used_meta":103360,"kb_avail":92183908,"statfs":{"total":95995035648,"available":94396321792,"internally_reserved":0,"allocated":1492733952,"data_stored":1492201225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":105840640},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:49 2023","interfaces":[{"interface":"back","average":{"1min":7.6790000000000003,"5min":3.6749999999999998,"15min":1.583},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":108.148,"5min":122.69199999999999,"15min":122.69199999999999},"last":0.22800000000000001},{"interface":"front","average":{"1min":8.0969999999999995,"5min":3.992,"15min":1.6879999999999999},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":108.124,"5min":122.745,"15min":122.745},"last":7.665}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:09:49 2023","interfaces":[{"interface":"back","average":{"1min":10.913,"5min":3.5449999999999999,"15min":1.5760000000000001},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":109.541,"5min":109.541,"15min":109.541},"last":2.2370000000000001},{"interface":"front","average":{"1min":11.193,"5min":3.8290000000000002,"15min":1.768},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":109.619,"5min":109.619,"15min":109.619},"last":2.2690000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.468,"15min":0.86299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.48399999999999999},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.008,"15min":0.74299999999999999},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.9279999999999999,"5min":6.9279999999999999,"15min":6.9279999999999999},"last":1.9530000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:10:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.9990000000000001,"15min":1.0169999999999999},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.516,"5min":28.891999999999999,"15min":28.891999999999999},"last":2.552},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.661,"15min":0.92000000000000004},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":8.0790000000000006,"5min":32.100999999999999,"15min":32.100999999999999},"last":1.921}]},{"osd":6,"last update":"Sat Sep 23 15:10:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.304,"15min":0.82899999999999996},"min":{"1min":0.35999999999999999,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":10.509,"5min":10.509,"15min":10.509},"last":1.843},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.218,"15min":0.79500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":14.711,"5min":14.711,"15min":14.711},"last":2.0129999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":1.8340000000000001,"15min":1.016},"min":{"1min":0.309,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":18.23,"5min":18.23,"15min":18.23},"last":2.1859999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.506,"15min":0.94699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.5549999999999997,"5min":14.653,"15min":14.653},"last":1.8799999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724742,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1710548,"kb_used_data":1616332,"kb_used_omap":0,"kb_used_meta":94208,"kb_avail":92034604,"statfs":{"total":95995035648,"available":94243434496,"internally_reserved":0,"allocated":1655123968,"data_stored":1654595220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":96468992},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:50 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":1.1479999999999999,"15min":0.747},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":6.9569999999999999,"5min":24.962,"15min":24.962},"last":6.0810000000000004},{"interface":"front","average":{"1min":1.879,"5min":1.24,"15min":0.77000000000000002},"min":{"1min":0.29199999999999998,"5min":0.222,"15min":0.222},"max":{"1min":13.788,"5min":25.317,"15min":25.317},"last":6.1349999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3610000000000002,"5min":1.589,"15min":0.90400000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":15.228999999999999,"5min":19.236999999999998,"15min":19.236999999999998},"last":8.9459999999999997},{"interface":"front","average":{"1min":2.1970000000000001,"5min":1.4259999999999999,"15min":0.86799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.1980000000000004,"5min":19.699999999999999,"15min":19.699999999999999},"last":10.404999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0499999999999998,"5min":1.8069999999999999,"15min":1.0069999999999999},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.321,"5min":15.265000000000001,"15min":15.265000000000001},"last":0.307},{"interface":"front","average":{"1min":1.726,"5min":1.27,"15min":0.83999999999999997},"min":{"1min":0.33100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.6760000000000002,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.1850000000000001,"5min":0.877,"15min":0.59999999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":6.8799999999999999,"5min":6.8799999999999999,"15min":6.8799999999999999},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.9550000000000001,"5min":0.94999999999999996,"15min":0.65900000000000003},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":13.6,"5min":13.6,"15min":13.6},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:09:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4159999999999999,"5min":1.3149999999999999,"15min":0.78900000000000003},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":15.196,"5min":15.196,"15min":15.196},"last":0.45400000000000001},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.028,"15min":0.74399999999999999},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4689999999999994,"5min":9.4689999999999994,"15min":9.4689999999999994},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.048,"5min":1.919,"15min":1.034},"min":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":11.353999999999999,"5min":15.301,"15min":15.301},"last":0.35099999999999998},{"interface":"front","average":{"1min":3.8479999999999999,"5min":1.7669999999999999,"15min":0.98199999999999998},"min":{"1min":0.36199999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.095000000000001,"5min":13.361000000000001,"15min":13.361000000000001},"last":0.41699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1461583872,"data_stored":1461583872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1027149824,"data_stored":1027145752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":552796160,"data_stored":552796160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1469448192,"data_stored":1469448192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1639387136,"data_stored":1639383064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":731643904,"data_stored":731643904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":915017728,"data_stored":915013656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:10:40.095 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:10:40.095 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T15:10:40.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:40.206 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:40.344 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:40.382 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:40.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:40.395 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:40.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:40.553 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:40.721 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:40.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:10:40.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:40.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:40.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:40.969 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:41.109 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:41.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:41.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:41.254 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:41.292 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:10:41.307 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:41.312 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:41.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:41.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:41.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:41.478 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:41.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:41 smithi027 ceph-mon[195233]: pgmap v367: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:41.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2770302713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.692 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[215646]: pgmap v367: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2770302713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[222878]: pgmap v367: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:41.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2770302713' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:41.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:41.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:41.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:41.873 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:42.056 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:42.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:42.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:42.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:42.200 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:42.267 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 12809/123384 objects degraded (10.381%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:42.350 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:42.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:42.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:42.454 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:42.510 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 12809/123384 objects degraded (10.381%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: osdmap e150: 8 total, 7 up, 8 in 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: failed to encode map e150 with expected crc 2023-09-23T15:10:42.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 12809/123384 objects degraded (10.381%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:42.685 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:42.716 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:42.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:42.783 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:42.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:42.856 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:42.856 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:42.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:42.900 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:42.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.900 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.901 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:10:42.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:43.030 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:43.280 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:43.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:43.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:43.450 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:43.541 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:43.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:43 smithi027 ceph-mon[195233]: pgmap v369: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:43.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1638646661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:43.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:43.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:43.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:43.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[215646]: pgmap v369: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1638646661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[222878]: pgmap v369: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.5 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 12809/123384 objects degraded (10.381%) 2023-09-23T15:10:43.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1638646661' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:43.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:43.600 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:43.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:43.656 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:185793 5.f 5:f557b448:::benchmark_data_smithi017_288048_object74024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e150)", 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:43.651637+0000", 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0046243409999999997, 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046594169999999999, 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 185793 2023-09-23T15:10:43.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651637+0000", 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651637+0000", 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651635+0000", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651644+0000", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5830000000000007e-06 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651664+0000", 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0381e-05 2023-09-23T15:10:43.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.651672+0000", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.824e-06 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.652208+0000", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053664000000000001 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.652342+0000", 2023-09-23T15:10:43.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133118 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:43.652461+0000", 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119936 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2023-09-23T15:10:43.661 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:10:43.678 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:43.895 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:43.909 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:44.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:44.069 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:44.120 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:44.183 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:44.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:44.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:44.288 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:44.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:10:44.357 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:44.552 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:44.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:10:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:44.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:44.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:44.751 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:44.907 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:44.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:44.949 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:45.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:45.094 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:45.218 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:45.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:45.320 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 21 2023-09-23T15:10:45.346 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:45.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:45.394 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:45.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:45 smithi017 ceph-mon[215646]: pgmap v370: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12354/116676 objects degraded (10.588%) 2023-09-23T15:10:45.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T15:10:45.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:45 smithi017 ceph-mon[222878]: pgmap v370: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12354/116676 objects degraded (10.588%) 2023-09-23T15:10:45.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T15:10:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:45 smithi027 ceph-mon[195233]: pgmap v370: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.4 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 743 op/s; 12354/116676 objects degraded (10.588%) 2023-09-23T15:10:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T15:10:46.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:10:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T15:10:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T15:10:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:46 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2235970859' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: osdmap e151: 8 total, 7 up, 8 in 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: failed to encode map e151 with expected crc 2023-09-23T15:10:46.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:46 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T15:10:46.633 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:46.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:46.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:46.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:46.804 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:46.902 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:46.972 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:46.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:47.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:47.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:47.177 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:47.420 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:47.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:47.433 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:47.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: pgmap v372: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11722/112986 objects degraded (10.375%) 2023-09-23T15:10:47.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T15:10:47.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T15:10:47.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.434 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.435 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.435 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.435 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.435 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 11722/112986 objects degraded (10.375%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:47.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:47.624 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:47.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: pgmap v372: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11722/112986 objects degraded (10.375%) 2023-09-23T15:10:47.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T15:10:47.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T15:10:47.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 11722/112986 objects degraded (10.375%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: pgmap v372: 99 pgs: 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.3 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 763 op/s; 11722/112986 objects degraded (10.375%) 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: osdmap e152: 8 total, 7 up, 8 in 2023-09-23T15:10:47.690 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: failed to encode map e152 with expected crc 2023-09-23T15:10:47.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 11722/112986 objects degraded (10.375%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:47.708 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:47.708 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:10:47.708 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:10:47.708 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:10:47.709 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:10:47.762 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:47.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:47.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:47.951 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:48.090 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:48.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:48.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:48.273 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:48.369 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:48] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.43.0" 2023-09-23T15:10:48.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[215646]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[215646]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[215646]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[215646]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[222878]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[222878]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[222878]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[222878]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:48.380 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:10:48.407 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:48.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:48.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:48.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:48.549 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:48.687 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:48.687 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:48 smithi027 ceph-mon[195233]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.687 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:48 smithi027 ceph-mon[195233]: osdmap e153: 8 total, 7 up, 8 in 2023-09-23T15:10:48.687 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:48 smithi027 ceph-mon[195233]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.688 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:48 smithi027 ceph-mon[195233]: failed to encode map e153 with expected crc 2023-09-23T15:10:48.688 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:10:48.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:48.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:48.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:10:48.806 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:10:48.907 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:49.101 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:10:49.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:49.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:10:49.215 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:49.304 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:49.426 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:49.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:49.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:49.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: pgmap v375: 109 pgs: 10 unknown, 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 11506/109503 objects degraded (10.507%) 2023-09-23T15:10:49.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:49.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[215646]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T15:10:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: pgmap v375: 109 pgs: 10 unknown, 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 11506/109503 objects degraded (10.507%) 2023-09-23T15:10:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:49.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:49.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:49 smithi017 ceph-mon[222878]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T15:10:49.520 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:49.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:49.548 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:49.556 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:49.590 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:49.590 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:49.641 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:49.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:49.681 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:49.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:49.682 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:49.721 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:49.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:49.761 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.761 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:49.761 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.762 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:10:49.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:49.844 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: pgmap v375: 109 pgs: 10 unknown, 11 active+undersized+degraded, 26 active+undersized, 62 active+clean; 2.2 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 11506/109503 objects degraded (10.507%) 2023-09-23T15:10:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:10:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: failed to encode map e154 with expected crc 2023-09-23T15:10:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/141426766' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:10:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:49 smithi027 ceph-mon[195233]: osdmap e154: 8 total, 7 up, 8 in 2023-09-23T15:10:49.960 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:49.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:50.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:50.115 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:50.159 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:50.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:50.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:50.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:50.357 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:50.451 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:50.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:50.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:50.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:50.600 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:50.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:50 smithi027 ceph-mon[195233]: 1.a scrub starts 2023-09-23T15:10:50.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:50 smithi027 ceph-mon[195233]: 1.a scrub ok 2023-09-23T15:10:50.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1655414650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:50.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[215646]: 1.a scrub starts 2023-09-23T15:10:50.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[215646]: 1.a scrub ok 2023-09-23T15:10:50.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1655414650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:50.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[222878]: 1.a scrub starts 2023-09-23T15:10:50.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[222878]: 1.a scrub ok 2023-09-23T15:10:50.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1655414650' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:50.626 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:50.670 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:50.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:50.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:50.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:50.863 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:51.095 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:51.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:51.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:10:51.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:51.279 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:51.389 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:51 smithi027 ceph-mon[195233]: pgmap v377: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:51.417 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:51.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:51.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:51.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:51.636 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:51.691 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:51.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:51 smithi017 ceph-mon[215646]: pgmap v377: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:51 smithi017 ceph-mon[222878]: pgmap v377: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:51.751 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:10:51.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:10:51.752 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-09-23T15:10:51.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:10:51.840 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:51.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:51.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:51.878 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:51.879 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:51.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:52.039 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:52.113 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:52.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:52.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:52.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:52.289 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:52.333 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:52.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:52.336 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:52.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:52.465 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 10550/101652 objects degraded (10.379%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:52.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:10:52.626 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 10550/101652 objects degraded (10.379%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:52.627 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 10550/101652 objects degraded (10.379%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:52.795 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:52.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:52.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:52.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:10:52.913 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:53.005 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:53.131 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:53.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:53.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:53.204 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:53.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:53.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:53.317 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:53.557 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:53.618 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:53.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:53.627 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:53.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:53.797 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:53.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:53 smithi017 ceph-mon[215646]: pgmap v378: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:53.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:53 smithi017 ceph-mon[222878]: pgmap v378: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:53.901 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:53.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:53.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:54.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:54.071 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:53 smithi027 ceph-mon[195233]: pgmap v378: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 2.1 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 629 op/s; 10550/101652 objects degraded (10.379%) 2023-09-23T15:10:54.075 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:54.313 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:54.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:54.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:10:54.472 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:54.592 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:10:54.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:54.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:10:54.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:10:54.760 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:54.828 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:10:54.881 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:10:54.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:54.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:10:54.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:55.004 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1413487592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.004 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:54 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.006 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1413487592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.006 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:54 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.042 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1413487592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:54 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:10:55.805 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:10:55.805 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:10:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:56.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:55 smithi027 ceph-mon[195233]: pgmap v379: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10079/95016 objects degraded (10.608%) 2023-09-23T15:10:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:55 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:55 smithi027 ceph-mon[195233]: failed to encode map e155 with expected crc 2023-09-23T15:10:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:55 smithi027 ceph-mon[195233]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T15:10:56.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:56.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[215646]: pgmap v379: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10079/95016 objects degraded (10.608%) 2023-09-23T15:10:56.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:56.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[215646]: failed to encode map e155 with expected crc 2023-09-23T15:10:56.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:56.175 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[215646]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T15:10:56.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[222878]: pgmap v379: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 756 op/s; 10079/95016 objects degraded (10.608%) 2023-09-23T15:10:56.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:10:56.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[222878]: failed to encode map e155 with expected crc 2023-09-23T15:10:56.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:10:56.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:55 smithi017 ceph-mon[222878]: osdmap e155: 8 total, 7 up, 8 in 2023-09-23T15:10:56.182 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:56.289 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:56.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:56.398 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:56.440 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:56.718 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:56.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:56.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:56.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:10:56.745 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:10:56.775 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:10:56.805 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:10:56.809 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:10:56.811 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:10:56.834 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:10:56.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:56.881 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:10:56.911 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:56.924 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:10:56.968 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.969 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:10:56.970 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:10:57.050 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:57.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:56 smithi027 ceph-mon[195233]: 1.2 scrub starts 2023-09-23T15:10:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:56 smithi027 ceph-mon[195233]: 1.2 scrub ok 2023-09-23T15:10:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1105431184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:57.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:56 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 10079/95016 objects degraded (10.608%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:57.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:57.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[215646]: 1.2 scrub starts 2023-09-23T15:10:57.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[215646]: 1.2 scrub ok 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1105431184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 10079/95016 objects degraded (10.608%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[222878]: 1.2 scrub starts 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[222878]: 1.2 scrub ok 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1105431184' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:10:57.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:56 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 10079/95016 objects degraded (10.608%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:10:57.247 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:57.296 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:57.302 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:57.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:10:57.419 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:10:57.433 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:57.480 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:10:57.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:57.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:10:57.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:57.638 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:57.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:57 smithi017 ceph-mon[215646]: pgmap v381: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9419/90939 objects degraded (10.357%) 2023-09-23T15:10:57.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:57 smithi017 ceph-mon[215646]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:57 smithi017 ceph-mon[222878]: pgmap v381: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9419/90939 objects degraded (10.357%) 2023-09-23T15:10:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:57 smithi017 ceph-mon[222878]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:58.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:57 smithi027 ceph-mon[195233]: pgmap v381: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.9 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 9419/90939 objects degraded (10.357%) 2023-09-23T15:10:58.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:57 smithi027 ceph-mon[195233]: from='client.44951 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:10:58.165 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":381,"stamp":"2023-09-23T15:10:56.222451+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1989577917,"num_objects":30313,"num_object_clones":0,"num_object_copies":90939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9419,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30313,"num_whiteouts":0,"num_read":6133,"num_read_kb":13465,"num_write":194223,"num_write_kb":7166410,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":133950,"ondisk_log_size":133950,"up":290,"acting":290,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7862280,"kb_used_data":7144264,"kb_used_omap":30,"kb_used_meta":717601,"kb_avail":742098936,"statfs":{"total":767960285184,"available":759909310464,"internally_reserved":0,"allocated":7315726336,"data_stored":7311076014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":734824024},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-405590437,"num_objects":-6188,"num_object_clones":0,"num_object_copies":-18564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2087,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6188,"num_whiteouts":0,"num_read":-45,"num_read_kb":-84,"num_write":6167,"num_write_kb":-219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003225"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":138,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.365282+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:10:47.365282+0000","last_peered":"2023-09-23T15:10:47.365282+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:10:47.365282+0000","last_undegraded":"2023-09-23T15:10:47.365282+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.367776+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:10:47.367776+0000","last_peered":"2023-09-23T15:10:47.367776+0000","last_clean":"2023-09-23T15:10:47.367776+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:10:47.367776+0000","last_undegraded":"2023-09-23T15:10:47.367776+0000","last_fullsized":"2023-09-23T15:10:47.367776+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":68,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.370618+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:10:47.370618+0000","last_peered":"2023-09-23T15:10:47.370618+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:10:47.370618+0000","last_undegraded":"2023-09-23T15:10:47.370618+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":97,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.374838+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:10:47.374838+0000","last_peered":"2023-09-23T15:10:47.374838+0000","last_clean":"2023-09-23T15:10:47.374838+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:10:47.374838+0000","last_undegraded":"2023-09-23T15:10:47.374838+0000","last_fullsized":"2023-09-23T15:10:47.374838+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":140,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.367462+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:10:47.367462+0000","last_peered":"2023-09-23T15:10:47.367462+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:10:47.367462+0000","last_undegraded":"2023-09-23T15:10:47.367462+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":118,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.359902+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:10:47.359902+0000","last_peered":"2023-09-23T15:10:47.359902+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:10:47.359902+0000","last_undegraded":"2023-09-23T15:10:47.359902+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":96,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.372566+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:10:47.372566+0000","last_peered":"2023-09-23T15:10:47.372566+0000","last_clean":"2023-09-23T15:10:47.372566+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:10:47.372566+0000","last_undegraded":"2023-09-23T15:10:47.372566+0000","last_fullsized":"2023-09-23T15:10:47.372566+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":123,"reported_epoch":154,"state":"active+undersized","last_fresh":"2023-09-23T15:10:48.386632+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:10:48.386632+0000","last_peered":"2023-09-23T15:10:48.386632+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:10:48.386632+0000","last_undegraded":"2023-09-23T15:10:48.386632+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":522,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.400411+0000","last_change":"2023-09-23T15:10:47.399720+0000","last_active":"2023-09-23T15:10:47.400411+0000","last_peered":"2023-09-23T15:10:47.400411+0000","last_clean":"2023-09-23T15:10:47.400411+0000","last_became_active":"2023-09-23T15:10:47.399348+0000","last_became_peered":"2023-09-23T15:10:47.399348+0000","last_unstale":"2023-09-23T15:10:47.400411+0000","last_undegraded":"2023-09-23T15:10:47.400411+0000","last_fullsized":"2023-09-23T15:10:47.400411+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":120,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.378489+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:10:47.378489+0000","last_peered":"2023-09-23T15:10:47.378489+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:10:47.378489+0000","last_undegraded":"2023-09-23T15:10:47.378489+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":66,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:48.387133+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:10:48.387133+0000","last_peered":"2023-09-23T15:10:48.387133+0000","last_clean":"2023-09-23T15:10:48.387133+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:10:48.387133+0000","last_undegraded":"2023-09-23T15:10:48.387133+0000","last_fullsized":"2023-09-23T15:10:48.387133+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.360850+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:10:47.360850+0000","last_peered":"2023-09-23T15:10:47.360850+0000","last_clean":"2023-09-23T15:10:47.360850+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:10:47.360850+0000","last_undegraded":"2023-09-23T15:10:47.360850+0000","last_fullsized":"2023-09-23T15:10:47.360850+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.362766+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:10:47.362766+0000","last_peered":"2023-09-23T15:10:47.362766+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:10:47.362766+0000","last_undegraded":"2023-09-23T15:10:47.362766+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":118,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.363802+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:10:47.363802+0000","last_peered":"2023-09-23T15:10:47.363802+0000","last_clean":"2023-09-23T15:10:47.363802+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:10:47.363802+0000","last_undegraded":"2023-09-23T15:10:47.363802+0000","last_fullsized":"2023-09-23T15:10:47.363802+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.363218+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:10:47.363218+0000","last_peered":"2023-09-23T15:10:47.363218+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:10:47.363218+0000","last_undegraded":"2023-09-23T15:10:47.363218+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":521,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.417317+0000","last_change":"2023-09-23T15:10:47.405764+0000","last_active":"2023-09-23T15:10:47.417317+0000","last_peered":"2023-09-23T15:10:47.417317+0000","last_clean":"2023-09-23T15:10:47.417317+0000","last_became_active":"2023-09-23T15:10:47.404706+0000","last_became_peered":"2023-09-23T15:10:47.404706+0000","last_unstale":"2023-09-23T15:10:47.417317+0000","last_undegraded":"2023-09-23T15:10:47.417317+0000","last_fullsized":"2023-09-23T15:10:47.417317+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:24.157332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:24.157332+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0084559189999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":114,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.358570+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:10:47.358570+0000","last_peered":"2023-09-23T15:10:47.358570+0000","last_clean":"2023-09-23T15:10:47.358570+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:10:47.358570+0000","last_undegraded":"2023-09-23T15:10:47.358570+0000","last_fullsized":"2023-09-23T15:10:47.358570+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":154,"state":"active+undersized","last_fresh":"2023-09-23T15:10:48.387647+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:10:48.387647+0000","last_peered":"2023-09-23T15:10:48.387647+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:10:48.387647+0000","last_undegraded":"2023-09-23T15:10:48.387647+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":523,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.408785+0000","last_change":"2023-09-23T15:10:47.400462+0000","last_active":"2023-09-23T15:10:47.408785+0000","last_peered":"2023-09-23T15:10:47.408785+0000","last_clean":"2023-09-23T15:10:47.408785+0000","last_became_active":"2023-09-23T15:10:47.400102+0000","last_became_peered":"2023-09-23T15:10:47.400102+0000","last_unstale":"2023-09-23T15:10:47.408785+0000","last_undegraded":"2023-09-23T15:10:47.408785+0000","last_fullsized":"2023-09-23T15:10:47.408785+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0070307169999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":96,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.366285+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:10:47.366285+0000","last_peered":"2023-09-23T15:10:47.366285+0000","last_clean":"2023-09-23T15:10:47.366285+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:10:47.366285+0000","last_undegraded":"2023-09-23T15:10:47.366285+0000","last_fullsized":"2023-09-23T15:10:47.366285+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.378569+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:10:47.378569+0000","last_peered":"2023-09-23T15:10:47.378569+0000","last_clean":"2023-09-23T15:10:47.378569+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:10:47.378569+0000","last_undegraded":"2023-09-23T15:10:47.378569+0000","last_fullsized":"2023-09-23T15:10:47.378569+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":512,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.401514+0000","last_change":"2023-09-23T15:10:47.399759+0000","last_active":"2023-09-23T15:10:47.401514+0000","last_peered":"2023-09-23T15:10:47.401514+0000","last_clean":"2023-09-23T15:10:47.401514+0000","last_became_active":"2023-09-23T15:10:47.399393+0000","last_became_peered":"2023-09-23T15:10:47.399393+0000","last_unstale":"2023-09-23T15:10:47.401514+0000","last_undegraded":"2023-09-23T15:10:47.401514+0000","last_fullsized":"2023-09-23T15:10:47.401514+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.361323+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:10:47.361323+0000","last_peered":"2023-09-23T15:10:47.361323+0000","last_clean":"2023-09-23T15:10:47.361323+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:10:47.361323+0000","last_undegraded":"2023-09-23T15:10:47.361323+0000","last_fullsized":"2023-09-23T15:10:47.361323+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":96,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.374919+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:10:47.374919+0000","last_peered":"2023-09-23T15:10:47.374919+0000","last_clean":"2023-09-23T15:10:47.374919+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:10:47.374919+0000","last_undegraded":"2023-09-23T15:10:47.374919+0000","last_fullsized":"2023-09-23T15:10:47.374919+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":512,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.411768+0000","last_change":"2023-09-23T15:10:47.400107+0000","last_active":"2023-09-23T15:10:47.411768+0000","last_peered":"2023-09-23T15:10:47.411768+0000","last_clean":"2023-09-23T15:10:47.411768+0000","last_became_active":"2023-09-23T15:10:47.399747+0000","last_became_peered":"2023-09-23T15:10:47.399747+0000","last_unstale":"2023-09-23T15:10:47.411768+0000","last_undegraded":"2023-09-23T15:10:47.411768+0000","last_fullsized":"2023-09-23T15:10:47.411768+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":120,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.361165+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:10:47.361165+0000","last_peered":"2023-09-23T15:10:47.361165+0000","last_clean":"2023-09-23T15:10:47.361165+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:10:47.361165+0000","last_undegraded":"2023-09-23T15:10:47.361165+0000","last_fullsized":"2023-09-23T15:10:47.361165+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.367929+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:10:47.367929+0000","last_peered":"2023-09-23T15:10:47.367929+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:10:47.367929+0000","last_undegraded":"2023-09-23T15:10:47.367929+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":522,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.400934+0000","last_change":"2023-09-23T15:10:47.399819+0000","last_active":"2023-09-23T15:10:47.400934+0000","last_peered":"2023-09-23T15:10:47.400934+0000","last_clean":"2023-09-23T15:10:47.400934+0000","last_became_active":"2023-09-23T15:10:47.399322+0000","last_became_peered":"2023-09-23T15:10:47.399322+0000","last_unstale":"2023-09-23T15:10:47.400934+0000","last_undegraded":"2023-09-23T15:10:47.400934+0000","last_fullsized":"2023-09-23T15:10:47.400934+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0046530740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"154'12209","reported_seq":12256,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.180461+0000","last_change":"2023-09-23T15:07:30.131603+0000","last_active":"2023-09-23T15:10:54.180461+0000","last_peered":"2023-09-23T15:10:54.180461+0000","last_clean":"2023-09-23T15:10:54.180461+0000","last_became_active":"2023-09-23T15:07:30.131166+0000","last_became_peered":"2023-09-23T15:07:30.131166+0000","last_unstale":"2023-09-23T15:10:54.180461+0000","last_undegraded":"2023-09-23T15:10:54.180461+0000","last_fullsized":"2023-09-23T15:10:54.180461+0000","mapping_epoch":114,"log_start":"134'6686","ondisk_log_start":"134'6686","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5523,"log_dups_size":0,"ondisk_log_size":5523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:12.358580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122486784,"num_objects":1869,"num_object_clones":0,"num_object_copies":5607,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1869,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12209,"num_write_kb":450496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.364053+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:10:47.364053+0000","last_peered":"2023-09-23T15:10:47.364053+0000","last_clean":"2023-09-23T15:10:47.364053+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:10:47.364053+0000","last_undegraded":"2023-09-23T15:10:47.364053+0000","last_fullsized":"2023-09-23T15:10:47.364053+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.377918+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:10:47.377918+0000","last_peered":"2023-09-23T15:10:47.377918+0000","last_clean":"2023-09-23T15:10:47.377918+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:10:47.377918+0000","last_undegraded":"2023-09-23T15:10:47.377918+0000","last_fullsized":"2023-09-23T15:10:47.377918+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":523,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.417148+0000","last_change":"2023-09-23T15:10:47.413934+0000","last_active":"2023-09-23T15:10:47.417148+0000","last_peered":"2023-09-23T15:10:47.417148+0000","last_clean":"2023-09-23T15:10:47.417148+0000","last_became_active":"2023-09-23T15:10:47.413557+0000","last_became_peered":"2023-09-23T15:10:47.413557+0000","last_unstale":"2023-09-23T15:10:47.417148+0000","last_undegraded":"2023-09-23T15:10:47.417148+0000","last_fullsized":"2023-09-23T15:10:47.417148+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"154'12002","reported_seq":12057,"reported_epoch":154,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:53.057129+0000","last_change":"2023-09-23T15:08:51.249550+0000","last_active":"2023-09-23T15:10:53.057129+0000","last_peered":"2023-09-23T15:10:53.057129+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:08:51.249550+0000","last_became_peered":"2023-09-23T15:08:51.249550+0000","last_unstale":"2023-09-23T15:10:53.057129+0000","last_undegraded":"2023-09-23T15:08:51.245609+0000","last_fullsized":"2023-09-23T15:08:51.245485+0000","mapping_epoch":132,"log_start":"121'2000","ondisk_log_start":"121'2000","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10002,"log_dups_size":0,"ondisk_log_size":10002,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:33.928709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127795200,"num_objects":1950,"num_object_clones":0,"num_object_copies":5850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1950,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12002,"num_write_kb":446464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1950}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":121,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.360591+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:10:47.360591+0000","last_peered":"2023-09-23T15:10:47.360591+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:10:47.360591+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.362978+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:10:47.362978+0000","last_peered":"2023-09-23T15:10:47.362978+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:10:47.362978+0000","last_undegraded":"2023-09-23T15:10:47.362978+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":512,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.408347+0000","last_change":"2023-09-23T15:10:47.401819+0000","last_active":"2023-09-23T15:10:47.408347+0000","last_peered":"2023-09-23T15:10:47.408347+0000","last_clean":"2023-09-23T15:10:47.408347+0000","last_became_active":"2023-09-23T15:10:47.401463+0000","last_became_peered":"2023-09-23T15:10:47.401463+0000","last_unstale":"2023-09-23T15:10:47.408347+0000","last_undegraded":"2023-09-23T15:10:47.408347+0000","last_fullsized":"2023-09-23T15:10:47.408347+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.b","version":"154'12095","reported_seq":12145,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:53.123107+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:10:53.123107+0000","last_peered":"2023-09-23T15:10:53.123107+0000","last_clean":"2023-09-23T15:10:53.123107+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:10:53.123107+0000","last_undegraded":"2023-09-23T15:10:53.123107+0000","last_fullsized":"2023-09-23T15:10:53.123107+0000","mapping_epoch":114,"log_start":"121'2000","ondisk_log_start":"121'2000","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10095,"log_dups_size":0,"ondisk_log_size":10095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126156800,"num_objects":1925,"num_object_clones":0,"num_object_copies":5775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12095,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":96,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.360663+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:10:47.360663+0000","last_peered":"2023-09-23T15:10:47.360663+0000","last_clean":"2023-09-23T15:10:47.360663+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:10:47.360663+0000","last_undegraded":"2023-09-23T15:10:47.360663+0000","last_fullsized":"2023-09-23T15:10:47.360663+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":118,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.362476+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:10:47.362476+0000","last_peered":"2023-09-23T15:10:47.362476+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:10:47.362476+0000","last_undegraded":"2023-09-23T15:10:47.362476+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":522,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.415582+0000","last_change":"2023-09-23T15:10:47.403858+0000","last_active":"2023-09-23T15:10:47.415582+0000","last_peered":"2023-09-23T15:10:47.415582+0000","last_clean":"2023-09-23T15:10:47.415582+0000","last_became_active":"2023-09-23T15:10:47.400491+0000","last_became_peered":"2023-09-23T15:10:47.400491+0000","last_unstale":"2023-09-23T15:10:47.415582+0000","last_undegraded":"2023-09-23T15:10:47.415582+0000","last_fullsized":"2023-09-23T15:10:47.415582+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.115976862,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"154'12019","reported_seq":12069,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:53.126539+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:10:53.126539+0000","last_peered":"2023-09-23T15:10:53.126539+0000","last_clean":"2023-09-23T15:10:53.126539+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:10:53.126539+0000","last_undegraded":"2023-09-23T15:10:53.126539+0000","last_fullsized":"2023-09-23T15:10:53.126539+0000","mapping_epoch":114,"log_start":"121'2000","ondisk_log_start":"121'2000","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10019,"log_dups_size":0,"ondisk_log_size":10019,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":128909312,"num_objects":1967,"num_object_clones":0,"num_object_copies":5901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12019,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.378517+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:10:47.378517+0000","last_peered":"2023-09-23T15:10:47.378517+0000","last_clean":"2023-09-23T15:10:47.378517+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:10:47.378517+0000","last_undegraded":"2023-09-23T15:10:47.378517+0000","last_fullsized":"2023-09-23T15:10:47.378517+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":102,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.370189+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:10:47.370189+0000","last_peered":"2023-09-23T15:10:47.370189+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:10:47.370189+0000","last_undegraded":"2023-09-23T15:10:47.370189+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":513,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.358041+0000","last_change":"2023-09-23T15:10:25.167704+0000","last_active":"2023-09-23T15:10:47.358041+0000","last_peered":"2023-09-23T15:10:47.358041+0000","last_clean":"2023-09-23T15:10:47.358041+0000","last_became_active":"2023-09-23T15:09:53.859972+0000","last_became_peered":"2023-09-23T15:09:53.859972+0000","last_unstale":"2023-09-23T15:10:47.358041+0000","last_undegraded":"2023-09-23T15:10:47.358041+0000","last_fullsized":"2023-09-23T15:10:47.358041+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:25.167633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:25.167633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:36.115620+0000","scrub_duration":0.022723641999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"154'12000","reported_seq":12050,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:53.144565+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:10:53.144565+0000","last_peered":"2023-09-23T15:10:53.144565+0000","last_clean":"2023-09-23T15:10:53.144565+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:10:53.144565+0000","last_undegraded":"2023-09-23T15:10:53.144565+0000","last_fullsized":"2023-09-23T15:10:53.144565+0000","mapping_epoch":114,"log_start":"120'1900","ondisk_log_start":"120'1900","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124977176,"num_objects":1908,"num_object_clones":0,"num_object_copies":5724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1908,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12000,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.367360+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:10:47.367360+0000","last_peered":"2023-09-23T15:10:47.367360+0000","last_clean":"2023-09-23T15:10:47.367360+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:10:47.367360+0000","last_undegraded":"2023-09-23T15:10:47.367360+0000","last_fullsized":"2023-09-23T15:10:47.367360+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":112,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.363872+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:10:47.363872+0000","last_peered":"2023-09-23T15:10:47.363872+0000","last_clean":"2023-09-23T15:10:47.363872+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:10:47.363872+0000","last_undegraded":"2023-09-23T15:10:47.363872+0000","last_fullsized":"2023-09-23T15:10:47.363872+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"91'161","reported_seq":513,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.357915+0000","last_change":"2023-09-23T15:10:30.198919+0000","last_active":"2023-09-23T15:10:47.357915+0000","last_peered":"2023-09-23T15:10:47.357915+0000","last_clean":"2023-09-23T15:10:47.357915+0000","last_became_active":"2023-09-23T15:09:53.868759+0000","last_became_peered":"2023-09-23T15:09:53.868759+0000","last_unstale":"2023-09-23T15:10:47.357915+0000","last_undegraded":"2023-09-23T15:10:47.357915+0000","last_fullsized":"2023-09-23T15:10:47.357915+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:30.198818+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:30.198818+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:52.349088+0000","scrub_duration":0.0099226060000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"154'12100","reported_seq":12154,"reported_epoch":154,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:54.203874+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:10:54.203874+0000","last_peered":"2023-09-23T15:10:54.203874+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:10:54.203874+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"128'4700","ondisk_log_start":"128'4700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7400,"log_dups_size":0,"ondisk_log_size":7400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122683392,"num_objects":1872,"num_object_clones":0,"num_object_copies":5616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1872,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12100,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1872}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.362276+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:10:47.362276+0000","last_peered":"2023-09-23T15:10:47.362276+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:10:47.362276+0000","last_undegraded":"2023-09-23T15:10:47.362276+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.364100+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:10:47.364100+0000","last_peered":"2023-09-23T15:10:47.364100+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:10:47.364100+0000","last_undegraded":"2023-09-23T15:10:47.364100+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":512,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.401537+0000","last_change":"2023-09-23T15:10:47.399902+0000","last_active":"2023-09-23T15:10:47.401537+0000","last_peered":"2023-09-23T15:10:47.401537+0000","last_clean":"2023-09-23T15:10:47.401537+0000","last_became_active":"2023-09-23T15:10:47.399572+0000","last_became_peered":"2023-09-23T15:10:47.399572+0000","last_unstale":"2023-09-23T15:10:47.401537+0000","last_undegraded":"2023-09-23T15:10:47.401537+0000","last_fullsized":"2023-09-23T15:10:47.401537+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.f","version":"154'12287","reported_seq":12335,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.271611+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:10:54.271611+0000","last_peered":"2023-09-23T15:10:54.271611+0000","last_clean":"2023-09-23T15:10:54.271611+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:10:54.271611+0000","last_undegraded":"2023-09-23T15:10:54.271611+0000","last_fullsized":"2023-09-23T15:10:54.271611+0000","mapping_epoch":114,"log_start":"121'2200","ondisk_log_start":"121'2200","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10087,"log_dups_size":0,"ondisk_log_size":10087,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126681088,"num_objects":1933,"num_object_clones":0,"num_object_copies":5799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12287,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.362845+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:10:47.362845+0000","last_peered":"2023-09-23T15:10:47.362845+0000","last_clean":"2023-09-23T15:10:47.362845+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:10:47.362845+0000","last_undegraded":"2023-09-23T15:10:47.362845+0000","last_fullsized":"2023-09-23T15:10:47.362845+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.364581+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:10:47.364581+0000","last_peered":"2023-09-23T15:10:47.364581+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:10:47.364581+0000","last_undegraded":"2023-09-23T15:10:47.364581+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":513,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:49.276445+0000","last_change":"2023-09-23T15:10:49.276388+0000","last_active":"2023-09-23T15:10:49.276445+0000","last_peered":"2023-09-23T15:10:49.276445+0000","last_clean":"2023-09-23T15:10:49.276445+0000","last_became_active":"2023-09-23T15:09:53.869639+0000","last_became_peered":"2023-09-23T15:09:53.869639+0000","last_unstale":"2023-09-23T15:10:49.276445+0000","last_undegraded":"2023-09-23T15:10:49.276445+0000","last_fullsized":"2023-09-23T15:10:49.276445+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:49.276278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:49.276278+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:48.467104+0000","scrub_duration":0.021880852999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"154'11960","reported_seq":12008,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.257368+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:10:54.257368+0000","last_peered":"2023-09-23T15:10:54.257368+0000","last_clean":"2023-09-23T15:10:54.257368+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:10:54.257368+0000","last_undegraded":"2023-09-23T15:10:54.257368+0000","last_fullsized":"2023-09-23T15:10:54.257368+0000","mapping_epoch":114,"log_start":"128'4600","ondisk_log_start":"128'4600","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7360,"log_dups_size":0,"ondisk_log_size":7360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122028032,"num_objects":1862,"num_object_clones":0,"num_object_copies":5586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11960,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.367919+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:10:47.367919+0000","last_peered":"2023-09-23T15:10:47.367919+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:10:47.367919+0000","last_undegraded":"2023-09-23T15:10:47.367919+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":68,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.368133+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:10:47.368133+0000","last_peered":"2023-09-23T15:10:47.368133+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:10:47.368133+0000","last_undegraded":"2023-09-23T15:10:47.368133+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":513,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.398306+0000","last_change":"2023-09-23T15:10:47.389034+0000","last_active":"2023-09-23T15:10:47.398306+0000","last_peered":"2023-09-23T15:10:47.398306+0000","last_clean":"2023-09-23T15:10:47.398306+0000","last_became_active":"2023-09-23T15:10:47.388731+0000","last_became_peered":"2023-09-23T15:10:47.388731+0000","last_unstale":"2023-09-23T15:10:47.398306+0000","last_undegraded":"2023-09-23T15:10:47.398306+0000","last_fullsized":"2023-09-23T15:10:47.398306+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":345,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.372879+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:10:47.372879+0000","last_peered":"2023-09-23T15:10:47.372879+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:10:47.372879+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":130,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.378728+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:10:47.378728+0000","last_peered":"2023-09-23T15:10:47.378728+0000","last_clean":"2023-09-23T15:10:47.378728+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:10:47.378728+0000","last_undegraded":"2023-09-23T15:10:47.378728+0000","last_fullsized":"2023-09-23T15:10:47.378728+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":73,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.371666+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:10:47.371666+0000","last_peered":"2023-09-23T15:10:47.371666+0000","last_clean":"2023-09-23T15:10:47.371666+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:10:47.371666+0000","last_undegraded":"2023-09-23T15:10:47.371666+0000","last_fullsized":"2023-09-23T15:10:47.371666+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"154'11757","reported_seq":11805,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.067493+0000","last_change":"2023-09-23T15:07:30.130363+0000","last_active":"2023-09-23T15:10:54.067493+0000","last_peered":"2023-09-23T15:10:54.067493+0000","last_clean":"2023-09-23T15:10:54.067493+0000","last_became_active":"2023-09-23T15:07:30.130092+0000","last_became_peered":"2023-09-23T15:07:30.130092+0000","last_unstale":"2023-09-23T15:10:54.067493+0000","last_undegraded":"2023-09-23T15:10:54.067493+0000","last_fullsized":"2023-09-23T15:10:54.067493+0000","mapping_epoch":114,"log_start":"134'6174","ondisk_log_start":"134'6174","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5583,"log_dups_size":0,"ondisk_log_size":5583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:11.712148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119865344,"num_objects":1829,"num_object_clones":0,"num_object_copies":5487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11757,"num_write_kb":434752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":522,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.393428+0000","last_change":"2023-09-23T15:10:47.388261+0000","last_active":"2023-09-23T15:10:47.393428+0000","last_peered":"2023-09-23T15:10:47.393428+0000","last_clean":"2023-09-23T15:10:47.393428+0000","last_became_active":"2023-09-23T15:10:47.387868+0000","last_became_peered":"2023-09-23T15:10:47.387868+0000","last_unstale":"2023-09-23T15:10:47.393428+0000","last_undegraded":"2023-09-23T15:10:47.393428+0000","last_fullsized":"2023-09-23T15:10:47.393428+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:24.157332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:24.157332+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0084559189999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"154'12267","reported_seq":12322,"reported_epoch":154,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:54.054833+0000","last_change":"2023-09-23T15:08:51.256935+0000","last_active":"2023-09-23T15:10:54.054833+0000","last_peered":"2023-09-23T15:10:54.054833+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:08:51.256935+0000","last_became_peered":"2023-09-23T15:08:51.256935+0000","last_unstale":"2023-09-23T15:10:54.054833+0000","last_undegraded":"2023-09-23T15:08:51.248529+0000","last_fullsized":"2023-09-23T15:08:51.248430+0000","mapping_epoch":132,"log_start":"134'6647","ondisk_log_start":"134'6647","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5620,"log_dups_size":0,"ondisk_log_size":5620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:33.609757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123142144,"num_objects":1879,"num_object_clones":0,"num_object_copies":5637,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1879,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1879,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12267,"num_write_kb":452672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1879}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":662,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.380206+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:10:47.380206+0000","last_peered":"2023-09-23T15:10:47.380206+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:10:47.380206+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":113,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.363122+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:10:47.363122+0000","last_peered":"2023-09-23T15:10:47.363122+0000","last_clean":"2023-09-23T15:10:47.363122+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:10:47.363122+0000","last_undegraded":"2023-09-23T15:10:47.363122+0000","last_fullsized":"2023-09-23T15:10:47.363122+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":83,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.371959+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:10:47.371959+0000","last_peered":"2023-09-23T15:10:47.371959+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:10:47.371959+0000","last_undegraded":"2023-09-23T15:10:47.371959+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":513,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.380732+0000","last_change":"2023-09-23T15:10:47.375316+0000","last_active":"2023-09-23T15:10:47.380732+0000","last_peered":"2023-09-23T15:10:47.380732+0000","last_clean":"2023-09-23T15:10:47.380732+0000","last_became_active":"2023-09-23T15:10:47.374985+0000","last_became_peered":"2023-09-23T15:10:47.374985+0000","last_unstale":"2023-09-23T15:10:47.380732+0000","last_undegraded":"2023-09-23T15:10:47.380732+0000","last_fullsized":"2023-09-23T15:10:47.380732+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"154'12015","reported_seq":12065,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:53.113274+0000","last_change":"2023-09-23T15:07:30.124956+0000","last_active":"2023-09-23T15:10:53.113274+0000","last_peered":"2023-09-23T15:10:53.113274+0000","last_clean":"2023-09-23T15:10:53.113274+0000","last_became_active":"2023-09-23T15:07:30.124563+0000","last_became_peered":"2023-09-23T15:07:30.124563+0000","last_unstale":"2023-09-23T15:10:53.113274+0000","last_undegraded":"2023-09-23T15:10:53.113274+0000","last_fullsized":"2023-09-23T15:10:53.113274+0000","mapping_epoch":114,"log_start":"121'2000","ondisk_log_start":"121'2000","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:14.017951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125632512,"num_objects":1917,"num_object_clones":0,"num_object_copies":5751,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1917,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12015,"num_write_kb":445824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":764,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.369345+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:10:47.369345+0000","last_peered":"2023-09-23T15:10:47.369345+0000","last_clean":"2023-09-23T15:10:47.369345+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:10:47.369345+0000","last_undegraded":"2023-09-23T15:10:47.369345+0000","last_fullsized":"2023-09-23T15:10:47.369345+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":64,"reported_epoch":154,"state":"active+undersized","last_fresh":"2023-09-23T15:10:48.385886+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:10:48.385886+0000","last_peered":"2023-09-23T15:10:48.385886+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:10:48.385886+0000","last_undegraded":"2023-09-23T15:10:48.385886+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":88,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.379780+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:10:47.379780+0000","last_peered":"2023-09-23T15:10:47.379780+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:10:47.379780+0000","last_undegraded":"2023-09-23T15:10:47.379780+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":399,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.377920+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:10:47.377920+0000","last_peered":"2023-09-23T15:10:47.377920+0000","last_clean":"2023-09-23T15:10:47.377920+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:10:47.377920+0000","last_undegraded":"2023-09-23T15:10:47.377920+0000","last_fullsized":"2023-09-23T15:10:47.377920+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":524,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.380725+0000","last_change":"2023-09-23T15:10:47.376493+0000","last_active":"2023-09-23T15:10:47.380725+0000","last_peered":"2023-09-23T15:10:47.380725+0000","last_clean":"2023-09-23T15:10:47.380725+0000","last_became_active":"2023-09-23T15:10:47.376180+0000","last_became_peered":"2023-09-23T15:10:47.376180+0000","last_unstale":"2023-09-23T15:10:47.380725+0000","last_undegraded":"2023-09-23T15:10:47.380725+0000","last_fullsized":"2023-09-23T15:10:47.380725+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0070307169999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":66,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.368169+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:10:47.368169+0000","last_peered":"2023-09-23T15:10:47.368169+0000","last_clean":"2023-09-23T15:10:47.368169+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:10:47.368169+0000","last_undegraded":"2023-09-23T15:10:47.368169+0000","last_fullsized":"2023-09-23T15:10:47.368169+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":94,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.360899+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:10:47.360899+0000","last_peered":"2023-09-23T15:10:47.360899+0000","last_clean":"2023-09-23T15:10:47.360899+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:10:47.360899+0000","last_undegraded":"2023-09-23T15:10:47.360899+0000","last_fullsized":"2023-09-23T15:10:47.360899+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"154'11961","reported_seq":12008,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.170204+0000","last_change":"2023-09-23T15:07:30.130101+0000","last_active":"2023-09-23T15:10:54.170204+0000","last_peered":"2023-09-23T15:10:54.170204+0000","last_clean":"2023-09-23T15:10:54.170204+0000","last_became_active":"2023-09-23T15:07:30.129681+0000","last_became_peered":"2023-09-23T15:07:30.129681+0000","last_unstale":"2023-09-23T15:10:54.170204+0000","last_undegraded":"2023-09-23T15:10:54.170204+0000","last_fullsized":"2023-09-23T15:10:54.170204+0000","mapping_epoch":114,"log_start":"134'6452","ondisk_log_start":"134'6452","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:48.738558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121700352,"num_objects":1857,"num_object_clones":0,"num_object_copies":5571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11961,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":513,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.384784+0000","last_change":"2023-09-23T15:10:47.381204+0000","last_active":"2023-09-23T15:10:47.384784+0000","last_peered":"2023-09-23T15:10:47.384784+0000","last_clean":"2023-09-23T15:10:47.384784+0000","last_became_active":"2023-09-23T15:10:47.380916+0000","last_became_peered":"2023-09-23T15:10:47.380916+0000","last_unstale":"2023-09-23T15:10:47.384784+0000","last_undegraded":"2023-09-23T15:10:47.384784+0000","last_fullsized":"2023-09-23T15:10:47.384784+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"154'12254","reported_seq":12309,"reported_epoch":154,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:54.269471+0000","last_change":"2023-09-23T15:08:51.252737+0000","last_active":"2023-09-23T15:10:54.269471+0000","last_peered":"2023-09-23T15:10:54.269471+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:08:51.252737+0000","last_became_peered":"2023-09-23T15:08:51.252737+0000","last_unstale":"2023-09-23T15:10:54.269471+0000","last_undegraded":"2023-09-23T15:08:51.248004+0000","last_fullsized":"2023-09-23T15:08:51.247871+0000","mapping_epoch":132,"log_start":"121'2200","ondisk_log_start":"121'2200","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.686400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122028032,"num_objects":1862,"num_object_clones":0,"num_object_copies":5586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1862,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12254,"num_write_kb":451712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1862}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":365,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.357717+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:10:47.357717+0000","last_peered":"2023-09-23T15:10:47.357717+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:10:47.357717+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":97,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:48.383427+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:10:48.383427+0000","last_peered":"2023-09-23T15:10:48.383427+0000","last_clean":"2023-09-23T15:10:48.383427+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:10:48.383427+0000","last_undegraded":"2023-09-23T15:10:48.383427+0000","last_fullsized":"2023-09-23T15:10:48.383427+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":117,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.374595+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:10:47.374595+0000","last_peered":"2023-09-23T15:10:47.374595+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:10:47.374595+0000","last_undegraded":"2023-09-23T15:10:47.374595+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":523,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.380900+0000","last_change":"2023-09-23T15:10:47.379908+0000","last_active":"2023-09-23T15:10:47.380900+0000","last_peered":"2023-09-23T15:10:47.380900+0000","last_clean":"2023-09-23T15:10:47.380900+0000","last_became_active":"2023-09-23T15:10:47.379581+0000","last_became_peered":"2023-09-23T15:10:47.379581+0000","last_unstale":"2023-09-23T15:10:47.380900+0000","last_undegraded":"2023-09-23T15:10:47.380900+0000","last_fullsized":"2023-09-23T15:10:47.380900+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.115976862,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":823,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.360756+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:10:47.360756+0000","last_peered":"2023-09-23T15:10:47.360756+0000","last_clean":"2023-09-23T15:10:47.360756+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:10:47.360756+0000","last_undegraded":"2023-09-23T15:10:47.360756+0000","last_fullsized":"2023-09-23T15:10:47.360756+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":63,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.372060+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:10:47.372060+0000","last_peered":"2023-09-23T15:10:47.372060+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:10:47.372060+0000","last_undegraded":"2023-09-23T15:10:47.372060+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":82,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:48.384452+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:10:48.384452+0000","last_peered":"2023-09-23T15:10:48.384452+0000","last_clean":"2023-09-23T15:10:48.384452+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:10:48.384452+0000","last_undegraded":"2023-09-23T15:10:48.384452+0000","last_fullsized":"2023-09-23T15:10:48.384452+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"154'12301","reported_seq":12348,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.073131+0000","last_change":"2023-09-23T15:07:30.131683+0000","last_active":"2023-09-23T15:10:54.073131+0000","last_peered":"2023-09-23T15:10:54.073131+0000","last_clean":"2023-09-23T15:10:54.073131+0000","last_became_active":"2023-09-23T15:07:30.131278+0000","last_became_peered":"2023-09-23T15:07:30.131278+0000","last_unstale":"2023-09-23T15:10:54.073131+0000","last_undegraded":"2023-09-23T15:10:54.073131+0000","last_fullsized":"2023-09-23T15:10:54.073131+0000","mapping_epoch":114,"log_start":"133'6353","ondisk_log_start":"133'6353","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5948,"log_dups_size":0,"ondisk_log_size":5948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:52.716553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":125763584,"num_objects":1919,"num_object_clones":0,"num_object_copies":5757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12301,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":523,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.393408+0000","last_change":"2023-09-23T15:10:47.388729+0000","last_active":"2023-09-23T15:10:47.393408+0000","last_peered":"2023-09-23T15:10:47.393408+0000","last_clean":"2023-09-23T15:10:47.393408+0000","last_became_active":"2023-09-23T15:10:47.388390+0000","last_became_peered":"2023-09-23T15:10:47.388390+0000","last_unstale":"2023-09-23T15:10:47.393408+0000","last_undegraded":"2023-09-23T15:10:47.393408+0000","last_fullsized":"2023-09-23T15:10:47.393408+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0046530740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":115,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.368050+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:10:47.368050+0000","last_peered":"2023-09-23T15:10:47.368050+0000","last_clean":"2023-09-23T15:10:47.368050+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:10:47.368050+0000","last_undegraded":"2023-09-23T15:10:47.368050+0000","last_fullsized":"2023-09-23T15:10:47.368050+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":648,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.359230+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:10:47.359230+0000","last_peered":"2023-09-23T15:10:47.359230+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:10:47.359230+0000","last_undegraded":"2023-09-23T15:10:47.359230+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":112,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.367833+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:10:47.367833+0000","last_peered":"2023-09-23T15:10:47.367833+0000","last_clean":"2023-09-23T15:10:47.367833+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:10:47.367833+0000","last_undegraded":"2023-09-23T15:10:47.367833+0000","last_fullsized":"2023-09-23T15:10:47.367833+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"154'12063","reported_seq":12109,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:54.221906+0000","last_change":"2023-09-23T15:07:30.129155+0000","last_active":"2023-09-23T15:10:54.221906+0000","last_peered":"2023-09-23T15:10:54.221906+0000","last_clean":"2023-09-23T15:10:54.221906+0000","last_became_active":"2023-09-23T15:07:30.128821+0000","last_became_peered":"2023-09-23T15:07:30.128821+0000","last_unstale":"2023-09-23T15:10:54.221906+0000","last_undegraded":"2023-09-23T15:10:54.221906+0000","last_fullsized":"2023-09-23T15:10:54.221906+0000","mapping_epoch":114,"log_start":"128'4700","ondisk_log_start":"128'4700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7363,"log_dups_size":0,"ondisk_log_size":7363,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:34.313939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123535360,"num_objects":1885,"num_object_clones":0,"num_object_copies":5655,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1885,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12063,"num_write_kb":446336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":523,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.380795+0000","last_change":"2023-09-23T15:10:47.376390+0000","last_active":"2023-09-23T15:10:47.380795+0000","last_peered":"2023-09-23T15:10:47.380795+0000","last_clean":"2023-09-23T15:10:47.380795+0000","last_became_active":"2023-09-23T15:10:47.376025+0000","last_became_peered":"2023-09-23T15:10:47.376025+0000","last_unstale":"2023-09-23T15:10:47.380795+0000","last_undegraded":"2023-09-23T15:10:47.380795+0000","last_fullsized":"2023-09-23T15:10:47.380795+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":584,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.359707+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:10:47.359707+0000","last_peered":"2023-09-23T15:10:47.359707+0000","last_clean":"2023-09-23T15:10:47.359707+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:10:47.359707+0000","last_undegraded":"2023-09-23T15:10:47.359707+0000","last_fullsized":"2023-09-23T15:10:47.359707+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":105,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.374703+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:10:47.374703+0000","last_peered":"2023-09-23T15:10:47.374703+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:10:47.374703+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":95,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.362955+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:10:47.362955+0000","last_peered":"2023-09-23T15:10:47.362955+0000","last_clean":"2023-09-23T15:10:47.362955+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:10:47.362955+0000","last_undegraded":"2023-09-23T15:10:47.362955+0000","last_fullsized":"2023-09-23T15:10:47.362955+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"154'11960","reported_seq":12015,"reported_epoch":154,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:54.218133+0000","last_change":"2023-09-23T15:08:51.257194+0000","last_active":"2023-09-23T15:10:54.218133+0000","last_peered":"2023-09-23T15:10:54.218133+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:08:51.257194+0000","last_became_peered":"2023-09-23T15:08:51.257194+0000","last_unstale":"2023-09-23T15:10:54.218133+0000","last_undegraded":"2023-09-23T15:08:51.246208+0000","last_fullsized":"2023-09-23T15:08:51.246087+0000","mapping_epoch":132,"log_start":"123'3100","ondisk_log_start":"123'3100","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8860,"log_dups_size":0,"ondisk_log_size":8860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.554621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121241600,"num_objects":1850,"num_object_clones":0,"num_object_copies":5550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1850,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11960,"num_write_kb":441920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":1850}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":84,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.374276+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:10:47.374276+0000","last_peered":"2023-09-23T15:10:47.374276+0000","last_clean":"2023-09-23T15:10:47.374276+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:10:47.374276+0000","last_undegraded":"2023-09-23T15:10:47.374276+0000","last_fullsized":"2023-09-23T15:10:47.374276+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":143,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.371883+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:10:47.371883+0000","last_peered":"2023-09-23T15:10:47.371883+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:10:47.371883+0000","last_undegraded":"2023-09-23T15:10:47.371883+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":122,"reported_epoch":153,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:10:47.378615+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:10:47.378615+0000","last_peered":"2023-09-23T15:10:47.378615+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:10:47.378615+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":136,"reported_epoch":153,"state":"active+undersized","last_fresh":"2023-09-23T15:10:47.363388+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:10:47.363388+0000","last_peered":"2023-09-23T15:10:47.363388+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:10:47.363388+0000","last_undegraded":"2023-09-23T15:10:47.363388+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":98,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.378814+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:10:47.378814+0000","last_peered":"2023-09-23T15:10:47.378814+0000","last_clean":"2023-09-23T15:10:47.378814+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:10:47.378814+0000","last_undegraded":"2023-09-23T15:10:47.378814+0000","last_fullsized":"2023-09-23T15:10:47.378814+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.365221+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:10:47.365221+0000","last_peered":"2023-09-23T15:10:47.365221+0000","last_clean":"2023-09-23T15:10:47.365221+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:10:47.365221+0000","last_undegraded":"2023-09-23T15:10:47.365221+0000","last_fullsized":"2023-09-23T15:10:47.365221+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":123,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.367625+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:10:47.367625+0000","last_peered":"2023-09-23T15:10:47.367625+0000","last_clean":"2023-09-23T15:10:47.367625+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:10:47.367625+0000","last_undegraded":"2023-09-23T15:10:47.367625+0000","last_fullsized":"2023-09-23T15:10:47.367625+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.369910+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:10:47.369910+0000","last_peered":"2023-09-23T15:10:47.369910+0000","last_clean":"2023-09-23T15:10:47.369910+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:10:47.369910+0000","last_undegraded":"2023-09-23T15:10:47.369910+0000","last_fullsized":"2023-09-23T15:10:47.369910+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":100,"reported_epoch":154,"state":"active+clean","last_fresh":"2023-09-23T15:10:48.383797+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:10:48.383797+0000","last_peered":"2023-09-23T15:10:48.383797+0000","last_clean":"2023-09-23T15:10:48.383797+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:10:48.383797+0000","last_undegraded":"2023-09-23T15:10:48.383797+0000","last_fullsized":"2023-09-23T15:10:48.383797+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":153,"state":"active+clean","last_fresh":"2023-09-23T15:10:47.362768+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:10:47.362768+0000","last_peered":"2023-09-23T15:10:47.362768+0000","last_clean":"2023-09-23T15:10:47.362768+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:10:47.362768+0000","last_undegraded":"2023-09-23T15:10:47.362768+0000","last_fullsized":"2023-09-23T15:10:47.362768+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1984626712,"num_objects":30284,"num_object_clones":0,"num_object_copies":90852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9413,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30284,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":193250,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7234596864,"data_stored":7234584648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129538,"ondisk_log_size":129538,"up":43,"acting":43,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":751378,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":137,"num_read_kb":252,"num_write":66,"num_write_kb":660,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":483029,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201474,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":689040,"kb_used_data":618888,"kb_used_omap":0,"kb_used_meta":70144,"kb_avail":93056112,"statfs":{"total":95995035648,"available":95289458688,"internally_reserved":0,"allocated":633741312,"data_stored":633196333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":71827456},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":4.29,"5min":1.6850000000000001,"15min":0.95399999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":46.814,"5min":46.814,"15min":46.814},"last":2.3999999999999999},{"interface":"front","average":{"1min":6.2210000000000001,"5min":2.133,"15min":1.1020000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":46.966999999999999,"5min":46.966999999999999,"15min":46.966999999999999},"last":0.44900000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:09:56 2023","interfaces":[{"interface":"back","average":{"1min":10.708,"5min":4.8899999999999997,"15min":2.0409999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":89.450999999999993,"5min":170.63800000000001,"15min":170.63800000000001},"last":2.4670000000000001},{"interface":"front","average":{"1min":12.073,"5min":5.1319999999999997,"15min":2.0880000000000001},"min":{"1min":0.35999999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":93.789000000000001,"5min":170.47999999999999,"15min":170.47999999999999},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:10:29 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.5069999999999999,"15min":0.879},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5999999999999996,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":3.0019999999999998},{"interface":"front","average":{"1min":2.1150000000000002,"5min":1.4419999999999999,"15min":0.95699999999999996},"min":{"1min":0.42199999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0979999999999999,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":2.4420000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":12.598000000000001,"5min":3.2360000000000002,"15min":1.425},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":150.119,"5min":150.119,"15min":150.119},"last":2.4940000000000002},{"interface":"front","average":{"1min":13.884,"5min":3.6749999999999998,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.23999999999999999},"max":{"1min":150.596,"5min":150.596,"15min":150.596},"last":0.997}]},{"osd":5,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":5.4809999999999999,"5min":1.9039999999999999,"15min":1.012},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":33.115000000000002,"5min":33.115000000000002,"15min":33.115000000000002},"last":0.67500000000000004},{"interface":"front","average":{"1min":4.2210000000000001,"5min":1.8819999999999999,"15min":1.0189999999999999},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":33.314,"5min":33.314,"15min":33.314},"last":10.676}]},{"osd":6,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":1.246,"15min":0.81599999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":10.063000000000001,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.435},{"interface":"front","average":{"1min":4.3819999999999997,"5min":1.873,"15min":1.0289999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":16.007999999999999,"5min":16.007999999999999,"15min":16.007999999999999},"last":0.52600000000000002}]}]},{"osd":6,"up_from":47,"seq":201863463110,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":550084,"kb_used_data":489980,"kb_used_omap":0,"kb_used_meta":60096,"kb_avail":93195068,"statfs":{"total":95995035648,"available":95431749632,"internally_reserved":0,"allocated":501739520,"data_stored":501206678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":61538304},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":5.4450000000000003,"15min":2.1779999999999999},"min":{"1min":0.20799999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":22.460999999999999,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.20799999999999999},{"interface":"front","average":{"1min":3.996,"5min":5.7450000000000001,"15min":2.3010000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.831,"5min":108.452,"15min":108.452},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":10.074,"5min":11.917,"15min":4.3849999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":101.17400000000001,"5min":180.374,"15min":180.374},"last":0.39700000000000002},{"interface":"front","average":{"1min":8.8580000000000005,"5min":11.778,"15min":4.3399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":101.483,"5min":181.685,"15min":181.685},"last":0.32200000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.405,"15min":0.871},"min":{"1min":0.47799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7330000000000005,"5min":17.084,"15min":17.084},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.044,"5min":1.179,"15min":0.80000000000000004},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.9509999999999996,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.36699999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.0840000000000001,"15min":0.69999999999999996},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.6879999999999997,"5min":7.4539999999999997,"15min":7.4539999999999997},"last":0.28000000000000003},{"interface":"front","average":{"1min":2.7679999999999998,"5min":1.1910000000000001,"15min":0.751},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.25600000000000001},"max":{"1min":17.120000000000001,"5min":17.120000000000001,"15min":17.120000000000001},"last":0.49199999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:10:04 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":1.135,"15min":0.73699999999999999},"min":{"1min":0.31900000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.6459999999999999,"5min":7.9669999999999996,"15min":7.9669999999999996},"last":0.65100000000000002},{"interface":"front","average":{"1min":2.8860000000000001,"5min":1.232,"15min":0.78600000000000003},"min":{"1min":0.37,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":17.224,"5min":17.224,"15min":17.224},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":1.194,"15min":0.79400000000000004},"min":{"1min":0.33700000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":4.2610000000000001,"5min":17,"15min":17},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.135,"15min":0.80400000000000005},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2489999999999997,"5min":7.4080000000000004,"15min":7.4080000000000004},"last":0.55100000000000005}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893197,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1124048,"kb_used_data":971784,"kb_used_omap":8,"kb_used_meta":152247,"kb_avail":92621104,"statfs":{"total":95995035648,"available":94844010496,"internally_reserved":0,"allocated":995106816,"data_stored":994447431,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":155901524},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":7.657,"5min":3.1259999999999999,"15min":1.3100000000000001},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":122.20099999999999,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.23100000000000001},{"interface":"front","average":{"1min":7.8289999999999997,"5min":2.7909999999999999,"15min":1.2030000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":0.21299999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.242,"15min":0.72599999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.165,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.923,"5min":1.619,"15min":1.2110000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.3819999999999997,"5min":11.978,"15min":11.978},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":3.496,"5min":1.7749999999999999,"15min":1.024},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.309},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.2410000000000001,"15min":0.83799999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5389999999999997,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":1.615,"15min":1.0029999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2050000000000001,"5min":13.49,"15min":13.49},"last":6.3410000000000002},{"interface":"front","average":{"1min":3.0750000000000002,"5min":1.6220000000000001,"15min":1.028},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.929,"5min":11.929,"15min":11.929},"last":6.2850000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":1.5609999999999999,"15min":0.98599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.292999999999999,"5min":11.641999999999999,"15min":11.641999999999999},"last":6.5149999999999997},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.4810000000000001,"15min":0.94999999999999996},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":6.3940000000000001},"last":6.3940000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.55,"15min":0.96699999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.0739999999999998,"5min":8.2989999999999995,"15min":8.2989999999999995},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.675,"15min":1.034},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.109,"5min":9.109,"15min":9.109},"last":0.38200000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566144,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864136,"kb_used_data":752128,"kb_used_omap":11,"kb_used_meta":111988,"kb_avail":92881016,"statfs":{"total":95995035648,"available":95110160384,"internally_reserved":0,"allocated":770179072,"data_stored":769526059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":114676496},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":0.76200000000000001,"15min":0.52100000000000002},"min":{"1min":0.19400000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":4.7309999999999999,"5min":6.6189999999999998,"15min":6.6189999999999998},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.141,"5min":0.74199999999999999,"15min":0.504},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":4.7729999999999997,"5min":7.7149999999999999,"15min":7.7149999999999999},"last":0.31}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.4330000000000001,"15min":0.73699999999999999},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.556,"5min":57.695,"15min":57.695},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.258,"5min":1.6339999999999999,"15min":0.81200000000000006},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.0330000000000004,"5min":58.387999999999998,"15min":58.387999999999998},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":1.3049999999999999,"5min":0.97699999999999998,"15min":0.75900000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.0300000000000002,"5min":7.0300000000000002,"15min":7.0300000000000002},"last":0.35399999999999998},{"interface":"front","average":{"1min":2.7480000000000002,"5min":1.2529999999999999,"15min":0.85999999999999999},"min":{"1min":0.46100000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":5.8360000000000003,"5min":5.8360000000000003,"15min":5.8360000000000003},"last":9.032}]},{"osd":5,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.2589999999999999,"5min":1.3540000000000001,"15min":0.874},"min":{"1min":0.41199999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6310000000000002,"5min":6.6310000000000002,"15min":6.6310000000000002},"last":3.1629999999999998},{"interface":"front","average":{"1min":2.379,"5min":1.232,"15min":0.86599999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.9359999999999999,"5min":6.9359999999999999,"15min":6.9359999999999999},"last":3.2250000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.0329999999999999,"5min":1.2050000000000001,"15min":0.88900000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.8209999999999997,"5min":6.9790000000000001,"15min":6.9790000000000001},"last":3.012},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.272,"15min":0.89700000000000002},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":6.2220000000000004,"5min":6.7720000000000002,"15min":6.7720000000000002},"last":2.9929999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:09:53 2023","interfaces":[{"interface":"back","average":{"1min":2.7509999999999999,"5min":1.296,"15min":0.90400000000000003},"min":{"1min":0.45100000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":9.2680000000000007,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":2.8610000000000002},{"interface":"front","average":{"1min":1.744,"5min":1.1910000000000001,"15min":0.86199999999999999},"min":{"1min":0.41899999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":5.8239999999999998,"5min":5.8239999999999998,"15min":5.8239999999999998},"last":8.7319999999999993}]}]},{"osd":3,"up_from":110,"seq":472446402618,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":433108,"kb_used_data":365452,"kb_used_omap":3,"kb_used_meta":67644,"kb_avail":93312044,"statfs":{"total":95995035648,"available":95551533056,"internally_reserved":0,"allocated":374222848,"data_stored":373616687,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":69267625},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":2.347,"15min":1.0329999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":40.578000000000003,"5min":79.611000000000004,"15min":79.611000000000004},"last":5.266},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.4740000000000002,"15min":1.083},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.326000000000001,"5min":79.656000000000006,"15min":79.656000000000006},"last":5.3899999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":11.795,"5min":5.8499999999999996,"15min":2.3639999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":109.077,"5min":109.077,"15min":109.077},"last":0.42899999999999999},{"interface":"front","average":{"1min":12.923,"5min":6.3159999999999998,"15min":2.387},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":119.72799999999999,"5min":119.72799999999999,"15min":119.72799999999999},"last":7.5499999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.085,"15min":0.76100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.9700000000000006,"5min":8.9700000000000006,"15min":8.9700000000000006},"last":5.4740000000000002},{"interface":"front","average":{"1min":2.5310000000000001,"5min":1.464,"15min":0.98199999999999998},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.3989999999999991,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":7.2249999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":4.4409999999999998,"5min":1.6850000000000001,"15min":0.98699999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.960999999999999,"5min":56.960999999999999,"15min":56.960999999999999},"last":5.4400000000000004},{"interface":"front","average":{"1min":5.3239999999999998,"5min":2.0619999999999998,"15min":1.2230000000000001},"min":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":56.981999999999999,"5min":56.981999999999999,"15min":56.981999999999999},"last":7.2619999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.2310000000000001,"15min":0.85999999999999999},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.238,"5min":11.238,"15min":11.238},"last":5.5010000000000003},{"interface":"front","average":{"1min":3.0670000000000002,"5min":1.7230000000000001,"15min":1.131},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.254,"5min":11.254,"15min":11.254},"last":8.6259999999999994}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.8280000000000001,"15min":1.137},"min":{"1min":0.42399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":8.1899999999999995},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.2210000000000001,"15min":0.84099999999999997},"min":{"1min":0.38100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.403,"5min":13.403,"15min":13.403},"last":0.60999999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986383,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1084292,"kb_used_data":982460,"kb_used_omap":0,"kb_used_meta":101568,"kb_avail":92660860,"statfs":{"total":95995035648,"available":94884720640,"internally_reserved":0,"allocated":1006039040,"data_stored":1005505711,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":104005632},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:49 2023","interfaces":[{"interface":"back","average":{"1min":5.867,"5min":4.7409999999999997,"15min":1.9379999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.605999999999995,"5min":122.69199999999999,"15min":122.69199999999999},"last":7.7910000000000004},{"interface":"front","average":{"1min":5.7619999999999996,"5min":5.0380000000000003,"15min":2.036},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":66.384,"5min":122.745,"15min":122.745},"last":0.25600000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:10:49 2023","interfaces":[{"interface":"back","average":{"1min":5.9100000000000001,"5min":4.6079999999999997,"15min":1.9299999999999999},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":69.275000000000006,"5min":109.541,"15min":109.541},"last":0.309},{"interface":"front","average":{"1min":6.29,"5min":4.9390000000000001,"15min":2.1379999999999999},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":73.986000000000004,"5min":109.619,"15min":109.619},"last":0.34200000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.468,"15min":0.86299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":7.7480000000000002},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.008,"15min":0.74299999999999999},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.9279999999999999,"5min":6.9279999999999999,"15min":6.9279999999999999},"last":0.79700000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:10:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.9990000000000001,"15min":1.0169999999999999},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.516,"5min":28.891999999999999,"15min":28.891999999999999},"last":11.069000000000001},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.661,"15min":0.92000000000000004},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":8.0790000000000006,"5min":32.100999999999999,"15min":32.100999999999999},"last":1.5669999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:10:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.304,"15min":0.82899999999999996},"min":{"1min":0.35999999999999999,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":10.509,"5min":10.509,"15min":10.509},"last":8.0909999999999993},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.218,"15min":0.79500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":14.711,"5min":14.711,"15min":14.711},"last":1.472}]},{"osd":7,"last update":"Sat Sep 23 15:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":1.8340000000000001,"15min":1.016},"min":{"1min":0.309,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":18.23,"5min":18.23,"15min":18.23},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.506,"15min":0.94699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.5549999999999997,"5min":14.653,"15min":14.653},"last":11.099}]}]},{"osd":5,"up_from":39,"seq":167503724746,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199996,"kb_used_data":1091636,"kb_used_omap":0,"kb_used_meta":108288,"kb_avail":92545156,"statfs":{"total":95995035648,"available":94766239744,"internally_reserved":0,"allocated":1117835264,"data_stored":1117305914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":110886912},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":1.7350000000000001,"15min":0.94299999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.044,"5min":24.962,"15min":24.962},"last":1.512},{"interface":"front","average":{"1min":2.6949999999999998,"5min":1.671,"15min":0.91400000000000003},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":14.529,"5min":25.317,"15min":25.317},"last":1.145}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":10.397,"5min":3.556,"15min":1.5600000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":97.087999999999994,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.39500000000000002},{"interface":"front","average":{"1min":10.038,"5min":3.3159999999999998,"15min":1.498},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":97.305999999999997,"5min":97.305999999999997,"15min":97.305999999999997},"last":1.1890000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0499999999999998,"5min":1.8069999999999999,"15min":1.0069999999999999},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.321,"5min":15.265000000000001,"15min":15.265000000000001},"last":3.5600000000000001},{"interface":"front","average":{"1min":1.726,"5min":1.27,"15min":0.83999999999999997},"min":{"1min":0.33100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.6760000000000002,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":1.456}]},{"osd":4,"last update":"Sat Sep 23 15:10:38 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":1.4099999999999999,"15min":0.78500000000000003},"min":{"1min":0.27700000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":11.539999999999999,"5min":11.539999999999999,"15min":11.539999999999999},"last":1.4870000000000001},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.409,"15min":0.82099999999999995},"min":{"1min":0.29099999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.385999999999999,"5min":13.6,"15min":13.6},"last":4.3150000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:09:57 2023","interfaces":[{"interface":"back","average":{"1min":3.4159999999999999,"5min":1.3149999999999999,"15min":0.78900000000000003},"min":{"1min":0.32800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":15.196,"5min":15.196,"15min":15.196},"last":1.716},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.028,"15min":0.74399999999999999},"min":{"1min":0.32600000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":9.4689999999999994,"5min":9.4689999999999994,"15min":9.4689999999999994},"last":4.2809999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.048,"5min":1.919,"15min":1.034},"min":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":11.353999999999999,"5min":15.301,"15min":15.301},"last":0.36499999999999999},{"interface":"front","average":{"1min":3.8479999999999999,"5min":1.7669999999999999,"15min":0.98199999999999998},"min":{"1min":0.36199999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.095000000000001,"5min":13.361000000000001,"15min":13.361000000000001},"last":1.5389999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":985202688,"data_stored":985202688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":759173120,"data_stored":759169048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":369885184,"data_stored":369885184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":986644480,"data_stored":986644480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1100091392,"data_stored":1100087320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":495452160,"data_stored":495452160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":610996224,"data_stored":610992152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:10:58.173 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:10:58.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 26 2023-09-23T15:10:58.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:10:58.228 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:58.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:58.277 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:10:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:10:58] "GET /metrics HTTP/1.1" 200 36307 "" "Prometheus/2.43.0" 2023-09-23T15:10:58.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:10:58.363 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:58.385 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:10:58.385 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:10:58.412 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:10:58.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:58.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:10:58.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:58.574 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:58.662 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:10:58.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:58.675 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:10:58.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:10:58.782 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:58.833 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:10:58.862 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:59.020 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:10:59.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:10:59.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:10:59.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:10:59.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:10:59.315 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:59.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:59.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:59 smithi027 ceph-mon[195233]: pgmap v382: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 9174/87441 objects degraded (10.492%) 2023-09-23T15:10:59.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:59 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:10:59.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:10:59 smithi027 ceph-mon[195233]: 1.1 scrub ok 2023-09-23T15:10:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[215646]: pgmap v382: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 9174/87441 objects degraded (10.492%) 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[215646]: 1.1 scrub ok 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[222878]: pgmap v382: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.8 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 744 op/s; 9174/87441 objects degraded (10.492%) 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:10:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:10:59 smithi017 ceph-mon[222878]: 1.1 scrub ok 2023-09-23T15:10:59.574 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:10:59.574 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:10:59.574 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:10:59.574 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:10:59.574 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:11:00.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:11:00.623 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:11:00.676 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:00.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:00.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:00.857 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:01.038 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:01.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:01.149 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:01.209 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:01.319 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:01.335 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:01.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:01.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:01.381 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:11:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:01 smithi017 ceph-mon[222878]: pgmap v383: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:01.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:01.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:01 smithi017 ceph-mon[215646]: pgmap v383: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:01.491 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:01 smithi027 ceph-mon[195233]: pgmap v383: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:01.502 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:01.660 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:01.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:01.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:01.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:01.841 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:02.040 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:02.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:02.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:02.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:02.251 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:02.324 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:11:02.325 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 5 pg_num to 26 2023-09-23T15:11:02.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3085727417' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 8259/79950 objects degraded (10.330%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3682296205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3085727417' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 8259/79950 objects degraded (10.330%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:02.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3682296205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:02.351 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:02 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:02.399 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:02.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:02.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:02.531 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3085727417' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:02 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T15:11:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 8259/79950 objects degraded (10.330%), 11 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3682296205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:02 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: pgmap v384: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.291 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.292 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:03 smithi027 ceph-mon[195233]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: pgmap v384: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:03.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:03.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T15:11:03.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:03.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[222878]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: pgmap v384: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.6 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 719 op/s; 8259/79950 objects degraded (10.330%) 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "26"}]': finished 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:03.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: osdmap e156: 8 total, 7 up, 8 in 2023-09-23T15:11:03.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:03 smithi017 ceph-mon[215646]: failed to encode map e156 with expected crc 2023-09-23T15:11:03.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:03.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:03.620 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:03.718 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:7549 5.2 5:45103a7d:::benchmark_data_smithi017_288048_object7548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:40.425193+0000", 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 203.28953170700001, 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17770498500000001, 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.719 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 7549 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425193+0000", 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425193+0000", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425192+0000", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425221+0000", 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9343999999999999e-05 2023-09-23T15:11:03.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425236+0000", 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4557e-05 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425243+0000", 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7950000000000002e-06 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425281+0000", 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8389999999999997e-05 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425407+0000", 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012632 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.425553+0000", 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000145711 2023-09-23T15:11:03.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.426153+0000", 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00060032900000000003 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.427028+0000", 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00087471700000000001 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.602801+0000", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17577283599999999 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.602857+0000", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6651999999999999e-05 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.602898+0000", 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0556000000000003e-05 2023-09-23T15:11:03.725 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:7560 5.2 5:4ebcb1d7:::benchmark_data_smithi017_288048_object7559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:40.428956+0000", 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 203.28576902, 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17459664599999999, 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.726 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 7560 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.428956+0000", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.428956+0000", 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.428954+0000", 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429008+0000", 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3659999999999998e-05 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429025+0000", 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7118000000000001e-05 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429032+0000", 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429081+0000", 2023-09-23T15:11:03.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9237999999999999e-05 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429236+0000", 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015488599999999999 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.429398+0000", 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016204899999999999 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.430125+0000", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00072677899999999999 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.430373+0000", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000248061 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.603450+0000", 2023-09-23T15:11:03.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17307683800000001 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.603508+0000", 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8406000000000003e-05 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:40.603552+0000", 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4081e-05 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.732 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:10057 5.2 5:49e47c4a:::benchmark_data_smithi017_288048_object10056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:41.545163+0000", 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.16956138200001, 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16156737500000001, 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.733 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 10057 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545163+0000", 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545163+0000", 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545163+0000", 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545191+0000", 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7931000000000001e-05 2023-09-23T15:11:03.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545204+0000", 2023-09-23T15:11:03.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2819999999999999e-05 2023-09-23T15:11:03.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.545209+0000", 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3129999999999998e-06 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.546258+0000", 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010493259999999999 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.546363+0000", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104936 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.546536+0000", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017329199999999999 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.547219+0000", 2023-09-23T15:11:03.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068218700000000001 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.549524+0000", 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023053259999999999 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.706644+0000", 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15712009599999999 2023-09-23T15:11:03.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.706687+0000", 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3387999999999999e-05 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.706731+0000", 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3383999999999997e-05 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:10068 5.2 5:439ed2ea:::benchmark_data_smithi017_288048_object10067:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:41.548317+0000", 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.166407647, 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16088187600000001, 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 10068 2023-09-23T15:11:03.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548317+0000", 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548317+0000", 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548316+0000", 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548384+0000", 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7811999999999994e-05 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548409+0000", 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4995999999999999e-05 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548416+0000", 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5689999999999999e-06 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548465+0000", 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8782999999999997e-05 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548587+0000", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012207 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.548736+0000", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000148665 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.549470+0000", 2023-09-23T15:11:03.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073453400000000001 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.549802+0000", 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033136899999999998 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.708423+0000", 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15862080100000001 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709152+0000", 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00072890400000000005 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709199+0000", 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7441000000000002e-05 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:10080 5.2 5:4a705979:::benchmark_data_smithi017_288048_object10079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:41.551263+0000", 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.163461708, 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15810178799999999, 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 10080 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551263+0000", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551263+0000", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551262+0000", 2023-09-23T15:11:03.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551320+0000", 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8397999999999999e-05 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551337+0000", 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7243000000000001e-05 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551344+0000", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9999999999999999e-06 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551390+0000", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5534000000000001e-05 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551503+0000", 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011307 2023-09-23T15:11:03.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.551662+0000", 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015888499999999999 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.552343+0000", 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068097199999999996 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.552739+0000", 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039601399999999998 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709275+0000", 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15653576299999999 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709328+0000", 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3205999999999998e-05 2023-09-23T15:11:03.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709365+0000", 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7055999999999999e-05 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:10086 5.2 5:426738a7:::benchmark_data_smithi017_288048_object10085:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:41.555039+0000", 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.15968538000001, 2023-09-23T15:11:03.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15446627199999999, 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 10086 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555039+0000", 2023-09-23T15:11:03.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555039+0000", 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555038+0000", 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555097+0000", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8288000000000002e-05 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555114+0000", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7243000000000001e-05 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555121+0000", 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3540000000000002e-06 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555179+0000", 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7399999999999999e-05 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555326+0000", 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014752999999999999 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555488+0000", 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000162077 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556460+0000", 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097171599999999996 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556487+0000", 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6906999999999998e-05 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709423+0000", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.152936085 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709469+0000", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5918000000000003e-05 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709506+0000", 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6804000000000002e-05 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.760 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:10087 5.2 5:47ae2a86:::benchmark_data_smithi017_288048_object10086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:41.555368+0000", 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 202.15935693099999, 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15428365999999999, 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 10087 2023-09-23T15:11:03.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555368+0000", 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555368+0000", 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555367+0000", 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555430+0000", 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2973999999999997e-05 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555438+0000", 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9920000000000007e-06 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555442+0000", 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0600000000000001e-06 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.555936+0000", 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00049335999999999998 2023-09-23T15:11:03.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556034+0000", 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8636999999999998e-05 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556142+0000", 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010819699999999999 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556589+0000", 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000446978 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.556808+0000", 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021846699999999999 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709562+0000", 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15275393900000001 2023-09-23T15:11:03.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709614+0000", 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2459e-05 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:41.709651+0000", 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7302999999999998e-05 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:41922 5.2 5:47943cc1:::benchmark_data_smithi017_288048_object41921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 185.01829969600001, 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.246440672, 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.768 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 41922 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696423+0000", 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696532+0000", 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108953 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696546+0000", 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3548e-05 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696553+0000", 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7490000000000001e-06 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696709+0000", 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015584999999999999 2023-09-23T15:11:03.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696849+0000", 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014056499999999999 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.697002+0000", 2023-09-23T15:11:03.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015253599999999999 2023-09-23T15:11:03.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.698626+0000", 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00162414 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.699083+0000", 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045717099999999998 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942745+0000", 2023-09-23T15:11:03.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24366205099999999 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942810+0000", 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4618999999999999e-05 2023-09-23T15:11:03.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942866+0000", 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6088999999999998e-05 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:104999 5.2 5:4250d023:::benchmark_data_smithi017_288048_object104998:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:08:58.867154+0000", 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 124.84757052099999, 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18271728500000001, 2023-09-23T15:11:03.778 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 104999 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.779 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867154+0000", 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867154+0000", 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867153+0000", 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867184+0000", 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0905000000000003e-05 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867192+0000", 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9189999999999999e-06 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867196+0000", 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0620000000000002e-06 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867240+0000", 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4598000000000002e-05 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867370+0000", 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129369 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.867513+0000", 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000142786 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.870505+0000", 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029928060000000002 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:58.873599+0000", 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030932540000000001 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:59.049778+0000", 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17617917899999999 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:59.049830+0000", 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2583000000000001e-05 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:59.049872+0000", 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1062000000000001e-05 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:114938 5.2 5:48ebc11a:::benchmark_data_smithi017_288048_object3169:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:11.716167+0000", 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 111.99855748, 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.170043314, 2023-09-23T15:11:03.787 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 114938 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716167+0000", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716167+0000", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716166+0000", 2023-09-23T15:11:03.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716173+0000", 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.973e-06 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716185+0000", 2023-09-23T15:11:03.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2381999999999999e-05 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716192+0000", 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2689999999999996e-06 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.719155+0000", 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002963795 2023-09-23T15:11:03.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.719293+0000", 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137794 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.719434+0000", 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014112400000000001 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.726986+0000", 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075514809999999996 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.730787+0000", 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003801624 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886104+0000", 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.155316282 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886169+0000", 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5003000000000001e-05 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886211+0000", 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1958e-05 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:114940 5.2 5:442f14da:::benchmark_data_smithi017_288048_object3171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e136)", 2023-09-23T15:11:03.795 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:11.716202+0000", 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 111.998522846, 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.170580485, 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 114940 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.796 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716202+0000", 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716202+0000", 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716202+0000", 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716204+0000", 2023-09-23T15:11:03.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7590000000000002e-06 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716211+0000", 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4019999999999999e-06 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.716215+0000", 2023-09-23T15:11:03.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1710000000000004e-06 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.723048+0000", 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068330869999999998 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.723186+0000", 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137701 2023-09-23T15:11:03.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.723303+0000", 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117442 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.728118+0000", 2023-09-23T15:11:03.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048147479999999998 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.736258+0000", 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081404070000000005 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886704+0000", 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15044603600000001 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886748+0000", 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3785999999999997e-05 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:11.886782+0000", 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4319999999999997e-05 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126415 5.2 5:4a39f4d1:::benchmark_data_smithi017_288048_object14646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.280655+0000", 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.434069573000002, 2023-09-23T15:11:03.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.190547417, 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126415 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280655+0000", 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280655+0000", 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280654+0000", 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280661+0000", 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9360000000000002e-06 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280678+0000", 2023-09-23T15:11:03.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6708e-05 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280686+0000", 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4090000000000003e-06 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280737+0000", 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1144999999999999e-05 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280886+0000", 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014899100000000001 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.280967+0000", 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1097e-05 2023-09-23T15:11:03.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.284095+0000", 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031281149999999999 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.284270+0000", 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000174532 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.471092+0000", 2023-09-23T15:11:03.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18682190700000001 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.471157+0000", 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4976999999999998e-05 2023-09-23T15:11:03.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.471203+0000", 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5935e-05 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.814 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:127617 5.2 5:446eef6f:::benchmark_data_smithi017_288048_object15848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:27.853439+0000", 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.861285871999996, 2023-09-23T15:11:03.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.185056797, 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.816 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127617 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853439+0000", 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853439+0000", 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853438+0000", 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853442+0000", 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4429999999999996e-06 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853453+0000", 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1001e-05 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853458+0000", 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6160000000000004e-06 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853493+0000", 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5151999999999998e-05 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853580+0000", 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6784000000000001e-05 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.853649+0000", 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9039000000000001e-05 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.857657+0000", 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040081960000000003 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.857888+0000", 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023114699999999999 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.038403+0000", 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.180514917 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.038452+0000", 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8356999999999998e-05 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.038496+0000", 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4147999999999998e-05 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:127620 5.2 5:46c7c177:::benchmark_data_smithi017_288048_object15851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:27.861237+0000", 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.853488079000002, 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17785045999999999, 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.825 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127620 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861237+0000", 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861237+0000", 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861236+0000", 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861242+0000", 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6830000000000002e-06 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861256+0000", 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3682999999999999e-05 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861261+0000", 2023-09-23T15:11:03.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3240000000000002e-06 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861298+0000", 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6882999999999999e-05 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861391+0000", 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3317000000000001e-05 2023-09-23T15:11:03.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.861471+0000", 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9589e-05 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.864427+0000", 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029560400000000001 2023-09-23T15:11:03.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:27.864504+0000", 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7383000000000001e-05 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.039003+0000", 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17449898799999999 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.039050+0000", 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6801999999999997e-05 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:28.039087+0000", 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6872000000000002e-05 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.832 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:138489 5.2 5:45bffb57:::benchmark_data_smithi017_288048_object26720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e140)", 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:41.826006+0000", 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 81.888718291000004, 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16082212200000001, 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.833 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 138489 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826006+0000", 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826006+0000", 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826005+0000", 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826010+0000", 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8019999999999999e-06 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826022+0000", 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.132e-05 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826026+0000", 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7990000000000001e-06 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826076+0000", 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9268999999999999e-05 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826216+0000", 2023-09-23T15:11:03.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000140106 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.826326+0000", 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110493 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.830130+0000", 2023-09-23T15:11:03.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038036340000000002 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.830647+0000", 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000517531 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.986747+0000", 2023-09-23T15:11:03.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15609936999999999 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.986793+0000", 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6159999999999999e-05 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:41.986829+0000", 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5678e-05 2023-09-23T15:11:03.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:165888 5.2 5:44fcaf03:::benchmark_data_smithi017_288048_object54119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e146)", 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:16.550175+0000", 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 47.164549921999999, 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.174346628, 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 165888 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550175+0000", 2023-09-23T15:11:03.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550175+0000", 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550174+0000", 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550180+0000", 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4130000000000003e-06 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550192+0000", 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2007e-05 2023-09-23T15:11:03.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550198+0000", 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3090000000000001e-06 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550288+0000", 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9322999999999998e-05 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550397+0000", 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109294 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.550508+0000", 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111254 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.554296+0000", 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037877029999999999 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.558165+0000", 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003869047 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.724433+0000", 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16626848399999999 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.724482+0000", 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8514999999999999e-05 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:16.724521+0000", 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9622000000000001e-05 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166602 5.2 5:44eae416:::benchmark_data_smithi017_288048_object54833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:17.560806+0000", 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 46.153918515999997, 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.173672786, 2023-09-23T15:11:03.849 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166602 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560806+0000", 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560806+0000", 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560805+0000", 2023-09-23T15:11:03.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560812+0000", 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0700000000000001e-06 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560832+0000", 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9817999999999999e-05 2023-09-23T15:11:03.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560839+0000", 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7510000000000008e-06 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.560893+0000", 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3841000000000002e-05 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.561031+0000", 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013777799999999999 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.561162+0000", 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013044900000000001 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.564902+0000", 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037408659999999998 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.569689+0000", 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047866860000000001 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734417+0000", 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.1647275 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734452+0000", 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5636000000000002e-05 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734479+0000", 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6806000000000001e-05 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166608 5.2 5:4131c2df:::benchmark_data_smithi017_288048_object54839:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:17.565734+0000", 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 46.148991019, 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16883922100000001, 2023-09-23T15:11:03.857 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166608 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565734+0000", 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565734+0000", 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565732+0000", 2023-09-23T15:11:03.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565738+0000", 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7540000000000003e-06 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565752+0000", 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3543000000000001e-05 2023-09-23T15:11:03.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565757+0000", 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5140000000000001e-06 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565799+0000", 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1700999999999999e-05 2023-09-23T15:11:03.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.565927+0000", 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128064 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.566055+0000", 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012797899999999999 2023-09-23T15:11:03.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.569732+0000", 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036773520000000001 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.569789+0000", 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6471999999999997e-05 2023-09-23T15:11:03.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734508+0000", 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.164719005 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734550+0000", 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2389999999999999e-05 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:17.734573+0000", 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2906000000000001e-05 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166949 5.2 5:4bbd8647:::benchmark_data_smithi017_288048_object55180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:03.865 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 45.626973390000003, 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.2839622, 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166949 2023-09-23T15:11:03.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087750+0000", 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087759+0000", 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9779999999999994e-06 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087779+0000", 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0055999999999999e-05 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087787+0000", 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0169999999999996e-06 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087834+0000", 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7286000000000001e-05 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087944+0000", 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010972 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.088049+0000", 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010541 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091850+0000", 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038003780000000001 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099251+0000", 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007400968 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371615+0000", 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27236480099999999 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371672+0000", 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6801e-05 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371714+0000", 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1390000000000002e-05 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:175560 5.2 5:4219b957:::benchmark_data_smithi017_288048_object63791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:29.736467+0000", 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 33.978258025000002, 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17610619599999999, 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:03.874 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 175560 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736467+0000", 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736467+0000", 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736465+0000", 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736471+0000", 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6219999999999997e-06 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736483+0000", 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2130000000000001e-05 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:03.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.736490+0000", 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5799999999999997e-06 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.737181+0000", 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000691531 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:03.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.737261+0000", 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9583e-05 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.737331+0000", 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0159000000000004e-05 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.741308+0000", 2023-09-23T15:11:03.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039773009999999999 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.742100+0000", 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00079198099999999998 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.912484+0000", 2023-09-23T15:11:03.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17038362900000001 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.912532+0000", 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7568000000000003e-05 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.912573+0000", 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1434999999999999e-05 2023-09-23T15:11:03.881 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.882 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.882 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:03.882 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:03.882 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:11:03.882 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:03.942 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:04.082 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:04.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:04.186 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:04.220 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:04.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:04.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:04.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:04.306 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:04.362 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:04.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:04.425 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T15:11:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/887677304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:04.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T15:11:04.453 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/887677304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:04.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:04.505 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:04.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T15:11:04.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/887677304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:04.660 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:04.671 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:04.673 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:11:04.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:04.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:04.680 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:04.698 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.699 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:11:04.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:04.807 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:04.902 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:04.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:05.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:05.033 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:05.036 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:11:05.076 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:05.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:05.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:05.180 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:05.209 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:05.257 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:05.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:05.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:05.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:05.441 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:05.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[215646]: pgmap v386: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 7768/74280 objects degraded (10.458%) 2023-09-23T15:11:05.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[215646]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[215646]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[215646]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[222878]: pgmap v386: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 7768/74280 objects degraded (10.458%) 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[222878]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[222878]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T15:11:05.570 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:05 smithi017 ceph-mon[222878]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.590 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:05.591 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:05 smithi027 ceph-mon[195233]: pgmap v386: 109 pgs: 11 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.5 GiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 589 op/s; 7768/74280 objects degraded (10.458%) 2023-09-23T15:11:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:05 smithi027 ceph-mon[195233]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:05 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T15:11:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:05 smithi027 ceph-mon[195233]: osdmap e157: 8 total, 7 up, 8 in 2023-09-23T15:11:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:05 smithi027 ceph-mon[195233]: failed to encode map e157 with expected crc 2023-09-23T15:11:05.676 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:05.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:05.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:05.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:05.833 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:05.900 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:11:05.911 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:05.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:05.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:06.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:06.097 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:06.202 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:06.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:06.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:06.386 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[215646]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:06 smithi017 ceph-mon[222878]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.451 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:06.542 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:06.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:06.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:06.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: osdmap e158: 8 total, 7 up, 8 in 2023-09-23T15:11:06.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:06 smithi027 ceph-mon[195233]: failed to encode map e158 with expected crc 2023-09-23T15:11:06.653 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:06.699 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:06.780 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:06.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:06.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:07.002 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:11:07.002 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:11:07.078 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:07.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:07.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:07.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:07.225 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:07.301 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:07.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:07.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:07.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:07.435 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:07.480 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:07.482 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:07 smithi027 ceph-mon[195233]: pgmap v389: 119 pgs: 10 unknown, 1 peering, 10 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5677/66765 objects degraded (8.503%) 2023-09-23T15:11:07.482 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5677/66765 objects degraded (8.503%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:07.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:07.483 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:07.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:07.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:11:07.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:07 smithi017 ceph-mon[215646]: pgmap v389: 119 pgs: 10 unknown, 1 peering, 10 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5677/66765 objects degraded (8.503%) 2023-09-23T15:11:07.611 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5677/66765 objects degraded (8.503%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:07.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:07 smithi017 ceph-mon[222878]: pgmap v389: 119 pgs: 10 unknown, 1 peering, 10 active+undersized+degraded, 26 active+undersized, 72 active+clean; 1.4 GiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 5677/66765 objects degraded (8.503%) 2023-09-23T15:11:07.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5677/66765 objects degraded (8.503%), 10 pgs degraded, 36 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:07.614 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:07.902 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:07.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:07.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:07.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:11:08.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:08.111 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:08.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:08] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T15:11:08.268 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:08.278 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:08.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:08.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:08.466 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:08.577 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:08.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:08.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:08.663 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:11:08.689 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:08.757 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:08.884 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:11:08.884 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:11:08.884 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_0 2023-09-23T15:11:08.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:11:08.964 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:08.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:08.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:09.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:09.107 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:09.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:09.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:09.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:09.257 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:09.284 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:09.361 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:11:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[222878]: pgmap v390: 119 pgs: 8 unknown, 1 peering, 11 active+undersized+degraded, 26 active+undersized, 73 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 5454/63318 objects degraded (8.614%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/862924393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[215646]: pgmap v390: 119 pgs: 8 unknown, 1 peering, 11 active+undersized+degraded, 26 active+undersized, 73 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 5454/63318 objects degraded (8.614%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:09.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/862924393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:09 smithi027 ceph-mon[195233]: pgmap v390: 119 pgs: 8 unknown, 1 peering, 11 active+undersized+degraded, 26 active+undersized, 73 active+clean; 1.3 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 5454/63318 objects degraded (8.614%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/862924393' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:09.811 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:09.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:09.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:09.943 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:10.326 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:10.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:10.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:10.438 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:10.502 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:10.561 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:10.593 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:10.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:10 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:10.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:10 smithi027 ceph-mon[195233]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:10.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:10 smithi027 ceph-mon[195233]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T15:11:10.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:10 smithi027 ceph-mon[195233]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:10.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[215646]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[215646]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[215646]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[222878]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.612 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:10.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[222878]: osdmap e159: 8 total, 7 up, 8 in 2023-09-23T15:11:10.613 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:10 smithi017 ceph-mon[222878]: failed to encode map e159 with expected crc 2023-09-23T15:11:10.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:10.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:10.751 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:10.779 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:10.799 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:10.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:10.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:10.896 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:10.925 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:10.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:10.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.934 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:10.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.934 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.935 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:11:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:10.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:10.961 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:11:10.961 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:11:10.961 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:11:10.961 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:11:10.962 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:11:11.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:11.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:11.248 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:11.262 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:11.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:11.411 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:11 smithi027 ceph-mon[195233]: pgmap v392: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3206083679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:11.625 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:11.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:11 smithi017 ceph-mon[215646]: pgmap v392: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:11.628 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3206083679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:11.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:11 smithi017 ceph-mon[222878]: pgmap v392: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 929 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:11.629 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3206083679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:11.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:11.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:11.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:11.776 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:12.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:11:12.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:12.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:12.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:12.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:11:12.377 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:12.377 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:12.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:12.473 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:12.575 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:12.582 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5886/57537 objects degraded (10.230%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:12.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:12.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:12.584 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5886/57537 objects degraded (10.230%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:12.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:12.737 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5886/57537 objects degraded (10.230%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:12.745 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:12.791 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:11:12.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:12.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:12.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:12.940 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:13.020 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:13.109 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:13.138 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:13.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:13.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:13.247 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:13.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:13.386 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:11:13.393 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:11:13.579 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:13.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:13.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:13.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:13.752 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:13.814 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:13.819 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:13 smithi027 ceph-mon[195233]: pgmap v393: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:13.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:13.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[222878]: pgmap v393: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:13.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:13.960 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:13.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[215646]: pgmap v393: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.2 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 5886/57537 objects degraded (10.230%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:13.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:13.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:13.972 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:14.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:11:14.811 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:14.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:14.816 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:14.829 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:14 smithi027 ceph-mon[195233]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:11:14.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:14.979 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:14.982 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":393,"stamp":"2023-09-23T15:11:12.229123+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1259873258,"num_objects":19179,"num_object_clones":0,"num_object_copies":57537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5886,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19179,"num_whiteouts":0,"num_read":6110,"num_read_kb":13423,"num_write":205346,"num_write_kb":7166300,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":402524,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":207712,"ondisk_log_size":207712,"up":316,"acting":316,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6008776,"kb_used_data":5220296,"kb_used_omap":30,"kb_used_meta":788385,"kb_avail":743952440,"statfs":{"total":767960285184,"available":761807298560,"internally_reserved":0,"allocated":5345583104,"data_stored":5340907898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":807306840},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-320667648,"num_objects":-4893,"num_object_clones":0,"num_object_copies":-14679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1660,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4893,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4893,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.932830"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":142,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.310051+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:11:04.310051+0000","last_peered":"2023-09-23T15:11:04.310051+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:11:04.310051+0000","last_undegraded":"2023-09-23T15:11:04.310051+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":84,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.342058+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:11:04.342058+0000","last_peered":"2023-09-23T15:11:04.342058+0000","last_clean":"2023-09-23T15:11:04.342058+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:11:04.342058+0000","last_undegraded":"2023-09-23T15:11:04.342058+0000","last_fullsized":"2023-09-23T15:11:04.342058+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":72,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.349679+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:11:04.349679+0000","last_peered":"2023-09-23T15:11:04.349679+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:11:04.349679+0000","last_undegraded":"2023-09-23T15:11:04.349679+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":101,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.312855+0000","last_change":"2023-09-23T15:06:16.127977+0000","last_active":"2023-09-23T15:11:04.312855+0000","last_peered":"2023-09-23T15:11:04.312855+0000","last_clean":"2023-09-23T15:11:04.312855+0000","last_became_active":"2023-09-23T15:06:16.127561+0000","last_became_peered":"2023-09-23T15:06:16.127561+0000","last_unstale":"2023-09-23T15:11:04.312855+0000","last_undegraded":"2023-09-23T15:11:04.312855+0000","last_fullsized":"2023-09-23T15:11:04.312855+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:09:39.444518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.360816+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:11:04.360816+0000","last_peered":"2023-09-23T15:11:04.360816+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:11:04.360816+0000","last_undegraded":"2023-09-23T15:11:04.360816+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":122,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.432678+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:11:04.432678+0000","last_peered":"2023-09-23T15:11:04.432678+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:11:04.432678+0000","last_undegraded":"2023-09-23T15:11:04.432678+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.11","version":"158'12649","reported_seq":12711,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.084336+0000","last_change":"2023-09-23T15:11:05.445752+0000","last_active":"2023-09-23T15:11:09.084336+0000","last_peered":"2023-09-23T15:11:09.084336+0000","last_clean":"2023-09-23T15:11:09.084336+0000","last_became_active":"2023-09-23T15:11:05.407075+0000","last_became_peered":"2023-09-23T15:11:05.407075+0000","last_unstale":"2023-09-23T15:11:09.084336+0000","last_undegraded":"2023-09-23T15:11:09.084336+0000","last_fullsized":"2023-09-23T15:11:09.084336+0000","mapping_epoch":157,"log_start":"129'5100","ondisk_log_start":"129'5100","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38207488,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6391,"num_write_kb":223168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.313238+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:11:04.313238+0000","last_peered":"2023-09-23T15:11:04.313238+0000","last_clean":"2023-09-23T15:11:04.313238+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:11:04.313238+0000","last_undegraded":"2023-09-23T15:11:04.313238+0000","last_fullsized":"2023-09-23T15:11:04.313238+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":127,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.360666+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:11:04.360666+0000","last_peered":"2023-09-23T15:11:04.360666+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:11:04.360666+0000","last_undegraded":"2023-09-23T15:11:04.360666+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.10","version":"158'12541","reported_seq":12610,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.059694+0000","last_change":"2023-09-23T15:11:05.453187+0000","last_active":"2023-09-23T15:11:09.059694+0000","last_peered":"2023-09-23T15:11:09.059694+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:11:05.387635+0000","last_became_peered":"2023-09-23T15:11:05.387635+0000","last_unstale":"2023-09-23T15:11:09.059694+0000","last_undegraded":"2023-09-23T15:11:05.353372+0000","last_fullsized":"2023-09-23T15:11:05.323539+0000","mapping_epoch":157,"log_start":"124'3500","ondisk_log_start":"124'3500","created":157,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9041,"log_dups_size":0,"ondisk_log_size":9041,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37715968,"num_objects":576,"num_object_clones":0,"num_object_copies":1728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6330,"num_write_kb":220960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":576}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":526,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347872+0000","last_change":"2023-09-23T15:10:47.399720+0000","last_active":"2023-09-23T15:11:04.347872+0000","last_peered":"2023-09-23T15:11:04.347872+0000","last_clean":"2023-09-23T15:11:04.347872+0000","last_became_active":"2023-09-23T15:10:47.399348+0000","last_became_peered":"2023-09-23T15:10:47.399348+0000","last_unstale":"2023-09-23T15:11:04.347872+0000","last_undegraded":"2023-09-23T15:11:04.347872+0000","last_fullsized":"2023-09-23T15:11:04.347872+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":125,"reported_epoch":158,"state":"active+undersized","last_fresh":"2023-09-23T15:11:05.304353+0000","last_change":"2023-09-23T15:08:51.249627+0000","last_active":"2023-09-23T15:11:05.304353+0000","last_peered":"2023-09-23T15:11:05.304353+0000","last_clean":"2023-09-23T15:08:45.872663+0000","last_became_active":"2023-09-23T15:08:51.249627+0000","last_became_peered":"2023-09-23T15:08:51.249627+0000","last_unstale":"2023-09-23T15:11:05.304353+0000","last_undegraded":"2023-09-23T15:11:05.304353+0000","last_fullsized":"2023-09-23T15:08:51.247272+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:21.657536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.349577+0000","last_change":"2023-09-23T15:06:16.134068+0000","last_active":"2023-09-23T15:11:04.349577+0000","last_peered":"2023-09-23T15:11:04.349577+0000","last_clean":"2023-09-23T15:11:04.349577+0000","last_became_active":"2023-09-23T15:06:16.133794+0000","last_became_peered":"2023-09-23T15:06:16.133794+0000","last_unstale":"2023-09-23T15:11:04.349577+0000","last_undegraded":"2023-09-23T15:11:04.349577+0000","last_fullsized":"2023-09-23T15:11:04.349577+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:23.195345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.13","version":"158'12339","reported_seq":12399,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.305626+0000","last_change":"2023-09-23T15:11:05.377859+0000","last_active":"2023-09-23T15:11:09.305626+0000","last_peered":"2023-09-23T15:11:09.305626+0000","last_clean":"2023-09-23T15:11:09.305626+0000","last_became_active":"2023-09-23T15:11:05.357848+0000","last_became_peered":"2023-09-23T15:11:05.357848+0000","last_unstale":"2023-09-23T15:11:09.305626+0000","last_undegraded":"2023-09-23T15:11:09.305626+0000","last_fullsized":"2023-09-23T15:11:09.305626+0000","mapping_epoch":157,"log_start":"135'6674","ondisk_log_start":"135'6674","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5665,"log_dups_size":0,"ondisk_log_size":5665,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37388288,"num_objects":571,"num_object_clones":0,"num_object_copies":1713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6223,"num_write_kb":217376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347433+0000","last_change":"2023-09-23T15:06:16.137711+0000","last_active":"2023-09-23T15:11:04.347433+0000","last_peered":"2023-09-23T15:11:04.347433+0000","last_clean":"2023-09-23T15:11:04.347433+0000","last_became_active":"2023-09-23T15:06:16.137330+0000","last_became_peered":"2023-09-23T15:06:16.137330+0000","last_unstale":"2023-09-23T15:11:04.347433+0000","last_undegraded":"2023-09-23T15:11:04.347433+0000","last_fullsized":"2023-09-23T15:11:04.347433+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:16.856375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":68,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.436383+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:11:04.436383+0000","last_peered":"2023-09-23T15:11:04.436383+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:11:04.436383+0000","last_undegraded":"2023-09-23T15:11:04.436383+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.12","version":"158'12906","reported_seq":12967,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.068543+0000","last_change":"2023-09-23T15:11:05.385811+0000","last_active":"2023-09-23T15:11:09.068543+0000","last_peered":"2023-09-23T15:11:09.068543+0000","last_clean":"2023-09-23T15:11:09.068543+0000","last_became_active":"2023-09-23T15:11:05.357392+0000","last_became_peered":"2023-09-23T15:11:05.357392+0000","last_unstale":"2023-09-23T15:11:09.068543+0000","last_undegraded":"2023-09-23T15:11:09.068543+0000","last_fullsized":"2023-09-23T15:11:09.068543+0000","mapping_epoch":157,"log_start":"135'6853","ondisk_log_start":"135'6853","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6053,"log_dups_size":0,"ondisk_log_size":6053,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38567936,"num_objects":589,"num_object_clones":0,"num_object_copies":1767,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":589,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6522,"num_write_kb":227520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":122,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.311385+0000","last_change":"2023-09-23T14:56:31.333505+0000","last_active":"2023-09-23T15:11:04.311385+0000","last_peered":"2023-09-23T15:11:04.311385+0000","last_clean":"2023-09-23T15:11:04.311385+0000","last_became_active":"2023-09-23T14:56:31.333220+0000","last_became_peered":"2023-09-23T14:56:31.333220+0000","last_unstale":"2023-09-23T15:11:04.311385+0000","last_undegraded":"2023-09-23T15:11:04.311385+0000","last_fullsized":"2023-09-23T15:11:04.311385+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:55.323834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.309925+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:11:04.309925+0000","last_peered":"2023-09-23T15:11:04.309925+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:11:04.309925+0000","last_undegraded":"2023-09-23T15:11:04.309925+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.15","version":"158'12867","reported_seq":12933,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.084483+0000","last_change":"2023-09-23T15:11:05.375618+0000","last_active":"2023-09-23T15:11:09.084483+0000","last_peered":"2023-09-23T15:11:09.084483+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:11:05.375618+0000","last_became_peered":"2023-09-23T15:11:05.375618+0000","last_unstale":"2023-09-23T15:11:09.084483+0000","last_undegraded":"2023-09-23T15:11:05.345172+0000","last_fullsized":"2023-09-23T15:11:05.323562+0000","mapping_epoch":157,"log_start":"136'7147","ondisk_log_start":"136'7147","created":157,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5720,"log_dups_size":0,"ondisk_log_size":5720,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37912576,"num_objects":579,"num_object_clones":0,"num_object_copies":1737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":579,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6495,"num_write_kb":226336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":579}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":525,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344750+0000","last_change":"2023-09-23T15:10:47.405764+0000","last_active":"2023-09-23T15:11:04.344750+0000","last_peered":"2023-09-23T15:11:04.344750+0000","last_clean":"2023-09-23T15:11:04.344750+0000","last_became_active":"2023-09-23T15:10:47.404706+0000","last_became_peered":"2023-09-23T15:10:47.404706+0000","last_unstale":"2023-09-23T15:11:04.344750+0000","last_undegraded":"2023-09-23T15:11:04.344750+0000","last_fullsized":"2023-09-23T15:11:04.344750+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:24.157332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:24.157332+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0084559189999999992,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":118,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344770+0000","last_change":"2023-09-23T15:05:44.720718+0000","last_active":"2023-09-23T15:11:04.344770+0000","last_peered":"2023-09-23T15:11:04.344770+0000","last_clean":"2023-09-23T15:11:04.344770+0000","last_became_active":"2023-09-23T15:05:44.720152+0000","last_became_peered":"2023-09-23T15:05:44.720152+0000","last_unstale":"2023-09-23T15:11:04.344770+0000","last_undegraded":"2023-09-23T15:11:04.344770+0000","last_fullsized":"2023-09-23T15:11:04.344770+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:05.440419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.342336+0000","last_change":"2023-09-23T15:08:51.250743+0000","last_active":"2023-09-23T15:11:04.342336+0000","last_peered":"2023-09-23T15:11:04.342336+0000","last_clean":"2023-09-23T15:08:45.959806+0000","last_became_active":"2023-09-23T15:08:51.250743+0000","last_became_peered":"2023-09-23T15:08:51.250743+0000","last_unstale":"2023-09-23T15:11:04.342336+0000","last_undegraded":"2023-09-23T15:11:04.342336+0000","last_fullsized":"2023-09-23T15:08:51.246034+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:43.516361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.14","version":"158'12549","reported_seq":12609,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.061486+0000","last_change":"2023-09-23T15:11:05.385050+0000","last_active":"2023-09-23T15:11:09.061486+0000","last_peered":"2023-09-23T15:11:09.061486+0000","last_clean":"2023-09-23T15:11:09.061486+0000","last_became_active":"2023-09-23T15:11:05.356758+0000","last_became_peered":"2023-09-23T15:11:05.356758+0000","last_unstale":"2023-09-23T15:11:09.061486+0000","last_undegraded":"2023-09-23T15:11:09.061486+0000","last_fullsized":"2023-09-23T15:11:09.061486+0000","mapping_epoch":157,"log_start":"135'6952","ondisk_log_start":"135'6952","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5597,"log_dups_size":0,"ondisk_log_size":5597,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37814272,"num_objects":577,"num_object_clones":0,"num_object_copies":1731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6332,"num_write_kb":221088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345126+0000","last_change":"2023-09-23T15:10:47.400462+0000","last_active":"2023-09-23T15:11:04.345126+0000","last_peered":"2023-09-23T15:11:04.345126+0000","last_clean":"2023-09-23T15:11:04.345126+0000","last_became_active":"2023-09-23T15:10:47.400102+0000","last_became_peered":"2023-09-23T15:10:47.400102+0000","last_unstale":"2023-09-23T15:11:04.345126+0000","last_undegraded":"2023-09-23T15:11:04.345126+0000","last_fullsized":"2023-09-23T15:11:04.345126+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0070307169999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.309726+0000","last_change":"2023-09-23T15:06:16.126769+0000","last_active":"2023-09-23T15:11:04.309726+0000","last_peered":"2023-09-23T15:11:04.309726+0000","last_clean":"2023-09-23T15:11:04.309726+0000","last_became_active":"2023-09-23T15:06:16.126593+0000","last_became_peered":"2023-09-23T15:06:16.126593+0000","last_unstale":"2023-09-23T15:11:04.309726+0000","last_undegraded":"2023-09-23T15:11:04.309726+0000","last_fullsized":"2023-09-23T15:11:04.309726+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:15.319984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.303956+0000","last_change":"2023-09-23T15:06:16.138039+0000","last_active":"2023-09-23T15:11:05.303956+0000","last_peered":"2023-09-23T15:11:05.303956+0000","last_clean":"2023-09-23T15:11:05.303956+0000","last_became_active":"2023-09-23T15:06:16.137667+0000","last_became_peered":"2023-09-23T15:06:16.137667+0000","last_unstale":"2023-09-23T15:11:05.303956+0000","last_undegraded":"2023-09-23T15:11:05.303956+0000","last_fullsized":"2023-09-23T15:11:05.303956+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:52:46.768980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.17","version":"159'12841","reported_seq":12906,"reported_epoch":159,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.561880+0000","last_change":"2023-09-23T15:11:05.347061+0000","last_active":"2023-09-23T15:11:09.561880+0000","last_peered":"2023-09-23T15:11:09.561880+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:11:05.347061+0000","last_became_peered":"2023-09-23T15:11:05.347061+0000","last_unstale":"2023-09-23T15:11:09.561880+0000","last_undegraded":"2023-09-23T15:11:05.340859+0000","last_fullsized":"2023-09-23T15:11:05.312370+0000","mapping_epoch":157,"log_start":"122'2700","ondisk_log_start":"122'2700","created":157,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10141,"log_dups_size":0,"ondisk_log_size":10141,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37224448,"num_objects":568,"num_object_clones":0,"num_object_copies":1704,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":568,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":568,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6490,"num_write_kb":225856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":568}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":516,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347369+0000","last_change":"2023-09-23T15:10:47.399759+0000","last_active":"2023-09-23T15:11:04.347369+0000","last_peered":"2023-09-23T15:11:04.347369+0000","last_clean":"2023-09-23T15:11:04.347369+0000","last_became_active":"2023-09-23T15:10:47.399393+0000","last_became_peered":"2023-09-23T15:10:47.399393+0000","last_unstale":"2023-09-23T15:11:04.347369+0000","last_undegraded":"2023-09-23T15:11:04.347369+0000","last_fullsized":"2023-09-23T15:11:04.347369+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":99,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344986+0000","last_change":"2023-09-23T14:56:31.342860+0000","last_active":"2023-09-23T15:11:04.344986+0000","last_peered":"2023-09-23T15:11:04.344986+0000","last_clean":"2023-09-23T15:11:04.344986+0000","last_became_active":"2023-09-23T14:56:31.340962+0000","last_became_peered":"2023-09-23T14:56:31.340962+0000","last_unstale":"2023-09-23T15:11:04.344986+0000","last_undegraded":"2023-09-23T15:11:04.344986+0000","last_fullsized":"2023-09-23T15:11:04.344986+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:51.736885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.312921+0000","last_change":"2023-09-23T15:06:16.127799+0000","last_active":"2023-09-23T15:11:04.312921+0000","last_peered":"2023-09-23T15:11:04.312921+0000","last_clean":"2023-09-23T15:11:04.312921+0000","last_became_active":"2023-09-23T15:06:16.127227+0000","last_became_peered":"2023-09-23T15:06:16.127227+0000","last_unstale":"2023-09-23T15:11:04.312921+0000","last_undegraded":"2023-09-23T15:11:04.312921+0000","last_fullsized":"2023-09-23T15:11:04.312921+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:25.574542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.16","version":"158'12601","reported_seq":12667,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.149730+0000","last_change":"2023-09-23T15:11:05.424450+0000","last_active":"2023-09-23T15:11:08.149730+0000","last_peered":"2023-09-23T15:11:08.149730+0000","last_clean":"2023-09-23T15:11:08.149730+0000","last_became_active":"2023-09-23T15:11:05.366154+0000","last_became_peered":"2023-09-23T15:11:05.366154+0000","last_unstale":"2023-09-23T15:11:08.149730+0000","last_undegraded":"2023-09-23T15:11:08.149730+0000","last_fullsized":"2023-09-23T15:11:08.149730+0000","mapping_epoch":157,"log_start":"122'2500","ondisk_log_start":"122'2500","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10101,"log_dups_size":0,"ondisk_log_size":10101,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40697856,"num_objects":621,"num_object_clones":0,"num_object_copies":1863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6345,"num_write_kb":222912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":516,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344749+0000","last_change":"2023-09-23T15:10:47.400107+0000","last_active":"2023-09-23T15:11:04.344749+0000","last_peered":"2023-09-23T15:11:04.344749+0000","last_clean":"2023-09-23T15:11:04.344749+0000","last_became_active":"2023-09-23T15:10:47.399747+0000","last_became_peered":"2023-09-23T15:10:47.399747+0000","last_unstale":"2023-09-23T15:11:04.344749+0000","last_undegraded":"2023-09-23T15:11:04.344749+0000","last_fullsized":"2023-09-23T15:11:04.344749+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":124,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.431995+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:11:04.431995+0000","last_peered":"2023-09-23T15:11:04.431995+0000","last_clean":"2023-09-23T15:11:04.431995+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:11:04.431995+0000","last_undegraded":"2023-09-23T15:11:04.431995+0000","last_fullsized":"2023-09-23T15:11:04.431995+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":127,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.342534+0000","last_change":"2023-09-23T15:08:51.248341+0000","last_active":"2023-09-23T15:11:04.342534+0000","last_peered":"2023-09-23T15:11:04.342534+0000","last_clean":"2023-09-23T15:08:45.959865+0000","last_became_active":"2023-09-23T15:08:51.248341+0000","last_became_peered":"2023-09-23T15:08:51.248341+0000","last_unstale":"2023-09-23T15:11:04.342534+0000","last_undegraded":"2023-09-23T15:11:04.342534+0000","last_fullsized":"2023-09-23T15:08:51.245792+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.457701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":526,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347068+0000","last_change":"2023-09-23T15:10:47.399819+0000","last_active":"2023-09-23T15:11:04.347068+0000","last_peered":"2023-09-23T15:11:04.347068+0000","last_clean":"2023-09-23T15:11:04.347068+0000","last_became_active":"2023-09-23T15:10:47.399322+0000","last_became_peered":"2023-09-23T15:10:47.399322+0000","last_unstale":"2023-09-23T15:11:04.347068+0000","last_undegraded":"2023-09-23T15:11:04.347068+0000","last_fullsized":"2023-09-23T15:11:04.347068+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0046530740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.9","version":"158'12772","reported_seq":12834,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.936277+0000","last_change":"2023-09-23T15:11:05.366329+0000","last_active":"2023-09-23T15:11:08.936277+0000","last_peered":"2023-09-23T15:11:08.936277+0000","last_clean":"2023-09-23T15:11:08.936277+0000","last_became_active":"2023-09-23T15:11:05.343032+0000","last_became_peered":"2023-09-23T15:11:05.343032+0000","last_unstale":"2023-09-23T15:11:08.936277+0000","last_undegraded":"2023-09-23T15:11:08.936277+0000","last_fullsized":"2023-09-23T15:11:08.936277+0000","mapping_epoch":114,"log_start":"136'7186","ondisk_log_start":"136'7186","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6439,"num_write_kb":225248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347777+0000","last_change":"2023-09-23T15:04:40.706408+0000","last_active":"2023-09-23T15:11:04.347777+0000","last_peered":"2023-09-23T15:11:04.347777+0000","last_clean":"2023-09-23T15:11:04.347777+0000","last_became_active":"2023-09-23T15:04:40.706041+0000","last_became_peered":"2023-09-23T15:04:40.706041+0000","last_unstale":"2023-09-23T15:11:04.347777+0000","last_undegraded":"2023-09-23T15:11:04.347777+0000","last_fullsized":"2023-09-23T15:11:04.347777+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:06.226688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":100,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.303741+0000","last_change":"2023-09-23T14:56:31.640889+0000","last_active":"2023-09-23T15:11:05.303741+0000","last_peered":"2023-09-23T15:11:05.303741+0000","last_clean":"2023-09-23T15:11:05.303741+0000","last_became_active":"2023-09-23T14:56:31.640550+0000","last_became_peered":"2023-09-23T14:56:31.640550+0000","last_unstale":"2023-09-23T15:11:05.303741+0000","last_undegraded":"2023-09-23T15:11:05.303741+0000","last_fullsized":"2023-09-23T15:11:05.303741+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:19.275861+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.348094+0000","last_change":"2023-09-23T15:10:47.413934+0000","last_active":"2023-09-23T15:11:04.348094+0000","last_peered":"2023-09-23T15:11:04.348094+0000","last_clean":"2023-09-23T15:11:04.348094+0000","last_became_active":"2023-09-23T15:10:47.413557+0000","last_became_peered":"2023-09-23T15:10:47.413557+0000","last_unstale":"2023-09-23T15:11:04.348094+0000","last_undegraded":"2023-09-23T15:11:04.348094+0000","last_fullsized":"2023-09-23T15:11:04.348094+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.8","version":"158'12608","reported_seq":12674,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:08.170447+0000","last_change":"2023-09-23T15:11:05.308284+0000","last_active":"2023-09-23T15:11:08.170447+0000","last_peered":"2023-09-23T15:11:08.170447+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:11:05.308284+0000","last_became_peered":"2023-09-23T15:11:05.308284+0000","last_unstale":"2023-09-23T15:11:08.170447+0000","last_undegraded":"2023-09-23T15:11:05.303974+0000","last_fullsized":"2023-09-23T15:11:05.303844+0000","mapping_epoch":132,"log_start":"122'2500","ondisk_log_start":"122'2500","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10108,"log_dups_size":0,"ondisk_log_size":10108,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41156608,"num_objects":628,"num_object_clones":0,"num_object_copies":1884,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":628,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":628,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6348,"num_write_kb":223232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":628}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":125,"reported_epoch":157,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:04.354186+0000","last_change":"2023-09-23T15:08:51.258412+0000","last_active":"2023-09-23T15:11:04.354186+0000","last_peered":"2023-09-23T15:11:04.354186+0000","last_clean":"2023-09-23T15:08:45.870012+0000","last_became_active":"2023-09-23T15:08:51.258412+0000","last_became_peered":"2023-09-23T15:08:51.258412+0000","last_unstale":"2023-09-23T15:11:04.354186+0000","last_undegraded":"2023-09-23T15:08:51.246980+0000","last_fullsized":"2023-09-23T15:08:51.246914+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:37.537850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":140,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.508724+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:11:04.508724+0000","last_peered":"2023-09-23T15:11:04.508724+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:11:04.508724+0000","last_undegraded":"2023-09-23T15:11:04.508724+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":516,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345865+0000","last_change":"2023-09-23T15:10:47.401819+0000","last_active":"2023-09-23T15:11:04.345865+0000","last_peered":"2023-09-23T15:11:04.345865+0000","last_clean":"2023-09-23T15:11:04.345865+0000","last_became_active":"2023-09-23T15:10:47.401463+0000","last_became_peered":"2023-09-23T15:10:47.401463+0000","last_unstale":"2023-09-23T15:11:04.345865+0000","last_undegraded":"2023-09-23T15:11:04.345865+0000","last_fullsized":"2023-09-23T15:11:04.345865+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.b","version":"158'12796","reported_seq":12850,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.168282+0000","last_change":"2023-09-23T15:07:30.124976+0000","last_active":"2023-09-23T15:11:08.168282+0000","last_peered":"2023-09-23T15:11:08.168282+0000","last_clean":"2023-09-23T15:11:08.168282+0000","last_became_active":"2023-09-23T15:07:30.124692+0000","last_became_peered":"2023-09-23T15:07:30.124692+0000","last_unstale":"2023-09-23T15:11:08.168282+0000","last_undegraded":"2023-09-23T15:11:08.168282+0000","last_fullsized":"2023-09-23T15:11:08.168282+0000","mapping_epoch":114,"log_start":"122'2700","ondisk_log_start":"122'2700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10096,"log_dups_size":0,"ondisk_log_size":10096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:03.992007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80216064,"num_objects":1224,"num_object_clones":0,"num_object_copies":3672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12796,"num_write_kb":448640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":100,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347255+0000","last_change":"2023-09-23T14:56:31.344084+0000","last_active":"2023-09-23T15:11:04.347255+0000","last_peered":"2023-09-23T15:11:04.347255+0000","last_clean":"2023-09-23T15:11:04.347255+0000","last_became_active":"2023-09-23T14:56:31.343813+0000","last_became_peered":"2023-09-23T14:56:31.343813+0000","last_unstale":"2023-09-23T15:11:04.347255+0000","last_undegraded":"2023-09-23T15:11:04.347255+0000","last_fullsized":"2023-09-23T15:11:04.347255+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:50.715778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":122,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.434625+0000","last_change":"2023-09-23T15:08:51.251419+0000","last_active":"2023-09-23T15:11:04.434625+0000","last_peered":"2023-09-23T15:11:04.434625+0000","last_clean":"2023-09-23T15:08:45.874998+0000","last_became_active":"2023-09-23T15:08:51.251419+0000","last_became_peered":"2023-09-23T15:08:51.251419+0000","last_unstale":"2023-09-23T15:11:04.434625+0000","last_undegraded":"2023-09-23T15:11:04.434625+0000","last_fullsized":"2023-09-23T15:08:51.249901+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:25.327944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":526,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.354059+0000","last_change":"2023-09-23T15:10:47.403858+0000","last_active":"2023-09-23T15:11:04.354059+0000","last_peered":"2023-09-23T15:11:04.354059+0000","last_clean":"2023-09-23T15:11:04.354059+0000","last_became_active":"2023-09-23T15:10:47.400491+0000","last_became_peered":"2023-09-23T15:10:47.400491+0000","last_unstale":"2023-09-23T15:11:04.354059+0000","last_undegraded":"2023-09-23T15:11:04.354059+0000","last_fullsized":"2023-09-23T15:11:04.354059+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.115976862,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.a","version":"158'12707","reported_seq":12760,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.163872+0000","last_change":"2023-09-23T15:07:30.127949+0000","last_active":"2023-09-23T15:11:08.163872+0000","last_peered":"2023-09-23T15:11:08.163872+0000","last_clean":"2023-09-23T15:11:08.163872+0000","last_became_active":"2023-09-23T15:07:30.127659+0000","last_became_peered":"2023-09-23T15:07:30.127659+0000","last_unstale":"2023-09-23T15:11:08.163872+0000","last_undegraded":"2023-09-23T15:11:08.163872+0000","last_fullsized":"2023-09-23T15:11:08.163872+0000","mapping_epoch":114,"log_start":"122'2700","ondisk_log_start":"122'2700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:09.493385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83820544,"num_objects":1279,"num_object_clones":0,"num_object_copies":3837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12707,"num_write_kb":447552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.305572+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:11:05.305572+0000","last_peered":"2023-09-23T15:11:05.305572+0000","last_clean":"2023-09-23T15:11:05.305572+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:11:05.305572+0000","last_undegraded":"2023-09-23T15:11:05.305572+0000","last_fullsized":"2023-09-23T15:11:05.305572+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.310582+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:11:04.310582+0000","last_peered":"2023-09-23T15:11:04.310582+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:11:04.310582+0000","last_undegraded":"2023-09-23T15:11:04.310582+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":517,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344831+0000","last_change":"2023-09-23T15:10:25.167704+0000","last_active":"2023-09-23T15:11:04.344831+0000","last_peered":"2023-09-23T15:11:04.344831+0000","last_clean":"2023-09-23T15:11:04.344831+0000","last_became_active":"2023-09-23T15:09:53.859972+0000","last_became_peered":"2023-09-23T15:09:53.859972+0000","last_unstale":"2023-09-23T15:11:04.344831+0000","last_undegraded":"2023-09-23T15:11:04.344831+0000","last_fullsized":"2023-09-23T15:11:04.344831+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:25.167633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:25.167633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:55:36.115620+0000","scrub_duration":0.022723641999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.d","version":"158'12700","reported_seq":12755,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.135235+0000","last_change":"2023-09-23T15:07:30.124982+0000","last_active":"2023-09-23T15:11:08.135235+0000","last_peered":"2023-09-23T15:11:08.135235+0000","last_clean":"2023-09-23T15:11:08.135235+0000","last_became_active":"2023-09-23T15:07:30.124564+0000","last_became_peered":"2023-09-23T15:07:30.124564+0000","last_unstale":"2023-09-23T15:11:08.135235+0000","last_undegraded":"2023-09-23T15:11:08.135235+0000","last_fullsized":"2023-09-23T15:11:08.135235+0000","mapping_epoch":114,"log_start":"122'2600","ondisk_log_start":"122'2600","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10100,"log_dups_size":0,"ondisk_log_size":10100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:37.931250+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79101976,"num_objects":1208,"num_object_clones":0,"num_object_copies":3624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1208,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12700,"num_write_kb":444993,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.348146+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:11:04.348146+0000","last_peered":"2023-09-23T15:11:04.348146+0000","last_clean":"2023-09-23T15:11:04.348146+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:11:04.348146+0000","last_undegraded":"2023-09-23T15:11:04.348146+0000","last_fullsized":"2023-09-23T15:11:04.348146+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":116,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.311457+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:11:04.311457+0000","last_peered":"2023-09-23T15:11:04.311457+0000","last_clean":"2023-09-23T15:11:04.311457+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:11:04.311457+0000","last_undegraded":"2023-09-23T15:11:04.311457+0000","last_fullsized":"2023-09-23T15:11:04.311457+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"91'161","reported_seq":517,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345359+0000","last_change":"2023-09-23T15:10:30.198919+0000","last_active":"2023-09-23T15:11:04.345359+0000","last_peered":"2023-09-23T15:11:04.345359+0000","last_clean":"2023-09-23T15:11:04.345359+0000","last_became_active":"2023-09-23T15:09:53.868759+0000","last_became_peered":"2023-09-23T15:09:53.868759+0000","last_unstale":"2023-09-23T15:11:04.345359+0000","last_undegraded":"2023-09-23T15:11:04.345359+0000","last_fullsized":"2023-09-23T15:11:04.345359+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:30.198818+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:30.198818+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:52.349088+0000","scrub_duration":0.0099226060000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.c","version":"158'12819","reported_seq":12877,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.052043+0000","last_change":"2023-09-23T15:08:51.256660+0000","last_active":"2023-09-23T15:11:09.052043+0000","last_peered":"2023-09-23T15:11:09.052043+0000","last_clean":"2023-09-23T15:08:50.218263+0000","last_became_active":"2023-09-23T15:08:51.256660+0000","last_became_peered":"2023-09-23T15:08:51.256660+0000","last_unstale":"2023-09-23T15:11:09.052043+0000","last_undegraded":"2023-09-23T15:08:51.253657+0000","last_fullsized":"2023-09-23T15:08:51.253539+0000","mapping_epoch":132,"log_start":"131'6109","ondisk_log_start":"131'6109","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:06.440608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75563008,"num_objects":1153,"num_object_clones":0,"num_object_copies":3459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1153,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12819,"num_write_kb":447104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1153}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.434450+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:11:04.434450+0000","last_peered":"2023-09-23T15:11:04.434450+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:11:04.434450+0000","last_undegraded":"2023-09-23T15:11:04.434450+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":140,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.310799+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:11:04.310799+0000","last_peered":"2023-09-23T15:11:04.310799+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:11:04.310799+0000","last_undegraded":"2023-09-23T15:11:04.310799+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":516,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347520+0000","last_change":"2023-09-23T15:10:47.399902+0000","last_active":"2023-09-23T15:11:04.347520+0000","last_peered":"2023-09-23T15:11:04.347520+0000","last_clean":"2023-09-23T15:11:04.347520+0000","last_became_active":"2023-09-23T15:10:47.399572+0000","last_became_peered":"2023-09-23T15:10:47.399572+0000","last_unstale":"2023-09-23T15:11:04.347520+0000","last_undegraded":"2023-09-23T15:11:04.347520+0000","last_fullsized":"2023-09-23T15:11:04.347520+0000","mapping_epoch":152,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":16,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.f","version":"159'12991","reported_seq":13044,"reported_epoch":159,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.582018+0000","last_change":"2023-09-23T15:07:30.130549+0000","last_active":"2023-09-23T15:11:09.582018+0000","last_peered":"2023-09-23T15:11:09.582018+0000","last_clean":"2023-09-23T15:11:09.582018+0000","last_became_active":"2023-09-23T15:07:30.130216+0000","last_became_peered":"2023-09-23T15:07:30.130216+0000","last_unstale":"2023-09-23T15:11:09.582018+0000","last_undegraded":"2023-09-23T15:11:09.582018+0000","last_fullsized":"2023-09-23T15:11:09.582018+0000","mapping_epoch":114,"log_start":"126'4361","ondisk_log_start":"126'4361","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":8630,"log_dups_size":0,"ondisk_log_size":8630,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:32.502215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80543744,"num_objects":1229,"num_object_clones":0,"num_object_copies":3687,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1229,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12991,"num_write_kb":455040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.508441+0000","last_change":"2023-09-23T15:05:44.719255+0000","last_active":"2023-09-23T15:11:04.508441+0000","last_peered":"2023-09-23T15:11:04.508441+0000","last_clean":"2023-09-23T15:11:04.508441+0000","last_became_active":"2023-09-23T15:05:44.718633+0000","last_became_peered":"2023-09-23T15:05:44.718633+0000","last_unstale":"2023-09-23T15:11:04.508441+0000","last_undegraded":"2023-09-23T15:11:04.508441+0000","last_fullsized":"2023-09-23T15:11:04.508441+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:32.599823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.315192+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:11:04.315192+0000","last_peered":"2023-09-23T15:11:04.315192+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:11:04.315192+0000","last_undegraded":"2023-09-23T15:11:04.315192+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":517,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344839+0000","last_change":"2023-09-23T15:10:49.276388+0000","last_active":"2023-09-23T15:11:04.344839+0000","last_peered":"2023-09-23T15:11:04.344839+0000","last_clean":"2023-09-23T15:11:04.344839+0000","last_became_active":"2023-09-23T15:09:53.869639+0000","last_became_peered":"2023-09-23T15:09:53.869639+0000","last_unstale":"2023-09-23T15:11:04.344839+0000","last_undegraded":"2023-09-23T15:11:04.344839+0000","last_fullsized":"2023-09-23T15:11:04.344839+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":144,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:49.276278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:49.276278+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:48.467104+0000","scrub_duration":0.021880852999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.e","version":"158'12595","reported_seq":12647,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.089960+0000","last_change":"2023-09-23T15:07:30.130268+0000","last_active":"2023-09-23T15:11:09.089960+0000","last_peered":"2023-09-23T15:11:09.089960+0000","last_clean":"2023-09-23T15:11:09.089960+0000","last_became_active":"2023-09-23T15:07:30.129936+0000","last_became_peered":"2023-09-23T15:07:30.129936+0000","last_unstale":"2023-09-23T15:11:09.089960+0000","last_undegraded":"2023-09-23T15:11:09.089960+0000","last_fullsized":"2023-09-23T15:11:09.089960+0000","mapping_epoch":114,"log_start":"131'5910","ondisk_log_start":"131'5910","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6685,"log_dups_size":0,"ondisk_log_size":6685,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:01.865747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80412672,"num_objects":1227,"num_object_clones":0,"num_object_copies":3681,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1227,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12595,"num_write_kb":442304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.360993+0000","last_change":"2023-09-23T15:08:51.251517+0000","last_active":"2023-09-23T15:11:04.360993+0000","last_peered":"2023-09-23T15:11:04.360993+0000","last_clean":"2023-09-23T15:08:45.961507+0000","last_became_active":"2023-09-23T15:08:51.251517+0000","last_became_peered":"2023-09-23T15:08:51.251517+0000","last_unstale":"2023-09-23T15:11:04.360993+0000","last_undegraded":"2023-09-23T15:11:04.360993+0000","last_fullsized":"2023-09-23T15:08:51.246840+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:49.313417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":73,"reported_epoch":158,"state":"active+undersized","last_fresh":"2023-09-23T15:11:05.305046+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:11:05.305046+0000","last_peered":"2023-09-23T15:11:05.305046+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:11:05.305046+0000","last_undegraded":"2023-09-23T15:11:05.305046+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":517,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345941+0000","last_change":"2023-09-23T15:10:47.389034+0000","last_active":"2023-09-23T15:11:04.345941+0000","last_peered":"2023-09-23T15:11:04.345941+0000","last_clean":"2023-09-23T15:11:04.345941+0000","last_became_active":"2023-09-23T15:10:47.388731+0000","last_became_peered":"2023-09-23T15:10:47.388731+0000","last_unstale":"2023-09-23T15:11:04.345941+0000","last_undegraded":"2023-09-23T15:11:04.345941+0000","last_fullsized":"2023-09-23T15:11:04.345941+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":350,"reported_epoch":157,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:04.310173+0000","last_change":"2023-09-23T15:08:51.250559+0000","last_active":"2023-09-23T15:11:04.310173+0000","last_peered":"2023-09-23T15:11:04.310173+0000","last_clean":"2023-09-23T14:57:57.415972+0000","last_became_active":"2023-09-23T15:08:51.250559+0000","last_became_peered":"2023-09-23T15:08:51.250559+0000","last_unstale":"2023-09-23T15:11:04.310173+0000","last_undegraded":"2023-09-23T15:08:51.247621+0000","last_fullsized":"2023-09-23T15:08:51.247489+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:59.638993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.304128+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:11:05.304128+0000","last_peered":"2023-09-23T15:11:05.304128+0000","last_clean":"2023-09-23T15:11:05.304128+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:11:05.304128+0000","last_undegraded":"2023-09-23T15:11:05.304128+0000","last_fullsized":"2023-09-23T15:11:05.304128+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":77,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.333410+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:11:04.333410+0000","last_peered":"2023-09-23T15:11:04.333410+0000","last_clean":"2023-09-23T15:11:04.333410+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:11:04.333410+0000","last_undegraded":"2023-09-23T15:11:04.333410+0000","last_fullsized":"2023-09-23T15:11:04.333410+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.3","version":"158'12357","reported_seq":12418,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.084774+0000","last_change":"2023-09-23T15:11:05.346276+0000","last_active":"2023-09-23T15:11:09.084774+0000","last_peered":"2023-09-23T15:11:09.084774+0000","last_clean":"2023-09-23T15:11:09.084774+0000","last_became_active":"2023-09-23T15:11:05.320718+0000","last_became_peered":"2023-09-23T15:11:05.320718+0000","last_unstale":"2023-09-23T15:11:09.084774+0000","last_undegraded":"2023-09-23T15:11:09.084774+0000","last_fullsized":"2023-09-23T15:11:09.084774+0000","mapping_epoch":114,"log_start":"135'6674","ondisk_log_start":"135'6674","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5683,"log_dups_size":0,"ondisk_log_size":5683,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36208640,"num_objects":552,"num_object_clones":0,"num_object_copies":1656,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":552,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6240,"num_write_kb":217376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":536,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.346050+0000","last_change":"2023-09-23T15:10:58.450902+0000","last_active":"2023-09-23T15:11:04.346050+0000","last_peered":"2023-09-23T15:11:04.346050+0000","last_clean":"2023-09-23T15:11:04.346050+0000","last_became_active":"2023-09-23T15:10:47.387868+0000","last_became_peered":"2023-09-23T15:10:47.387868+0000","last_unstale":"2023-09-23T15:11:04.346050+0000","last_undegraded":"2023-09-23T15:11:04.346050+0000","last_fullsized":"2023-09-23T15:11:04.346050+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:41.528773+0000","scrub_duration":0.0049069459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.5","version":"158'12863","reported_seq":12929,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.084459+0000","last_change":"2023-09-23T15:11:05.345696+0000","last_active":"2023-09-23T15:11:09.084459+0000","last_peered":"2023-09-23T15:11:09.084459+0000","last_clean":"2023-09-23T15:08:50.220468+0000","last_became_active":"2023-09-23T15:11:05.345696+0000","last_became_peered":"2023-09-23T15:11:05.345696+0000","last_unstale":"2023-09-23T15:11:09.084459+0000","last_undegraded":"2023-09-23T15:11:05.305755+0000","last_fullsized":"2023-09-23T15:11:05.305677+0000","mapping_epoch":132,"log_start":"136'7147","ondisk_log_start":"136'7147","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5716,"log_dups_size":0,"ondisk_log_size":5716,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38174720,"num_objects":582,"num_object_clones":0,"num_object_copies":1746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":582,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6490,"num_write_kb":226336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":582}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":667,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:05.305357+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:11:05.305357+0000","last_peered":"2023-09-23T15:11:05.305357+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:11:05.305357+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":117,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.309832+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:11:04.309832+0000","last_peered":"2023-09-23T15:11:04.309832+0000","last_clean":"2023-09-23T15:11:04.309832+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:11:04.309832+0000","last_undegraded":"2023-09-23T15:11:04.309832+0000","last_fullsized":"2023-09-23T15:11:04.309832+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":88,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.324972+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:11:04.324972+0000","last_peered":"2023-09-23T15:11:04.324972+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:11:04.324972+0000","last_undegraded":"2023-09-23T15:11:04.324972+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.344915+0000","last_change":"2023-09-23T15:10:55.364707+0000","last_active":"2023-09-23T15:11:04.344915+0000","last_peered":"2023-09-23T15:11:04.344915+0000","last_clean":"2023-09-23T15:11:04.344915+0000","last_became_active":"2023-09-23T15:10:47.374985+0000","last_became_peered":"2023-09-23T15:10:47.374985+0000","last_unstale":"2023-09-23T15:11:04.344915+0000","last_undegraded":"2023-09-23T15:11:04.344915+0000","last_fullsized":"2023-09-23T15:11:04.344915+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:55.364609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:55.364609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:18.615454+0000","scrub_duration":0.013136897,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.6","version":"158'12606","reported_seq":12668,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.110289+0000","last_change":"2023-09-23T15:11:05.359249+0000","last_active":"2023-09-23T15:11:08.110289+0000","last_peered":"2023-09-23T15:11:08.110289+0000","last_clean":"2023-09-23T15:11:08.110289+0000","last_became_active":"2023-09-23T15:11:05.333469+0000","last_became_peered":"2023-09-23T15:11:05.333469+0000","last_unstale":"2023-09-23T15:11:08.110289+0000","last_undegraded":"2023-09-23T15:11:08.110289+0000","last_fullsized":"2023-09-23T15:11:08.110289+0000","mapping_epoch":114,"log_start":"122'2500","ondisk_log_start":"122'2500","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10106,"log_dups_size":0,"ondisk_log_size":10106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40370176,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6350,"num_write_kb":222912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":768,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.323533+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:11:04.323533+0000","last_peered":"2023-09-23T15:11:04.323533+0000","last_clean":"2023-09-23T15:11:04.323533+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:11:04.323533+0000","last_undegraded":"2023-09-23T15:11:04.323533+0000","last_fullsized":"2023-09-23T15:11:04.323533+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":68,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.323707+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:11:04.323707+0000","last_peered":"2023-09-23T15:11:04.323707+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:11:04.323707+0000","last_undegraded":"2023-09-23T15:11:04.323707+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":93,"reported_epoch":158,"state":"active+undersized","last_fresh":"2023-09-23T15:11:05.304583+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:11:05.304583+0000","last_peered":"2023-09-23T15:11:05.304583+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:11:05.304583+0000","last_undegraded":"2023-09-23T15:11:05.304583+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":404,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.305341+0000","last_change":"2023-09-23T15:05:44.723828+0000","last_active":"2023-09-23T15:11:05.305341+0000","last_peered":"2023-09-23T15:11:05.305341+0000","last_clean":"2023-09-23T15:11:05.305341+0000","last_became_active":"2023-09-23T15:05:44.723221+0000","last_became_peered":"2023-09-23T15:05:44.723221+0000","last_unstale":"2023-09-23T15:11:05.305341+0000","last_undegraded":"2023-09-23T15:11:05.305341+0000","last_fullsized":"2023-09-23T15:11:05.305341+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:39.516164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":528,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345439+0000","last_change":"2023-09-23T15:10:47.376493+0000","last_active":"2023-09-23T15:11:04.345439+0000","last_peered":"2023-09-23T15:11:04.345439+0000","last_clean":"2023-09-23T15:11:04.345439+0000","last_became_active":"2023-09-23T15:10:47.376180+0000","last_became_peered":"2023-09-23T15:10:47.376180+0000","last_unstale":"2023-09-23T15:11:04.345439+0000","last_undegraded":"2023-09-23T15:11:04.345439+0000","last_fullsized":"2023-09-23T15:11:04.345439+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0070307169999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":70,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.342582+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:11:04.342582+0000","last_peered":"2023-09-23T15:11:04.342582+0000","last_clean":"2023-09-23T15:11:04.342582+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:11:04.342582+0000","last_undegraded":"2023-09-23T15:11:04.342582+0000","last_fullsized":"2023-09-23T15:11:04.342582+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":98,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.432347+0000","last_change":"2023-09-23T14:56:31.642947+0000","last_active":"2023-09-23T15:11:04.432347+0000","last_peered":"2023-09-23T15:11:04.432347+0000","last_clean":"2023-09-23T15:11:04.432347+0000","last_became_active":"2023-09-23T14:56:31.642638+0000","last_became_peered":"2023-09-23T14:56:31.642638+0000","last_unstale":"2023-09-23T15:11:04.432347+0000","last_undegraded":"2023-09-23T15:11:04.432347+0000","last_fullsized":"2023-09-23T15:11:04.432347+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:23.965397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"158'12530","reported_seq":12590,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.042909+0000","last_change":"2023-09-23T15:11:05.356513+0000","last_active":"2023-09-23T15:11:09.042909+0000","last_peered":"2023-09-23T15:11:09.042909+0000","last_clean":"2023-09-23T15:11:09.042909+0000","last_became_active":"2023-09-23T15:11:05.320034+0000","last_became_peered":"2023-09-23T15:11:05.320034+0000","last_unstale":"2023-09-23T15:11:09.042909+0000","last_undegraded":"2023-09-23T15:11:09.042909+0000","last_fullsized":"2023-09-23T15:11:09.042909+0000","mapping_epoch":114,"log_start":"135'6952","ondisk_log_start":"135'6952","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39059456,"num_objects":596,"num_object_clones":0,"num_object_copies":1788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6313,"num_write_kb":221088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":517,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347284+0000","last_change":"2023-09-23T15:10:47.381204+0000","last_active":"2023-09-23T15:11:04.347284+0000","last_peered":"2023-09-23T15:11:04.347284+0000","last_clean":"2023-09-23T15:11:04.347284+0000","last_became_active":"2023-09-23T15:10:47.380916+0000","last_became_peered":"2023-09-23T15:10:47.380916+0000","last_unstale":"2023-09-23T15:11:04.347284+0000","last_undegraded":"2023-09-23T15:11:04.347284+0000","last_fullsized":"2023-09-23T15:11:04.347284+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"159'12828","reported_seq":12894,"reported_epoch":159,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.517376+0000","last_change":"2023-09-23T15:11:05.312654+0000","last_active":"2023-09-23T15:11:09.517376+0000","last_peered":"2023-09-23T15:11:09.517376+0000","last_clean":"2023-09-23T15:08:50.245067+0000","last_became_active":"2023-09-23T15:11:05.312654+0000","last_became_peered":"2023-09-23T15:11:05.312654+0000","last_unstale":"2023-09-23T15:11:09.517376+0000","last_undegraded":"2023-09-23T15:11:05.304791+0000","last_fullsized":"2023-09-23T15:11:05.304723+0000","mapping_epoch":132,"log_start":"122'2700","ondisk_log_start":"122'2700","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10128,"log_dups_size":0,"ondisk_log_size":10128,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38076416,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":581,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6477,"num_write_kb":225856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":581}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":369,"reported_epoch":157,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:04.345234+0000","last_change":"2023-09-23T15:08:51.250762+0000","last_active":"2023-09-23T15:11:04.345234+0000","last_peered":"2023-09-23T15:11:04.345234+0000","last_clean":"2023-09-23T15:08:45.869408+0000","last_became_active":"2023-09-23T15:08:51.250762+0000","last_became_peered":"2023-09-23T15:08:51.250762+0000","last_unstale":"2023-09-23T15:11:04.345234+0000","last_undegraded":"2023-09-23T15:08:51.246275+0000","last_fullsized":"2023-09-23T15:08:51.246180+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:38.662811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":101,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.309716+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:11:04.309716+0000","last_peered":"2023-09-23T15:11:04.309716+0000","last_clean":"2023-09-23T15:11:04.309716+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:11:04.309716+0000","last_undegraded":"2023-09-23T15:11:04.309716+0000","last_fullsized":"2023-09-23T15:11:04.309716+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":122,"reported_epoch":158,"state":"active+undersized","last_fresh":"2023-09-23T15:11:05.305950+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:11:05.305950+0000","last_peered":"2023-09-23T15:11:05.305950+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:11:05.305950+0000","last_undegraded":"2023-09-23T15:11:05.305950+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.354491+0000","last_change":"2023-09-23T15:10:47.379908+0000","last_active":"2023-09-23T15:11:04.354491+0000","last_peered":"2023-09-23T15:11:04.354491+0000","last_clean":"2023-09-23T15:11:04.354491+0000","last_became_active":"2023-09-23T15:10:47.379581+0000","last_became_peered":"2023-09-23T15:10:47.379581+0000","last_unstale":"2023-09-23T15:11:04.354491+0000","last_undegraded":"2023-09-23T15:11:04.354491+0000","last_fullsized":"2023-09-23T15:11:04.354491+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.115976862,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":827,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347343+0000","last_change":"2023-09-23T15:04:40.706423+0000","last_active":"2023-09-23T15:11:04.347343+0000","last_peered":"2023-09-23T15:11:04.347343+0000","last_clean":"2023-09-23T15:11:04.347343+0000","last_became_active":"2023-09-23T15:04:40.705905+0000","last_became_peered":"2023-09-23T15:04:40.705905+0000","last_unstale":"2023-09-23T15:11:04.347343+0000","last_undegraded":"2023-09-23T15:11:04.347343+0000","last_fullsized":"2023-09-23T15:11:04.347343+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:56.425370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.325105+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:11:04.325105+0000","last_peered":"2023-09-23T15:11:04.325105+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:11:04.325105+0000","last_undegraded":"2023-09-23T15:11:04.325105+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":86,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.310696+0000","last_change":"2023-09-23T15:04:40.704183+0000","last_active":"2023-09-23T15:11:04.310696+0000","last_peered":"2023-09-23T15:11:04.310696+0000","last_clean":"2023-09-23T15:11:04.310696+0000","last_became_active":"2023-09-23T15:04:40.703818+0000","last_became_peered":"2023-09-23T15:04:40.703818+0000","last_unstale":"2023-09-23T15:11:04.310696+0000","last_undegraded":"2023-09-23T15:11:04.310696+0000","last_fullsized":"2023-09-23T15:11:04.310696+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:35.200088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"158'12909","reported_seq":12971,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.085236+0000","last_change":"2023-09-23T15:11:05.349363+0000","last_active":"2023-09-23T15:11:09.085236+0000","last_peered":"2023-09-23T15:11:09.085236+0000","last_clean":"2023-09-23T15:11:09.085236+0000","last_became_active":"2023-09-23T15:11:05.328058+0000","last_became_peered":"2023-09-23T15:11:05.328058+0000","last_unstale":"2023-09-23T15:11:09.085236+0000","last_undegraded":"2023-09-23T15:11:09.085236+0000","last_fullsized":"2023-09-23T15:11:09.085236+0000","mapping_epoch":114,"log_start":"135'6853","ondisk_log_start":"135'6853","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":6056,"log_dups_size":0,"ondisk_log_size":6056,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38371328,"num_objects":585,"num_object_clones":0,"num_object_copies":1755,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":585,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6524,"num_write_kb":227520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.347163+0000","last_change":"2023-09-23T15:10:47.388729+0000","last_active":"2023-09-23T15:11:04.347163+0000","last_peered":"2023-09-23T15:11:04.347163+0000","last_clean":"2023-09-23T15:11:04.347163+0000","last_became_active":"2023-09-23T15:10:47.388390+0000","last_became_peered":"2023-09-23T15:10:47.388390+0000","last_unstale":"2023-09-23T15:11:04.347163+0000","last_undegraded":"2023-09-23T15:11:04.347163+0000","last_fullsized":"2023-09-23T15:11:04.347163+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0046530740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":119,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.342902+0000","last_change":"2023-09-23T15:06:16.133258+0000","last_active":"2023-09-23T15:11:04.342902+0000","last_peered":"2023-09-23T15:11:04.342902+0000","last_clean":"2023-09-23T15:11:04.342902+0000","last_became_active":"2023-09-23T15:06:16.132900+0000","last_became_peered":"2023-09-23T15:06:16.132900+0000","last_unstale":"2023-09-23T15:11:04.342902+0000","last_undegraded":"2023-09-23T15:11:04.342902+0000","last_fullsized":"2023-09-23T15:11:04.342902+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:36.738648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":652,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.432158+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:11:04.432158+0000","last_peered":"2023-09-23T15:11:04.432158+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:11:04.432158+0000","last_undegraded":"2023-09-23T15:11:04.432158+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.360914+0000","last_change":"2023-09-23T15:06:16.131999+0000","last_active":"2023-09-23T15:11:04.360914+0000","last_peered":"2023-09-23T15:11:04.360914+0000","last_clean":"2023-09-23T15:11:04.360914+0000","last_became_active":"2023-09-23T15:06:16.131107+0000","last_became_peered":"2023-09-23T15:06:16.131107+0000","last_unstale":"2023-09-23T15:11:04.360914+0000","last_undegraded":"2023-09-23T15:11:04.360914+0000","last_fullsized":"2023-09-23T15:11:04.360914+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:06.427308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"158'12619","reported_seq":12682,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:09.032067+0000","last_change":"2023-09-23T15:11:05.386783+0000","last_active":"2023-09-23T15:11:09.032067+0000","last_peered":"2023-09-23T15:11:09.032067+0000","last_clean":"2023-09-23T15:11:09.032067+0000","last_became_active":"2023-09-23T15:11:05.344035+0000","last_became_peered":"2023-09-23T15:11:05.344035+0000","last_unstale":"2023-09-23T15:11:09.032067+0000","last_undegraded":"2023-09-23T15:11:09.032067+0000","last_fullsized":"2023-09-23T15:11:09.032067+0000","mapping_epoch":114,"log_start":"129'5100","ondisk_log_start":"129'5100","created":114,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":7519,"log_dups_size":0,"ondisk_log_size":7519,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40173568,"num_objects":613,"num_object_clones":0,"num_object_copies":1839,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":613,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6361,"num_write_kb":223168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":527,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.527903+0000","last_change":"2023-09-23T15:10:47.376390+0000","last_active":"2023-09-23T15:11:04.527903+0000","last_peered":"2023-09-23T15:11:04.527903+0000","last_clean":"2023-09-23T15:11:04.527903+0000","last_became_active":"2023-09-23T15:10:47.376025+0000","last_became_peered":"2023-09-23T15:10:47.376025+0000","last_unstale":"2023-09-23T15:11:04.527903+0000","last_undegraded":"2023-09-23T15:11:04.527903+0000","last_fullsized":"2023-09-23T15:11:04.527903+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":153,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0081398029999999993,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":588,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.432545+0000","last_change":"2023-09-23T15:05:44.718544+0000","last_active":"2023-09-23T15:11:04.432545+0000","last_peered":"2023-09-23T15:11:04.432545+0000","last_clean":"2023-09-23T15:11:04.432545+0000","last_became_active":"2023-09-23T15:05:44.717996+0000","last_became_peered":"2023-09-23T15:05:44.717996+0000","last_unstale":"2023-09-23T15:11:04.432545+0000","last_undegraded":"2023-09-23T15:11:04.432545+0000","last_fullsized":"2023-09-23T15:11:04.432545+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:10:31.915815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":110,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:05.306026+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:11:05.306026+0000","last_peered":"2023-09-23T15:11:05.306026+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:11:05.306026+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":99,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.508651+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:11:04.508651+0000","last_peered":"2023-09-23T15:11:04.508651+0000","last_clean":"2023-09-23T15:11:04.508651+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:11:04.508651+0000","last_undegraded":"2023-09-23T15:11:04.508651+0000","last_fullsized":"2023-09-23T15:11:04.508651+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"158'12533","reported_seq":12598,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:09.086485+0000","last_change":"2023-09-23T15:11:05.346789+0000","last_active":"2023-09-23T15:11:09.086485+0000","last_peered":"2023-09-23T15:11:09.086485+0000","last_clean":"2023-09-23T15:08:50.227488+0000","last_became_active":"2023-09-23T15:11:05.346789+0000","last_became_peered":"2023-09-23T15:11:05.346789+0000","last_unstale":"2023-09-23T15:11:09.086485+0000","last_undegraded":"2023-09-23T15:11:05.307222+0000","last_fullsized":"2023-09-23T15:11:05.305463+0000","mapping_epoch":132,"log_start":"124'3500","ondisk_log_start":"124'3500","created":114,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":9033,"log_dups_size":0,"ondisk_log_size":9033,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38240256,"num_objects":583,"num_object_clones":0,"num_object_copies":1749,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":583,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":583,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6321,"num_write_kb":220960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[{"shards":"3,4","objects":583}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":88,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.310698+0000","last_change":"2023-09-23T15:04:40.703361+0000","last_active":"2023-09-23T15:11:04.310698+0000","last_peered":"2023-09-23T15:11:04.310698+0000","last_clean":"2023-09-23T15:11:04.310698+0000","last_became_active":"2023-09-23T15:04:40.702972+0000","last_became_peered":"2023-09-23T15:04:40.702972+0000","last_unstale":"2023-09-23T15:11:04.310698+0000","last_undegraded":"2023-09-23T15:11:04.310698+0000","last_fullsized":"2023-09-23T15:11:04.310698+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:13.558301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":147,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.324851+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:11:04.324851+0000","last_peered":"2023-09-23T15:11:04.324851+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:11:04.324851+0000","last_undegraded":"2023-09-23T15:11:04.324851+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":127,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:05.305705+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:11:05.305705+0000","last_peered":"2023-09-23T15:11:05.305705+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:11:05.305705+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":140,"reported_epoch":157,"state":"active+undersized","last_fresh":"2023-09-23T15:11:04.310115+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:11:04.310115+0000","last_peered":"2023-09-23T15:11:04.310115+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:11:04.310115+0000","last_undegraded":"2023-09-23T15:11:04.310115+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":103,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:05.304228+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:11:05.304228+0000","last_peered":"2023-09-23T15:11:05.304228+0000","last_clean":"2023-09-23T15:11:05.304228+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:11:05.304228+0000","last_undegraded":"2023-09-23T15:11:05.304228+0000","last_fullsized":"2023-09-23T15:11:05.304228+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":133,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.309952+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:11:04.309952+0000","last_peered":"2023-09-23T15:11:04.309952+0000","last_clean":"2023-09-23T15:11:04.309952+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:11:04.309952+0000","last_undegraded":"2023-09-23T15:11:04.309952+0000","last_fullsized":"2023-09-23T15:11:04.309952+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.18","version":"158'12606","reported_seq":12672,"reported_epoch":158,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:08.178198+0000","last_change":"2023-09-23T15:11:05.342100+0000","last_active":"2023-09-23T15:11:08.178198+0000","last_peered":"2023-09-23T15:11:08.178198+0000","last_clean":"2023-09-23T15:08:50.213111+0000","last_became_active":"2023-09-23T15:11:05.342100+0000","last_became_peered":"2023-09-23T15:11:05.342100+0000","last_unstale":"2023-09-23T15:11:08.178198+0000","last_undegraded":"2023-09-23T15:11:05.333357+0000","last_fullsized":"2023-09-23T15:11:05.307953+0000","mapping_epoch":157,"log_start":"122'2500","ondisk_log_start":"122'2500","created":157,"last_epoch_clean":115,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":10106,"log_dups_size":0,"ondisk_log_size":10106,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41287680,"num_objects":630,"num_object_clones":0,"num_object_copies":1890,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":630,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6346,"num_write_kb":223232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":630}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":127,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.342369+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:11:04.342369+0000","last_peered":"2023-09-23T15:11:04.342369+0000","last_clean":"2023-09-23T15:11:04.342369+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:11:04.342369+0000","last_undegraded":"2023-09-23T15:11:04.342369+0000","last_fullsized":"2023-09-23T15:11:04.342369+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":142,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.345940+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:11:04.345940+0000","last_peered":"2023-09-23T15:11:04.345940+0000","last_clean":"2023-09-23T15:11:04.345940+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:11:04.345940+0000","last_undegraded":"2023-09-23T15:11:04.345940+0000","last_fullsized":"2023-09-23T15:11:04.345940+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.19","version":"158'12772","reported_seq":12833,"reported_epoch":158,"state":"active+clean","last_fresh":"2023-09-23T15:11:08.957861+0000","last_change":"2023-09-23T15:11:05.406820+0000","last_active":"2023-09-23T15:11:08.957861+0000","last_peered":"2023-09-23T15:11:08.957861+0000","last_clean":"2023-09-23T15:11:08.957861+0000","last_became_active":"2023-09-23T15:11:05.365931+0000","last_became_peered":"2023-09-23T15:11:05.365931+0000","last_unstale":"2023-09-23T15:11:08.957861+0000","last_undegraded":"2023-09-23T15:11:08.957861+0000","last_fullsized":"2023-09-23T15:11:08.957861+0000","mapping_epoch":157,"log_start":"136'7186","ondisk_log_start":"136'7186","created":157,"last_epoch_clean":158,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:07:29.104521+0000","last_clean_scrub_stamp":"2023-09-23T15:07:29.104521+0000","objects_scrubbed":0,"log_size":5586,"log_dups_size":0,"ondisk_log_size":5586,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39321600,"num_objects":600,"num_object_clones":0,"num_object_copies":1800,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":600,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6439,"num_write_kb":225248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":104,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.313284+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:11:04.313284+0000","last_peered":"2023-09-23T15:11:04.313284+0000","last_clean":"2023-09-23T15:11:04.313284+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:11:04.313284+0000","last_undegraded":"2023-09-23T15:11:04.313284+0000","last_fullsized":"2023-09-23T15:11:04.313284+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":126,"reported_epoch":157,"state":"active+clean","last_fresh":"2023-09-23T15:11:04.310221+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:11:04.310221+0000","last_peered":"2023-09-23T15:11:04.310221+0000","last_clean":"2023-09-23T15:11:04.310221+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:11:04.310221+0000","last_undegraded":"2023-09-23T15:11:04.310221+0000","last_fullsized":"2023-09-23T15:11:04.310221+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":26,"stat_sum":{"num_bytes":1254948888,"num_objects":19150,"num_object_clones":0,"num_object_copies":57450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5880,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19150,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":204384,"num_write_kb":7153025,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5266026496,"data_stored":5266014280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":203300,"ondisk_log_size":203300,"up":69,"acting":69,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":79,"acting":79,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":724543,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":114,"num_read_kb":210,"num_write":55,"num_write_kb":550,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":402524,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201477,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469104,"kb_used_data":392168,"kb_used_omap":0,"kb_used_meta":76928,"kb_avail":93276048,"statfs":{"total":95995035648,"available":95514673152,"internally_reserved":0,"allocated":401580032,"data_stored":401034573,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":78774272},"hb_peers":[0,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:58 2023","interfaces":[{"interface":"back","average":{"1min":1.843,"5min":1.9359999999999999,"15min":1.038},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.0720000000000001,"5min":46.814,"15min":46.814},"last":1.4159999999999999},{"interface":"front","average":{"1min":2.7850000000000001,"5min":2.573,"15min":1.2490000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":10.887,"5min":46.966999999999999,"15min":46.966999999999999},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:08:18 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":1.052,"15min":0.71799999999999997},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":47.893999999999998,"5min":47.893999999999998,"15min":47.893999999999998},"last":0.58499999999999996},{"interface":"front","average":{"1min":3.202,"5min":1.107,"15min":0.75800000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":48.164000000000001,"5min":48.164000000000001,"15min":48.164000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:10:58 2023","interfaces":[{"interface":"back","average":{"1min":8.1609999999999996,"5min":6.399,"15min":2.544},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":101.807,"5min":170.63800000000001,"15min":170.63800000000001},"last":1.569},{"interface":"front","average":{"1min":9.6379999999999999,"5min":6.9459999999999997,"15min":2.6920000000000002},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":101.84099999999999,"5min":170.47999999999999,"15min":170.47999999999999},"last":0.53600000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:10:29 2023","interfaces":[{"interface":"back","average":{"1min":2.194,"5min":1.5069999999999999,"15min":0.879},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":6.5999999999999996,"5min":7.1980000000000004,"15min":7.1980000000000004},"last":0.85399999999999998},{"interface":"front","average":{"1min":2.1150000000000002,"5min":1.4419999999999999,"15min":0.95699999999999996},"min":{"1min":0.42199999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":5.0979999999999999,"5min":7.1849999999999996,"15min":7.1849999999999996},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":12.598000000000001,"5min":3.2360000000000002,"15min":1.425},"min":{"1min":0.29699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":150.119,"5min":150.119,"15min":150.119},"last":0.78900000000000003},{"interface":"front","average":{"1min":13.884,"5min":3.6749999999999998,"15min":1.585},"min":{"1min":0.35099999999999998,"5min":0.30399999999999999,"15min":0.23999999999999999},"max":{"1min":150.596,"5min":150.596,"15min":150.596},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":5.4809999999999999,"5min":1.9039999999999999,"15min":1.012},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":33.115000000000002,"5min":33.115000000000002,"15min":33.115000000000002},"last":1.2669999999999999},{"interface":"front","average":{"1min":4.2210000000000001,"5min":1.8819999999999999,"15min":1.0189999999999999},"min":{"1min":0.35899999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":33.314,"5min":33.314,"15min":33.314},"last":0.83799999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":2.7210000000000001,"5min":1.246,"15min":0.81599999999999995},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":10.063000000000001,"5min":10.063000000000001,"15min":10.063000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":4.3819999999999997,"5min":1.873,"15min":1.0289999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":16.007999999999999,"5min":16.007999999999999,"15min":16.007999999999999},"last":0.505}]}]},{"osd":6,"up_from":47,"seq":201863463113,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":369636,"kb_used_data":310684,"kb_used_omap":0,"kb_used_meta":58944,"kb_avail":93375516,"statfs":{"total":95995035648,"available":95616528384,"internally_reserved":0,"allocated":318140416,"data_stored":317607094,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":60358656},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":5.4450000000000003,"15min":2.1779999999999999},"min":{"1min":0.20799999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":22.460999999999999,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":3.996,"5min":5.7450000000000001,"15min":2.3010000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.831,"5min":108.452,"15min":108.452},"last":0.246}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":10.074,"5min":11.917,"15min":4.3849999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":101.17400000000001,"5min":180.374,"15min":180.374},"last":0.30299999999999999},{"interface":"front","average":{"1min":8.8580000000000005,"5min":11.778,"15min":4.3399999999999999},"min":{"1min":0.32200000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":101.483,"5min":181.685,"15min":181.685},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":1.405,"15min":0.871},"min":{"1min":0.47799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.7330000000000005,"5min":17.084,"15min":17.084},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.044,"5min":1.179,"15min":0.80000000000000004},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.9509999999999996,"5min":7.4880000000000004,"15min":7.4880000000000004},"last":0.629}]},{"osd":4,"last update":"Sat Sep 23 15:11:04 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":1.544,"15min":0.86199999999999999},"min":{"1min":0.28000000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.942,"5min":12.942,"15min":12.942},"last":0.28399999999999997},{"interface":"front","average":{"1min":3.008,"5min":1.669,"15min":0.92200000000000004},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.25600000000000001},"max":{"1min":9.7840000000000007,"5min":17.120000000000001,"15min":17.120000000000001},"last":0.45800000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.327,"5min":1.675,"15min":0.92700000000000005},"min":{"1min":0.34899999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.800000000000001,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.44},{"interface":"front","average":{"1min":3.1829999999999998,"5min":1.736,"15min":0.96599999999999997},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.852,"5min":17.224,"15min":17.224},"last":0.60999999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":1.5389999999999999,"5min":1.194,"15min":0.79400000000000004},"min":{"1min":0.33700000000000002,"5min":0.315,"15min":0.28000000000000003},"max":{"1min":4.2610000000000001,"5min":17,"15min":17},"last":2.2269999999999999},{"interface":"front","average":{"1min":1.9810000000000001,"5min":1.135,"15min":0.80400000000000005},"min":{"1min":0.38200000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.2489999999999997,"5min":7.4080000000000004,"15min":7.4080000000000004},"last":2.1909999999999998}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893200,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775160,"kb_used_data":617136,"kb_used_omap":8,"kb_used_meta":158007,"kb_avail":92969992,"statfs":{"total":95995035648,"available":95201271808,"internally_reserved":0,"allocated":631947264,"data_stored":631280265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":161799764},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":7.657,"5min":3.1259999999999999,"15min":1.3100000000000001},"min":{"1min":0.17000000000000001,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":122.20099999999999,"5min":122.20099999999999,"15min":122.20099999999999},"last":4.1319999999999997},{"interface":"front","average":{"1min":7.8289999999999997,"5min":2.7909999999999999,"15min":1.2030000000000001},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":122.071,"5min":122.071,"15min":122.071},"last":1.6559999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:10:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5960000000000001,"5min":1.242,"15min":0.72599999999999998},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.165,"5min":9.1180000000000003,"15min":9.1180000000000003},"last":3.8999999999999999},{"interface":"front","average":{"1min":1.923,"5min":1.619,"15min":1.2110000000000001},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":7.3819999999999997,"5min":11.978,"15min":11.978},"last":1.7010000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":3.496,"5min":1.7749999999999999,"15min":1.024},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":7.6829999999999998},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.2410000000000001,"15min":0.83799999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":5.5389999999999997,"5min":9.0039999999999996,"15min":9.0039999999999996},"last":3.9489999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":1.615,"15min":1.0029999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2050000000000001,"5min":13.49,"15min":13.49},"last":1.7410000000000001},{"interface":"front","average":{"1min":3.0750000000000002,"5min":1.6220000000000001,"15min":1.028},"min":{"1min":0.35399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":11.929,"5min":11.929,"15min":11.929},"last":8.3290000000000006}]},{"osd":6,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.3039999999999998,"5min":1.5609999999999999,"15min":0.98599999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":10.292999999999999,"5min":11.641999999999999,"15min":11.641999999999999},"last":6.4720000000000004},{"interface":"front","average":{"1min":2.3119999999999998,"5min":1.4810000000000001,"15min":0.94999999999999996},"min":{"1min":0.32900000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.3940000000000001,"5min":6.3940000000000001,"15min":6.3940000000000001},"last":2.992}]},{"osd":7,"last update":"Sat Sep 23 15:10:53 2023","interfaces":[{"interface":"back","average":{"1min":2.101,"5min":1.55,"15min":0.96699999999999997},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":6.0739999999999998,"5min":8.2989999999999995,"15min":8.2989999999999995},"last":3.9300000000000002},{"interface":"front","average":{"1min":2.4140000000000001,"5min":1.675,"15min":1.034},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.109,"5min":9.109,"15min":9.109},"last":7.2149999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566147,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":619952,"kb_used_data":489896,"kb_used_omap":11,"kb_used_meta":130036,"kb_avail":93125200,"statfs":{"total":95995035648,"available":95360204800,"internally_reserved":0,"allocated":501653504,"data_stored":500992877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":133157648},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":2.944,"5min":1.2709999999999999,"15min":0.69099999999999995},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":30.402000000000001,"5min":30.402000000000001,"15min":30.402000000000001},"last":0.22800000000000001},{"interface":"front","average":{"1min":2.976,"5min":1.2609999999999999,"15min":0.67700000000000005},"min":{"1min":0.26200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":30.550000000000001,"5min":30.550000000000001,"15min":30.550000000000001},"last":2.1760000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:10:20 2023","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":1.4330000000000001,"15min":0.73699999999999999},"min":{"1min":0.24299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.556,"5min":57.695,"15min":57.695},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.258,"5min":1.6339999999999999,"15min":0.81200000000000006},"min":{"1min":0.27900000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.0330000000000004,"5min":58.387999999999998,"15min":58.387999999999998},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":4.3360000000000003,"5min":1.714,"15min":1.0049999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":52.795999999999999,"5min":52.795999999999999,"15min":52.795999999999999},"last":2.6059999999999999},{"interface":"front","average":{"1min":7.0339999999999998,"5min":2.5270000000000001,"15min":1.2849999999999999},"min":{"1min":0.36099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":68.894999999999996,"5min":68.894999999999996,"15min":68.894999999999996},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":2.2730000000000001,"15min":1.181},"min":{"1min":0.42499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":41.723999999999997,"5min":41.723999999999997,"15min":41.723999999999997},"last":0.69499999999999995},{"interface":"front","average":{"1min":5.5410000000000004,"5min":2.2040000000000002,"15min":1.1899999999999999},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":39.822000000000003,"5min":39.822000000000003,"15min":39.822000000000003},"last":3.4020000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":6.008,"5min":2.2610000000000001,"15min":1.2410000000000001},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":37.834000000000003,"5min":37.834000000000003,"15min":37.834000000000003},"last":2.25},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.9970000000000001,"15min":1.139},"min":{"1min":0.437,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":37.749000000000002,"5min":37.749000000000002,"15min":37.749000000000002},"last":0.49099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.6850000000000001,"15min":1.0329999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.1280000000000001,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":1.0549999999999999},{"interface":"front","average":{"1min":3.0179999999999998,"5min":1.655,"15min":1.0169999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":8.2599999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402621,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307196,"kb_used_data":231348,"kb_used_omap":3,"kb_used_meta":75836,"kb_avail":93437956,"statfs":{"total":95995035648,"available":95680466944,"internally_reserved":0,"allocated":236900352,"data_stored":236286577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":77656233},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.9089999999999998,"5min":2.347,"15min":1.0329999999999999},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":40.578000000000003,"5min":79.611000000000004,"15min":79.611000000000004},"last":3.258},{"interface":"front","average":{"1min":3.3050000000000002,"5min":2.4740000000000002,"15min":1.083},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":43.326000000000001,"5min":79.656000000000006,"15min":79.656000000000006},"last":2.718}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":11.795,"5min":5.8499999999999996,"15min":2.3639999999999999},"min":{"1min":0.24199999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":109.077,"5min":109.077,"15min":109.077},"last":3.3159999999999998},{"interface":"front","average":{"1min":12.923,"5min":6.3159999999999998,"15min":2.387},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":119.72799999999999,"5min":119.72799999999999,"15min":119.72799999999999},"last":3.3919999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.028,"5min":1.085,"15min":0.76100000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":8.9700000000000006,"5min":8.9700000000000006,"15min":8.9700000000000006},"last":0.89000000000000001},{"interface":"front","average":{"1min":2.5310000000000001,"5min":1.464,"15min":0.98199999999999998},"min":{"1min":0.372,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":8.3989999999999991,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":6.3440000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":4.4409999999999998,"5min":1.6850000000000001,"15min":0.98699999999999999},"min":{"1min":0.29899999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":56.960999999999999,"5min":56.960999999999999,"15min":56.960999999999999},"last":3.3610000000000002},{"interface":"front","average":{"1min":5.3239999999999998,"5min":2.0619999999999998,"15min":1.2230000000000001},"min":{"1min":0.36199999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":56.981999999999999,"5min":56.981999999999999,"15min":56.981999999999999},"last":6.4740000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0800000000000001,"5min":1.2310000000000001,"15min":0.85999999999999999},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":11.238,"5min":11.238,"15min":11.238},"last":2.2730000000000001},{"interface":"front","average":{"1min":3.0670000000000002,"5min":1.7230000000000001,"15min":1.131},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":11.254,"5min":11.254,"15min":11.254},"last":6.298}]},{"osd":7,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":3.7509999999999999,"5min":1.8280000000000001,"15min":1.137},"min":{"1min":0.42399999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":6.5140000000000002},{"interface":"front","average":{"1min":2.4990000000000001,"5min":1.2210000000000001,"15min":0.84099999999999997},"min":{"1min":0.38100000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":13.403,"5min":13.403,"15min":13.403},"last":4.3659999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986386,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":753764,"kb_used_data":639132,"kb_used_omap":0,"kb_used_meta":114624,"kb_avail":92991388,"statfs":{"total":95995035648,"available":95223181312,"internally_reserved":0,"allocated":654471168,"data_stored":653937359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":117374976},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:49 2023","interfaces":[{"interface":"back","average":{"1min":5.867,"5min":4.7409999999999997,"15min":1.9379999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":66.605999999999995,"5min":122.69199999999999,"15min":122.69199999999999},"last":0.219},{"interface":"front","average":{"1min":5.7619999999999996,"5min":5.0380000000000003,"15min":2.036},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":66.384,"5min":122.745,"15min":122.745},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:10:49 2023","interfaces":[{"interface":"back","average":{"1min":5.9100000000000001,"5min":4.6079999999999997,"15min":1.9299999999999999},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":69.275000000000006,"5min":109.541,"15min":109.541},"last":0.318},{"interface":"front","average":{"1min":6.29,"5min":4.9390000000000001,"15min":2.1379999999999999},"min":{"1min":0.27800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":73.986000000000004,"5min":109.619,"15min":109.619},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 15:10:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.468,"15min":0.86299999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.423,"5min":7.423,"15min":7.423},"last":0.252},{"interface":"front","average":{"1min":1.5329999999999999,"5min":1.008,"15min":0.74299999999999999},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":6.9279999999999999,"5min":6.9279999999999999,"15min":6.9279999999999999},"last":2.9790000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:10:33 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":1.9990000000000001,"15min":1.0169999999999999},"min":{"1min":0.33100000000000002,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.516,"5min":28.891999999999999,"15min":28.891999999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.8080000000000001,"5min":1.661,"15min":0.92000000000000004},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.26200000000000001},"max":{"1min":8.0790000000000006,"5min":32.100999999999999,"15min":32.100999999999999},"last":6.5419999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:10:06 2023","interfaces":[{"interface":"back","average":{"1min":2.4510000000000001,"5min":1.304,"15min":0.82899999999999996},"min":{"1min":0.35999999999999999,"5min":0.34200000000000003,"15min":0.318},"max":{"1min":10.509,"5min":10.509,"15min":10.509},"last":2.4430000000000001},{"interface":"front","average":{"1min":2.1429999999999998,"5min":1.218,"15min":0.79500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":14.711,"5min":14.711,"15min":14.711},"last":6.3559999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:10:13 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":1.8340000000000001,"15min":1.016},"min":{"1min":0.309,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":18.23,"5min":18.23,"15min":18.23},"last":0.44},{"interface":"front","average":{"1min":1.7769999999999999,"5min":1.506,"15min":0.94699999999999995},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":9.5549999999999997,"5min":14.653,"15min":14.653},"last":1.8680000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724749,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":796388,"kb_used_data":667996,"kb_used_omap":0,"kb_used_meta":128384,"kb_avail":92948764,"statfs":{"total":95995035648,"available":95179534336,"internally_reserved":0,"allocated":684027904,"data_stored":683497952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":131465216},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":1.7350000000000001,"15min":0.94299999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.044,"5min":24.962,"15min":24.962},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.6949999999999998,"5min":1.671,"15min":0.91400000000000003},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":14.529,"5min":25.317,"15min":25.317},"last":1.3420000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":10.397,"5min":3.556,"15min":1.5600000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":97.087999999999994,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.23599999999999999},{"interface":"front","average":{"1min":10.038,"5min":3.3159999999999998,"15min":1.498},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":97.305999999999997,"5min":97.305999999999997,"15min":97.305999999999997},"last":1.3049999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.0499999999999998,"5min":1.8069999999999999,"15min":1.0069999999999999},"min":{"1min":0.36599999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":11.321,"5min":15.265000000000001,"15min":15.265000000000001},"last":0.27500000000000002},{"interface":"front","average":{"1min":1.726,"5min":1.27,"15min":0.83999999999999997},"min":{"1min":0.33100000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.6760000000000002,"5min":6.8390000000000004,"15min":6.8390000000000004},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:10:38 2023","interfaces":[{"interface":"back","average":{"1min":3.1899999999999999,"5min":1.4099999999999999,"15min":0.78500000000000003},"min":{"1min":0.27700000000000002,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":11.539999999999999,"5min":11.539999999999999,"15min":11.539999999999999},"last":0.71799999999999997},{"interface":"front","average":{"1min":2.8740000000000001,"5min":1.409,"15min":0.82099999999999995},"min":{"1min":0.29099999999999998,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":11.385999999999999,"5min":13.6,"15min":13.6},"last":1.3939999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:10:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":1.9550000000000001,"15min":1.008},"min":{"1min":0.40100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.076000000000001,"5min":15.196,"15min":15.196},"last":0.67700000000000005},{"interface":"front","average":{"1min":3.3290000000000002,"5min":1.5669999999999999,"15min":0.92900000000000005},"min":{"1min":0.33300000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.965999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":1.431}]},{"osd":7,"last update":"Sat Sep 23 15:10:27 2023","interfaces":[{"interface":"back","average":{"1min":4.048,"5min":1.919,"15min":1.034},"min":{"1min":0.41899999999999998,"5min":0.35699999999999998,"15min":0.28299999999999997},"max":{"1min":11.353999999999999,"5min":15.301,"15min":15.301},"last":0.73199999999999998},{"interface":"front","average":{"1min":3.8479999999999999,"5min":1.7669999999999999,"15min":0.98199999999999998},"min":{"1min":0.36199999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":13.095000000000001,"5min":13.361000000000001,"15min":13.361000000000001},"last":1.133}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":614662144,"data_stored":614662144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1927151616,"data_stored":1927151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":486871040,"data_stored":486866968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":233242624,"data_stored":233242624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":625541120,"data_stored":625541120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":682299392,"data_stored":682295320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":311099392,"data_stored":311099392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":385159168,"data_stored":385155096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:11:14.990 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:11:14.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pg_num 36 2023-09-23T15:11:15.123 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:15.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:15.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:15.185 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:11:15.185 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18560 5.4 5:24ff48df:::benchmark_data_smithi017_288048_object18559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.019903089, 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20308895499999999, 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.186 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18560 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163378+0000", 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163445+0000", 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6580000000000003e-05 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163462+0000", 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7041000000000002e-05 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163469+0000", 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3359999999999997e-06 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163522+0000", 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2321000000000003e-05 2023-09-23T15:11:15.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163644+0000", 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012266800000000001 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163794+0000", 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015003099999999999 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165622+0000", 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00182759 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165900+0000", 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027825799999999998 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366389+0000", 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20048898100000001 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366440+0000", 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0593000000000002e-05 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366469+0000", 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9448e-05 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18562 5.4 5:263b20e9:::benchmark_data_smithi017_288048_object18561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.01979841299999, 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.203517166, 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18562 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.193 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163484+0000", 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163517+0000", 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2676999999999999e-05 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163526+0000", 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4999999999999999e-06 2023-09-23T15:11:15.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163531+0000", 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9010000000000002e-06 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164584+0000", 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001053165 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164706+0000", 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012267800000000001 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164859+0000", 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152922 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165907+0000", 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001047926 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.166956+0000", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001048542 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366932+0000", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.199975919 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366968+0000", 2023-09-23T15:11:15.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5896000000000003e-05 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.367002+0000", 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4619000000000002e-05 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18578 5.4 5:24fb8b16:::benchmark_data_smithi017_288048_object18577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:11:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.00894169700001, 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.1939555, 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18578 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174341+0000", 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174411+0000", 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9587999999999998e-05 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174422+0000", 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1327e-05 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174427+0000", 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6149999999999999e-06 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174467+0000", 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0268000000000001e-05 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174552+0000", 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4720000000000002e-05 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174641+0000", 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9091999999999993e-05 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175376+0000", 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073561399999999999 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175539+0000", 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016279399999999999 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368206+0000", 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19266702899999999 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368260+0000", 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3872999999999997e-05 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368297+0000", 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7327999999999999e-05 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:46777 5.4 5:2f7b1ca2:::benchmark_data_smithi017_288048_object46776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 193.29926214599999, 2023-09-23T15:11:15.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19520114399999999, 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 46777 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884020+0000", 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884079+0000", 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9199e-05 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884093+0000", 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3969e-05 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884099+0000", 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8220000000000003e-06 2023-09-23T15:11:15.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884150+0000", 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1026000000000002e-05 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884247+0000", 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6985999999999996e-05 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884345+0000", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8879e-05 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885695+0000", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00134965 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885893+0000", 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019828700000000001 2023-09-23T15:11:15.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079139+0000", 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.193245841 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079186+0000", 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7052000000000003e-05 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079222+0000", 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6183999999999998e-05 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106173 5.9 5:9fa66f55:::benchmark_data_smithi017_288048_object106172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 134.46872084200001, 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26060794199999998, 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106173 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714562+0000", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714598+0000", 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5794999999999999e-05 2023-09-23T15:11:15.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714607+0000", 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2240000000000001e-06 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714612+0000", 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1070000000000004e-06 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718260+0000", 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036480330000000002 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718380+0000", 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120047 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718533+0000", 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015235699999999999 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.724954+0000", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064215239999999996 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728531+0000", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003576606 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975076+0000", 2023-09-23T15:11:15.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24654499199999999 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975123+0000", 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7216999999999999e-05 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975171+0000", 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7536e-05 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.219 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106182 5.9 5:932e2a56:::benchmark_data_smithi017_288048_object106181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 134.46172707100001, 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26155082800000001, 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106182 2023-09-23T15:11:15.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721554+0000", 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721601+0000", 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7094e-05 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721617+0000", 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.588e-05 2023-09-23T15:11:15.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721625+0000", 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6120000000000001e-06 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721968+0000", 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034316400000000002 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722089+0000", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121103 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722229+0000", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139791 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728355+0000", 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061265089999999996 2023-09-23T15:11:15.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728751+0000", 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039578699999999998 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983023+0000", 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25427224599999998 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983068+0000", 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5055000000000001e-05 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983107+0000", 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8952999999999998e-05 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126389 5.4 5:26968410:::benchmark_data_smithi017_288048_object14620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:15.226 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.94234645900001, 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236534309, 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126389 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240936+0000", 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240942+0000", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.212e-06 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240953+0000", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1722999999999999e-05 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240961+0000", 2023-09-23T15:11:15.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2060000000000001e-06 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241222+0000", 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026150199999999999 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241334+0000", 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111864 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241432+0000", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8293000000000002e-05 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253717+0000", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012284982999999999 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259156+0000", 2023-09-23T15:11:15.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054391600000000002 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477373+0000", 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218216728 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477429+0000", 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5939000000000001e-05 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477471+0000", 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2169000000000002e-05 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126392 5.4 5:22950050:::benchmark_data_smithi017_288048_object14623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.942074924, 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236365035, 2023-09-23T15:11:15.233 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126392 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241208+0000", 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241213+0000", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2070000000000003e-06 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241223+0000", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6930000000000002e-06 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241229+0000", 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8490000000000002e-06 2023-09-23T15:11:15.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249366+0000", 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081375960000000004 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249518+0000", 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015237200000000001 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249654+0000", 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013526400000000001 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253846+0000", 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041924930000000003 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259182+0000", 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053355219999999997 2023-09-23T15:11:15.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477502+0000", 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21831988299999999 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477543+0000", 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1448e-05 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477574+0000", 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0459000000000001e-05 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126405 5.4 5:2fffe5cb:::benchmark_data_smithi017_288048_object14636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 108.921395495, 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21577380400000001, 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.240 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126405 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261886+0000", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261894+0000", 2023-09-23T15:11:15.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0989999999999999e-06 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261912+0000", 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8020999999999999e-05 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261918+0000", 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6069999999999998e-06 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261956+0000", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7832e-05 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262070+0000", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011383399999999999 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262173+0000", 2023-09-23T15:11:15.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103539 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.267122+0000", 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049492349999999997 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.269803+0000", 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002680375 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477601+0000", 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.207798129 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477633+0000", 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2582000000000001e-05 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477662+0000", 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8342000000000001e-05 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154970 5.4 5:2dd36d32:::benchmark_data_smithi017_288048_object43201:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.775375206999996, 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.201665601, 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.247 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154970 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407907+0000", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407914+0000", 2023-09-23T15:11:15.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2860000000000002e-06 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407934+0000", 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9840999999999999e-05 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407942+0000", 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1790000000000008e-06 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407997+0000", 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5386000000000001e-05 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408150+0000", 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015222799999999999 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408276+0000", 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621600000000001 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414981+0000", 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067046579999999996 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414993+0000", 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2656e-05 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609483+0000", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19448981700000001 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609532+0000", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8523000000000003e-05 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609574+0000", 2023-09-23T15:11:15.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2299999999999998e-05 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154974 5.4 5:2790b49c:::benchmark_data_smithi017_288048_object43205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.772745998999994, 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19913220500000001, 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.254 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154974 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410536+0000", 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410542+0000", 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.417e-06 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410554+0000", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1855000000000001e-05 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410559+0000", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6640000000000003e-06 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411338+0000", 2023-09-23T15:11:15.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077873099999999995 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411463+0000", 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124938 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411583+0000", 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119839 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414997+0000", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034144940000000001 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.417807+0000", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028097830000000002 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609605+0000", 2023-09-23T15:11:15.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.191798363 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609639+0000", 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3667999999999997e-05 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609670+0000", 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0574999999999998e-05 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.260 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166946 5.4 5:2ea27988:::benchmark_data_smithi017_288048_object55177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 57.096690557000002, 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28508372300000001, 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.261 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166946 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086592+0000", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086600+0000", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4349999999999997e-06 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086612+0000", 2023-09-23T15:11:15.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2339e-05 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086619+0000", 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5060000000000001e-06 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087518+0000", 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000898762 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087616+0000", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8196999999999995e-05 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087683+0000", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6988000000000003e-05 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091939+0000", 2023-09-23T15:11:15.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042566660000000001 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095384+0000", 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003444866 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371564+0000", 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27617948799999997 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371623+0000", 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9219999999999999e-05 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371677+0000", 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3529999999999997e-05 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166951 5.4 5:254d5c06:::benchmark_data_smithi017_288048_object55182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 57.091616954999999, 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28024638499999999, 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166951 2023-09-23T15:11:15.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091665+0000", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091672+0000", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091685+0000", 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3312000000000001e-05 2023-09-23T15:11:15.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091692+0000", 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4989999999999999e-06 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091958+0000", 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026625599999999998 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092057+0000", 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9019999999999997e-05 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092171+0000", 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114345 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098893+0000", 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067218640000000001 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100313+0000", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014193369999999999 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371840+0000", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27152736500000002 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371876+0000", 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6191e-05 2023-09-23T15:11:15.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371913+0000", 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6745999999999997e-05 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:175544 5.4 5:26fea254:::benchmark_data_smithi017_288048_object63775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:11:15.274 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 45.460407138000001, 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18268135499999999, 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 175544 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:11:15.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722874+0000", 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722882+0000", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6249999999999998e-06 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722903+0000", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1369000000000002e-05 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722911+0000", 2023-09-23T15:11:15.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3560000000000004e-06 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722956+0000", 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5698000000000002e-05 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723091+0000", 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013489599999999999 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723244+0000", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152387 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.728221+0000", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049771060000000002 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.732170+0000", 2023-09-23T15:11:15.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039488830000000003 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905471+0000", 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17330090500000001 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905520+0000", 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9104999999999998e-05 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905558+0000", 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8028999999999998e-05 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:176231 5.9 5:95f16d04:::benchmark_data_smithi017_288048_object64462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 44.498550758999997, 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17770085299999999, 2023-09-23T15:11:15.281 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 176231 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684732+0000", 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684735+0000", 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1430000000000001e-06 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684738+0000", 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5710000000000002e-06 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684742+0000", 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1459999999999998e-06 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688383+0000", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036404229999999998 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688504+0000", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121162 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688627+0000", 2023-09-23T15:11:15.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123273 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691418+0000", 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002790837 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691423+0000", 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2460000000000003e-06 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862340+0000", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17091711200000001 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862391+0000", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0390999999999999e-05 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862434+0000", 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.286e-05 2023-09-23T15:11:15.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187773 5.9 5:98d8c468:::benchmark_data_smithi017_288048_object76004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 28.845769798999999, 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20575924800000001, 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187773 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:11:15.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337512+0000", 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337520+0000", 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8520000000000004e-06 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337539+0000", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9249000000000002e-05 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337547+0000", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3399999999999998e-06 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337601+0000", 2023-09-23T15:11:15.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4172000000000002e-05 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337728+0000", 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126198 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337851+0000", 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012301799999999999 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.341076+0000", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003225136 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342051+0000", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097561399999999997 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543167+0000", 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20111568099999999 2023-09-23T15:11:15.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543222+0000", 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4877000000000003e-05 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543273+0000", 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0952000000000003e-05 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.294 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187774 5.9 5:9a77e4c2:::benchmark_data_smithi017_288048_object76005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 28.840742693999999, 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20974965300000001, 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.295 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187774 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342539+0000", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342545+0000", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342558+0000", 2023-09-23T15:11:15.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2813000000000001e-05 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342569+0000", 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0916e-05 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342603+0000", 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4600000000000001e-05 2023-09-23T15:11:15.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342700+0000", 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6618000000000002e-05 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342775+0000", 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5092000000000005e-05 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.345881+0000", 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031061130000000002 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.346007+0000", 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012578200000000001 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552158+0000", 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20615127 2023-09-23T15:11:15.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552231+0000", 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2809e-05 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552290+0000", 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9240000000000002e-05 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197695 5.9 5:9f6a849a:::benchmark_data_smithi017_288048_object85926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.15258311, 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.204857551, 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197695 2023-09-23T15:11:15.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030699+0000", 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030707+0000", 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8480000000000001e-06 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030717+0000", 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414999999999999e-05 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030724+0000", 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9360000000000002e-06 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031186+0000", 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046191899999999998 2023-09-23T15:11:15.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031291+0000", 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010539000000000001 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031395+0000", 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010343199999999999 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.034429+0000", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030340480000000001 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039920+0000", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054914489999999998 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235467+0000", 2023-09-23T15:11:15.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19554693500000001 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235519+0000", 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1835000000000001e-05 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235558+0000", 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8813000000000003e-05 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.308 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197697 5.9 5:93334a62:::benchmark_data_smithi017_288048_object85928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.152550659999999, 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.216589857, 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.309 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197697 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:11:15.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030732+0000", 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030735+0000", 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6709999999999999e-06 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030741+0000", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0909999999999998e-06 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030745+0000", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2579999999999997e-06 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.036909+0000", 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061638029999999998 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037024+0000", 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011513 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037142+0000", 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118171 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039953+0000", 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002810707 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.040002+0000", 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8758999999999999e-05 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247229+0000", 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20722660500000001 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247277+0000", 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8078999999999998e-05 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247323+0000", 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6004000000000002e-05 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200855 5.4 5:2c8e5777:::benchmark_data_smithi017_288048_object89086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.035240308000001, 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37976222399999998, 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200855 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.316 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:11:15.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148042+0000", 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148048+0000", 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2020000000000001e-06 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148058+0000", 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5910000000000002e-06 2023-09-23T15:11:15.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148063+0000", 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8760000000000004e-06 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148114+0000", 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1851999999999998e-05 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:11:15.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148231+0000", 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011667699999999999 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148336+0000", 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010487799999999999 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.151117+0000", 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027812620000000001 2023-09-23T15:11:15.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.152434+0000", 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013172259999999999 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527791+0000", 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37535686699999998 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:11:15.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527805+0000", 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4005999999999999e-05 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:11:15.322 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:11:15.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:15.323 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:14 smithi017 ceph-mon[215646]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:11:15.323 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:14 smithi017 ceph-mon[222878]: from='client.54871 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:11:15.324 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:15.392 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:15.494 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:15.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:15.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:15.598 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:15.659 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:15.778 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:15.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:15.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:15.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:15.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:15 smithi017 ceph-mon[215646]: pgmap v394: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.33k op/s; 5576/52374 objects degraded (10.647%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:15.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:15 smithi017 ceph-mon[222878]: pgmap v394: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.33k op/s; 5576/52374 objects degraded (10.647%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:15.953 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:16.038 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:16.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:16.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:15 smithi027 ceph-mon[195233]: pgmap v394: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 1.1 GiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.33k op/s; 5576/52374 objects degraded (10.647%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:16.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:16.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:11:16.208 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:16.298 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:16.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:16.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:16.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:16.609 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:16.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:16.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:16.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:16.765 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:16.778 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:16.858 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:11:16.914 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:16.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:17.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1454032169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5576/52374 objects degraded (10.647%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:17.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1454032169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:16 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5576/52374 objects degraded (10.647%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:17.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1454032169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:16 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5576/52374 objects degraded (10.647%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:17.110 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:17.196 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:17.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:17.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:17.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:17.354 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:17.610 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:17.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:17.711 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:17.716 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:17.778 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:17.863 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:17.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:17.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: pgmap v395: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 904 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s wr, 1.49k op/s; 4350/43236 objects degraded (10.061%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: failed to encode map e160 with expected crc 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[222878]: failed to encode map e160 with expected crc 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: pgmap v395: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 904 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s wr, 1.49k op/s; 4350/43236 objects degraded (10.061%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:17.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:17.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: failed to encode map e160 with expected crc 2023-09-23T15:11:17.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:17.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T15:11:17.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:17 smithi017 ceph-mon[215646]: failed to encode map e160 with expected crc 2023-09-23T15:11:17.971 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:18.049 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: pgmap v395: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 904 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 22 MiB/s wr, 1.49k op/s; 4350/43236 objects degraded (10.061%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:18.049 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:18.049 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: failed to encode map e160 with expected crc 2023-09-23T15:11:18.049 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:18.050 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: osdmap e160: 8 total, 7 up, 8 in 2023-09-23T15:11:18.050 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:17 smithi027 ceph-mon[195233]: failed to encode map e160 with expected crc 2023-09-23T15:11:18.054 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:18.095 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:18.124 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:18.148 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:18.189 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:18.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:18] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T15:11:18.193 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:18.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:18.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:18.234 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:18.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.309 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:11:18.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:18.360 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:18.424 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:18.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:18.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:18.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:18.597 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:18.714 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:18.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:18.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:18.881 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:19.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:18 smithi027 ceph-mon[195233]: 5.1 scrub starts 2023-09-23T15:11:19.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:18 smithi027 ceph-mon[195233]: 5.1 scrub ok 2023-09-23T15:11:19.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:18 smithi027 ceph-mon[195233]: 5.3 scrub starts 2023-09-23T15:11:19.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:18 smithi027 ceph-mon[195233]: 5.3 scrub ok 2023-09-23T15:11:19.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3440839908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:19.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[215646]: 5.1 scrub starts 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[215646]: 5.1 scrub ok 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[215646]: 5.3 scrub starts 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[215646]: 5.3 scrub ok 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3440839908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[222878]: 5.1 scrub starts 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[222878]: 5.1 scrub ok 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[222878]: 5.3 scrub starts 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[222878]: 5.3 scrub ok 2023-09-23T15:11:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3440839908' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:19.221 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:19.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:19.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:19.353 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:19.397 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:19.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:19.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:19.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:19.588 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:19.706 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:19.720 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:19.749 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:11:19.775 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:19.776 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:19.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:19.874 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:19.955 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:19.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:20.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:20.087 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:19 smithi027 ceph-mon[195233]: pgmap v397: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.49k op/s; 4021/38460 objects degraded (10.455%) 2023-09-23T15:11:20.087 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:19 smithi027 ceph-mon[195233]: 5.13 scrub starts 2023-09-23T15:11:20.087 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:19 smithi027 ceph-mon[195233]: 5.13 scrub ok 2023-09-23T15:11:20.089 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:20.141 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:20.150 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[215646]: pgmap v397: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.49k op/s; 4021/38460 objects degraded (10.455%) 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[215646]: 5.13 scrub starts 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[215646]: 5.13 scrub ok 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[222878]: pgmap v397: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 804 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 25 MiB/s wr, 1.49k op/s; 4021/38460 objects degraded (10.455%) 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[222878]: 5.13 scrub starts 2023-09-23T15:11:20.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:19 smithi017 ceph-mon[222878]: 5.13 scrub ok 2023-09-23T15:11:20.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:20.333 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:20.438 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:20.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:20.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:20.627 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:20.730 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:20.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:20.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:20.803 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:20.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:20.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[215646]: 1.0 scrub starts 2023-09-23T15:11:20.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[215646]: 1.0 scrub ok 2023-09-23T15:11:20.887 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[215646]: 5.2 scrub starts 2023-09-23T15:11:20.935 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:20 smithi027 ceph-mon[195233]: 1.0 scrub starts 2023-09-23T15:11:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:20 smithi027 ceph-mon[195233]: 1.0 scrub ok 2023-09-23T15:11:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:20 smithi027 ceph-mon[195233]: 5.2 scrub starts 2023-09-23T15:11:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:20 smithi027 ceph-mon[195233]: 5.2 scrub ok 2023-09-23T15:11:21.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[215646]: 5.2 scrub ok 2023-09-23T15:11:21.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[222878]: 1.0 scrub starts 2023-09-23T15:11:21.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[222878]: 1.0 scrub ok 2023-09-23T15:11:21.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[222878]: 5.2 scrub starts 2023-09-23T15:11:21.171 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:20 smithi017 ceph-mon[222878]: 5.2 scrub ok 2023-09-23T15:11:21.174 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:21.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:21.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:21.251 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:11:21.251 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:11:21.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:21.344 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:21.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:11:22.014 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 5 pg_num to 36 2023-09-23T15:11:22.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: pgmap v398: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: 5.12 scrub starts 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: 5.12 scrub ok 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4009584134' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2762/27816 objects degraded (9.930%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: pgmap v398: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: 5.12 scrub starts 2023-09-23T15:11:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: 5.12 scrub ok 2023-09-23T15:11:22.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4009584134' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:21 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2762/27816 objects degraded (9.930%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: pgmap v398: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: 5.12 scrub starts 2023-09-23T15:11:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: 5.12 scrub ok 2023-09-23T15:11:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4009584134' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T15:11:22.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:21 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2762/27816 objects degraded (9.930%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: 5.11 scrub starts 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: 5.11 scrub ok 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: pgmap v400: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: 5.11 scrub starts 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: 5.11 scrub ok 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: pgmap v400: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:23.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T15:11:23.294 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:23.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:23.297 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:23.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: 5.11 scrub starts 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: 5.11 scrub ok 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num", "val": "36"}]': finished 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: osdmap e161: 8 total, 7 up, 8 in 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: failed to encode map e161 with expected crc 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: pgmap v400: 119 pgs: 15 active+undersized+degraded, 26 active+undersized, 78 active+clean; 582 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2762/27816 objects degraded (9.930%) 2023-09-23T15:11:23.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T15:11:23.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:23.427 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:23.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:11:23.577 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:23.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:23.580 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:23.670 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:23.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:23.713 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:23.820 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:23.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:23.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:23.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:23.991 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:23 smithi027 ceph-mon[195233]: failed to encode map e162 with expected crc 2023-09-23T15:11:23.991 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:23 smithi027 ceph-mon[195233]: failed to encode map e162 with expected crc 2023-09-23T15:11:23.991 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:23 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T15:11:23.991 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:23 smithi027 ceph-mon[195233]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T15:11:24.002 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:24.088 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:24.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:24.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[222878]: failed to encode map e162 with expected crc 2023-09-23T15:11:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[222878]: failed to encode map e162 with expected crc 2023-09-23T15:11:24.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T15:11:24.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[222878]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T15:11:24.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[215646]: failed to encode map e162 with expected crc 2023-09-23T15:11:24.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[215646]: failed to encode map e162 with expected crc 2023-09-23T15:11:24.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T15:11:24.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:23 smithi017 ceph-mon[215646]: osdmap e162: 8 total, 7 up, 8 in 2023-09-23T15:11:24.202 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:24.262 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:24.447 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:24.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:24.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:24.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:24.601 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: 5.6 scrub starts 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: 5.6 scrub ok 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: pgmap v403: 129 pgs: 9 unknown, 3 peering, 15 active+undersized+degraded, 26 active+undersized, 76 active+clean; 468 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2471/22338 objects degraded (11.062%) 2023-09-23T15:11:24.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1147053521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: 5.6 scrub starts 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: 5.6 scrub ok 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: failed to encode map e163 with expected crc 2023-09-23T15:11:24.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: pgmap v403: 129 pgs: 9 unknown, 3 peering, 15 active+undersized+degraded, 26 active+undersized, 76 active+clean; 468 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2471/22338 objects degraded (11.062%) 2023-09-23T15:11:24.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1147053521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:24.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:24 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:24.949 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:11:25.295 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:25.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:25.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: 5.6 scrub starts 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: 5.6 scrub ok 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: failed to encode map e163 with expected crc 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: osdmap e163: 8 total, 7 up, 8 in 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: failed to encode map e163 with expected crc 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: failed to encode map e163 with expected crc 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: failed to encode map e163 with expected crc 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: failed to encode map e163 with expected crc 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: pgmap v403: 129 pgs: 9 unknown, 3 peering, 15 active+undersized+degraded, 26 active+undersized, 76 active+clean; 468 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 2471/22338 objects degraded (11.062%) 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1147053521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:25.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:24 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:25.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:25.446 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:25.510 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:25.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:25.518 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:25.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:25.719 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:25.875 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:25.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:25.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:25.909 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:25.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:26.044 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: 5.1 scrub starts 2023-09-23T15:11:26.044 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: 5.1 scrub ok 2023-09-23T15:11:26.044 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.046 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.046 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.046 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: 5.3 scrub starts 2023-09-23T15:11:26.046 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:25 smithi027 ceph-mon[195233]: 5.3 scrub ok 2023-09-23T15:11:26.060 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:26.069 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: 5.1 scrub starts 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: 5.1 scrub ok 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: 5.3 scrub starts 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[215646]: 5.3 scrub ok 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: 5.1 scrub starts 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: 5.1 scrub ok 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: osdmap e164: 8 total, 7 up, 8 in 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: failed to encode map e164 with expected crc 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: 5.3 scrub starts 2023-09-23T15:11:26.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:25 smithi017 ceph-mon[222878]: 5.3 scrub ok 2023-09-23T15:11:26.173 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:26.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:26.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:26.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:26.358 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:26.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:26.394 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:26.425 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:26.464 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:26.489 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:26.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:26.498 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:26.500 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:26.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:26.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:11:26.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:26.666 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:26.866 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:26.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:26.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:26.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.e scrub starts 2023-09-23T15:11:26.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.e scrub ok 2023-09-23T15:11:26.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.23 scrub starts 2023-09-23T15:11:26.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.23 scrub ok 2023-09-23T15:11:26.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: pgmap v405: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:26.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2053797352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:26.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.f scrub starts 2023-09-23T15:11:26.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: 5.f scrub ok 2023-09-23T15:11:26.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:26 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2471/22338 objects degraded (11.062%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:26.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:27.048 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:27.144 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:27.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:27.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:27.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:27.336 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:27.434 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:27.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.e scrub starts 2023-09-23T15:11:27.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.e scrub ok 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.23 scrub starts 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.23 scrub ok 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: pgmap v405: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2053797352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.f scrub starts 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: 5.f scrub ok 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2471/22338 objects degraded (11.062%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.e scrub starts 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.e scrub ok 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.23 scrub starts 2023-09-23T15:11:27.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.23 scrub ok 2023-09-23T15:11:27.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: pgmap v405: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:27.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2053797352' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:27.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.f scrub starts 2023-09-23T15:11:27.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: 5.f scrub ok 2023-09-23T15:11:27.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:26 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2471/22338 objects degraded (11.062%), 15 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:27.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:27.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:27.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:27.591 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:27.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:11:28.078 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:28.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:28.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:28.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:28] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T15:11:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.235 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:28.235 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:28.242 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 111766 objects 2023-09-23T15:11:28.242 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :140.285 2023-09-23T15:11:28.256 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:28.405 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:28.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:28.489 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:28.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:28.523 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:11:28.523 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:11:28.523 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T15:11:28.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T15:11:28.538 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:28.583 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:28.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:28.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:28.717 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:28.761 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:28.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:28.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:28.869 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:28.895 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:28.901 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:11:28.901 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:11:28.901 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:11:28.901 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:11:28.901 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:11:28.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:28.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:28.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:29.046 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:29.079 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:29.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.16 scrub starts 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.16 scrub ok 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 1.6 scrub ok 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.2 scrub starts 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.2 scrub ok 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: pgmap v406: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.21 scrub starts 2023-09-23T15:11:29.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[215646]: 5.21 scrub ok 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.16 scrub starts 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.16 scrub ok 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 1.6 scrub ok 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.2 scrub starts 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.2 scrub ok 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: pgmap v406: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.21 scrub starts 2023-09-23T15:11:29.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:29 smithi017 ceph-mon[222878]: 5.21 scrub ok 2023-09-23T15:11:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.16 scrub starts 2023-09-23T15:11:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.16 scrub ok 2023-09-23T15:11:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 1.6 scrub ok 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.2 scrub starts 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.2 scrub ok 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: pgmap v406: 129 pgs: 3 unknown, 3 peering, 17 active+undersized+degraded, 26 active+undersized, 80 active+clean; 322 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 1588/15333 objects degraded (10.357%); 0 B/s, 0 objects/s recovering 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.21 scrub starts 2023-09-23T15:11:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:29 smithi027 ceph-mon[195233]: 5.21 scrub ok 2023-09-23T15:11:29.585 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:29.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:29.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:29.713 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T15:11:29.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T15:11:29.718 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:29.766 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:29.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:29.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:29.900 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:29.945 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:29.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:30.052 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:30.078 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:30.122 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:30.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:30.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:30.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:11:30.183 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:30.229 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:30.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:11:30.255 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:30.299 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:30.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 5.22 scrub starts 2023-09-23T15:11:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 5.22 scrub ok 2023-09-23T15:11:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 5.1e scrub starts 2023-09-23T15:11:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 5.1e scrub ok 2023-09-23T15:11:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:11:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:30 smithi027 ceph-mon[195233]: 1.5 scrub ok 2023-09-23T15:11:30.377 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:30.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:30.434 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 5.22 scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 5.22 scrub ok 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 5.1e scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 5.1e scrub ok 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[215646]: 1.5 scrub ok 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 5.22 scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 5.22 scrub ok 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 5.1e scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 5.1e scrub ok 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:11:30.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:30 smithi017 ceph-mon[222878]: 1.5 scrub ok 2023-09-23T15:11:30.478 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:30.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:30.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:30.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:30.614 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:30.668 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:30.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:30.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:30.805 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:31.167 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:11:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[215646]: pgmap v407: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[215646]: 5.1f scrub starts 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[215646]: 5.1f scrub ok 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1385333214' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[222878]: pgmap v407: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[222878]: 5.1f scrub starts 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[222878]: 5.1f scrub ok 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1385333214' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:31 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:31 smithi027 ceph-mon[195233]: pgmap v407: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:31 smithi027 ceph-mon[195233]: 5.1f scrub starts 2023-09-23T15:11:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:31 smithi027 ceph-mon[195233]: 5.1f scrub ok 2023-09-23T15:11:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1385333214' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:31 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:11:31.763 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:31.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:31.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:31.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:31.895 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:32.044 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:32.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:32.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:32.183 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:32.233 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:32.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:32.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:32.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:32.368 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:32 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:32.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:32 smithi027 ceph-mon[195233]: failed to encode map e165 with expected crc 2023-09-23T15:11:32.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:32 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:32.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:32 smithi027 ceph-mon[195233]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T15:11:32.369 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 60/336 objects degraded (17.857%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:32.371 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:32.419 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:32.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:32.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:32.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:32.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[215646]: failed to encode map e165 with expected crc 2023-09-23T15:11:32.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:32.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[215646]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T15:11:32.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 60/336 objects degraded (17.857%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:32.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:32.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[222878]: failed to encode map e165 with expected crc 2023-09-23T15:11:32.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:11:32.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[222878]: osdmap e165: 8 total, 7 up, 8 in 2023-09-23T15:11:32.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 60/336 objects degraded (17.857%), 12 pgs degraded, 32 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:32.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:32.562 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:33.118 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:32 smithi017 conmon[215622]: 2023-09-23T15:11:32.876+0000 7f618a1a9700 -1 mon.a@0(leader).osd e165 definitely_dead 0 2023-09-23T15:11:33.158 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.7. 2023-09-23T15:11:33.414 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:33.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[215646]: pgmap v409: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:33.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3470726869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[215646]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T15:11:33.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[222878]: pgmap v409: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:33.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3470726869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:33 smithi017 ceph-mon[222878]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T15:11:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:33.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:33.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:33.547 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:33 smithi027 ceph-mon[195233]: pgmap v409: 129 pgs: 12 active+undersized+degraded, 31 active+undersized, 86 active+clean; 9.9 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 60/336 objects degraded (17.857%) 2023-09-23T15:11:33.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3470726869' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:33 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T15:11:33.548 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:33 smithi027 ceph-mon[195233]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T15:11:33.550 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:33.601 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:33.609 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:33.641 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:33.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:33.739 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:33.938 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:33.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:33.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:34.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:34.076 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:34.127 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:34.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:34.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: osdmap e166: 8 total, 7 up, 7 in 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.259 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[215646]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: osdmap e166: 8 total, 7 up, 7 in 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.260 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:34 smithi017 ceph-mon[222878]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.262 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: osdmap e166: 8 total, 7 up, 7 in 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:34 smithi027 ceph-mon[195233]: failed to encode map e166 with expected crc 2023-09-23T15:11:34.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:11:34.861 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:34.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:34.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:34.901 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:34.966 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:34.998 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:35.050 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:35.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:35.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:35.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:35.192 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_0' removed 2023-09-23T15:11:35.278 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: osdmap e167: 8 total, 7 up, 7 in 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: pgmap v412: 129 pgs: 4 remapped+peering, 2 unknown, 4 peering, 12 active+undersized+degraded, 30 active+undersized, 77 active+clean; 9.9 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 60/333 objects degraded (18.018%) 2023-09-23T15:11:35.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:11:35.280 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.280 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:35.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: osdmap e167: 8 total, 7 up, 7 in 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: pgmap v412: 129 pgs: 4 remapped+peering, 2 unknown, 4 peering, 12 active+undersized+degraded, 30 active+undersized, 77 active+clean; 9.9 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 60/333 objects degraded (18.018%) 2023-09-23T15:11:35.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: osdmap e167: 8 total, 7 up, 7 in 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: failed to encode map e167 with expected crc 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: pgmap v412: 129 pgs: 4 remapped+peering, 2 unknown, 4 peering, 12 active+undersized+degraded, 30 active+undersized, 77 active+clean; 9.9 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 60/333 objects degraded (18.018%) 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.318 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:35.384 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:35.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:35.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:35.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:35.523 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: osdmap e168: 8 total, 7 up, 7 in 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: osdmap e168: 8 total, 7 up, 7 in 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:11:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:11:36.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:36.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/50487108' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: osdmap e168: 8 total, 7 up, 7 in 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: failed to encode map e168 with expected crc 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:11:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:36.676 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:11:36.676 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:11:36.676 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T15:11:36.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T15:11:36.825 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:36.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:36.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:36.960 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:36.977 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:37.076 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:37.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:37.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:37.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:37.241 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:37.245 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.278 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41909 5.3 5:c0f47ff1:::benchmark_data_smithi017_288048_object41908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58558228999999, 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24766596900000001, 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41909 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690624+0000", 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:37.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690647+0000", 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3186000000000001e-05 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690657+0000", 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8619999999999998e-06 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690664+0000", 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1189999999999999e-06 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690697+0000", 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2558000000000002e-05 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690819+0000", 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122149 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690983+0000", 2023-09-23T15:11:37.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016400900000000001 2023-09-23T15:11:37.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694628+0000", 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036453990000000001 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694757+0000", 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012914099999999999 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938211+0000", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243453118 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938252+0000", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1928000000000002e-05 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938291+0000", 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8724999999999999e-05 2023-09-23T15:11:37.287 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41914 5.5 5:ab998acd:::benchmark_data_smithi017_288048_object41913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58522219400001, 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25079286099999998, 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41914 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690984+0000", 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691006+0000", 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1824e-05 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691017+0000", 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0903e-05 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691025+0000", 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6860000000000006e-06 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691073+0000", 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8544000000000001e-05 2023-09-23T15:11:37.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691190+0000", 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116915 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691316+0000", 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012614200000000001 2023-09-23T15:11:37.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694247+0000", 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029306789999999998 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938074+0000", 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24382672699999999 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941696+0000", 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003622218 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941738+0000", 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2103e-05 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941778+0000", 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9956999999999997e-05 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.297 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41916 5.3 5:c430999f:::benchmark_data_smithi017_288048_object41915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58359119400001, 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24578065099999999, 2023-09-23T15:11:37.298 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41916 2023-09-23T15:11:37.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692615+0000", 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692639+0000", 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4159999999999999e-05 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692653+0000", 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3944000000000001e-05 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692662+0000", 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000001e-06 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692724+0000", 2023-09-23T15:11:37.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1750999999999999e-05 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692861+0000", 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013759800000000001 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:11:37.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693020+0000", 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158908 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694908+0000", 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001887583 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938087+0000", 2023-09-23T15:11:37.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24317931600000001 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938324+0000", 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023659700000000001 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938357+0000", 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3395000000000001e-05 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938397+0000", 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9937000000000001e-05 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41912 5.c e122/114 5:3e59ae70:::benchmark_data_smithi017_288048_object41911:head v 122'2612, mlcod=122'2610)", 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.583415336, 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24531519199999999, 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692791+0000", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692826+0000", 2023-09-23T15:11:37.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4484999999999999e-05 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692838+0000", 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1950000000000001e-05 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692845+0000", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9709999999999998e-06 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692890+0000", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5012999999999997e-05 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692913+0000", 2023-09-23T15:11:37.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3685000000000001e-05 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.696130+0000", 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003216394 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938107+0000", 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24197753799999999 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41910 5.7 e122/114 5:e1070690:::benchmark_data_smithi017_288048_object41909:head v 122'2667, mlcod=122'2664)", 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58312846199999, 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24497349800000001, 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693078+0000", 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693114+0000", 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5808e-05 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693124+0000", 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9410000000000002e-06 2023-09-23T15:11:37.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693130+0000", 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1149999999999999e-06 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693231+0000", 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010016799999999999 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693254+0000", 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3626e-05 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938007+0000", 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24475242799999999 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938053+0000", 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5976000000000002e-05 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.317 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41919 5.8 e122/114 5:1dfe6797:::benchmark_data_smithi017_288048_object41918:head v 122'2577, mlcod=122'2575)", 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58134942500001, 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243213651, 2023-09-23T15:11:37.318 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694857+0000", 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694884+0000", 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6109999999999999e-05 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694894+0000", 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0916e-05 2023-09-23T15:11:37.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694901+0000", 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8789999999999997e-06 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694939+0000", 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7837999999999999e-05 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694959+0000", 2023-09-23T15:11:37.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9411000000000001e-05 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938037+0000", 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243078029 2023-09-23T15:11:37.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938072+0000", 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5083999999999998e-05 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:41914 5.5 e122/114 ondisk, result = 0)", 2023-09-23T15:11:37.326 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58079150899999, 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.242661036, 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695415+0000", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695421+0000", 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2460000000000003e-06 2023-09-23T15:11:37.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695430+0000", 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.696e-06 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695437+0000", 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2990000000000002e-06 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938055+0000", 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.242618211 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938067+0000", 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2003e-05 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938077+0000", 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0231000000000001e-05 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.331 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41921 5.5 5:ab56ab5a:::benchmark_data_smithi017_288048_object41920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58028173400001, 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24943295900000001, 2023-09-23T15:11:37.332 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41921 2023-09-23T15:11:37.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:11:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695924+0000", 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695946+0000", 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2014e-05 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938001+0000", 2023-09-23T15:11:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24205537099999999 2023-09-23T15:11:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938016+0000", 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4109e-05 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938132+0000", 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116421 2023-09-23T15:11:37.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938297+0000", 2023-09-23T15:11:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000164709 2023-09-23T15:11:37.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938483+0000", 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018597500000000001 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945169+0000", 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066866069999999998 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945215+0000", 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5939999999999997e-05 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945262+0000", 2023-09-23T15:11:37.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7240000000000002e-05 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945319+0000", 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6205000000000001e-05 2023-09-23T15:11:37.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945359+0000", 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0071000000000003e-05 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.345 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:41916 5.3 e122/114 ondisk, result = 0)", 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:11:37.346 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.58024569299999, 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24212840399999999, 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695961+0000", 2023-09-23T15:11:37.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695966+0000", 2023-09-23T15:11:37.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0300000000000001e-06 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938006+0000", 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24203931400000001 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938015+0000", 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0350000000000007e-06 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938073+0000", 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8332999999999999e-05 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938082+0000", 2023-09-23T15:11:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5310000000000001e-06 2023-09-23T15:11:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938090+0000", 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6940000000000001e-06 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41926 5.b e122/114 5:dd0eef87:::benchmark_data_smithi017_288048_object41925:head v 122'2626, mlcod=122'2624)", 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 218.57700478199999, 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24265115500000001, 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.356 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:11:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699202+0000", 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699223+0000", 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1364000000000001e-05 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938011+0000", 2023-09-23T15:11:37.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23878813800000001 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938020+0000", 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6649999999999992e-06 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938082+0000", 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2292999999999995e-05 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938104+0000", 2023-09-23T15:11:37.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1537999999999999e-05 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941816+0000", 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037116190000000002 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941854+0000", 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8337000000000002e-05 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:105152 5.5 5:a2c1f34a:::benchmark_data_smithi017_288048_object105151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:11:37.364 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.08897580199999, 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21794708300000001, 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 105152 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:11:37.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187230+0000", 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:11:37.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187254+0000", 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4156e-05 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187266+0000", 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2211e-05 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187275+0000", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188123+0000", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0008476 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188222+0000", 2023-09-23T15:11:37.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9710999999999995e-05 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188306+0000", 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3798e-05 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.190666+0000", 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023600489999999999 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405032+0000", 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.214365852 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405088+0000", 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5915999999999998e-05 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405179+0000", 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0875e-05 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:105154 5.7 e134/132 5:e4b6367c:::benchmark_data_smithi017_288048_object105153:head v 134'6643, mlcod=134'6641)", 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.079650012, 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21553064, 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.371 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196557+0000", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196614+0000", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6879999999999998e-05 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196625+0000", 2023-09-23T15:11:37.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1421e-05 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196633+0000", 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7600000000000002e-06 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196677+0000", 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3894999999999999e-05 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196694+0000", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6900000000000001e-05 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412009+0000", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21531566999999999 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412088+0000", 2023-09-23T15:11:37.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8899999999999993e-05 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:105155 5.7 e134/132 5:e6d00935:::benchmark_data_smithi017_288048_object105154:head v 134'6644, mlcod=134'6641)", 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.079558932, 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21546711099999999, 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196648+0000", 2023-09-23T15:11:37.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196680+0000", 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1418000000000003e-05 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196687+0000", 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3780000000000003e-06 2023-09-23T15:11:37.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196692+0000", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1080000000000001e-06 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404924+0000", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20823191099999999 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404977+0000", 2023-09-23T15:11:37.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2981e-05 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412091+0000", 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071144800000000003 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412116+0000", 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4219999999999999e-05 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:105152 5.5 e134/132 ondisk, result = 0)", 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.198335+0000", 2023-09-23T15:11:37.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.07787264199999, 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20685822000000001, 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.198335+0000", 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.198335+0000", 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.198334+0000", 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.198338+0000", 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7510000000000002e-06 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404770+0000", 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.206432375 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404803+0000", 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2873000000000002e-05 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405013+0000", 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000210082 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405024+0000", 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0879e-05 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405193+0000", 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016889100000000001 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:105171 5.7 e134/132 5:e9116110:::benchmark_data_smithi017_288048_object105170:head v 134'6645, mlcod=134'6641)", 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.214232+0000", 2023-09-23T15:11:37.388 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.061975719, 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19790871600000001, 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.214232+0000", 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.214232+0000", 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.214231+0000", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.214260+0000", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9156999999999999e-05 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404781+0000", 2023-09-23T15:11:37.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.19052076500000001 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404804+0000", 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3065999999999999e-05 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.408188+0000", 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033840049999999998 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.408227+0000", 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8936000000000001e-05 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412118+0000", 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003891196 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412141+0000", 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2450999999999999e-05 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106188 5.3 5:cc2b8235:::benchmark_data_smithi017_288048_object106187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.54724525, 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25023266199999999, 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106188 2023-09-23T15:11:37.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728961+0000", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728985+0000", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4165999999999999e-05 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729002+0000", 2023-09-23T15:11:37.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6997e-05 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729010+0000", 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2709999999999992e-06 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729326+0000", 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031631699999999999 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729469+0000", 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014238 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729612+0000", 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014267699999999999 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732809+0000", 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031979479999999999 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732951+0000", 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014197299999999999 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979128+0000", 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.246176479 2023-09-23T15:11:37.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979168+0000", 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0108000000000003e-05 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979195+0000", 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6879000000000001e-05 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106190 5.5 5:a890e663:::benchmark_data_smithi017_288048_object106189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 156.54323433299999, 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24679059, 2023-09-23T15:11:37.404 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106190 2023-09-23T15:11:37.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732972+0000", 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:11:37.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732993+0000", 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1047e-05 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733006+0000", 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2432999999999999e-05 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733011+0000", 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733051+0000", 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9368e-05 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733147+0000", 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6447999999999995e-05 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733242+0000", 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5328000000000006e-05 2023-09-23T15:11:37.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.735970+0000", 2023-09-23T15:11:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027275289999999998 2023-09-23T15:11:37.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979622+0000", 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243651849 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979677+0000", 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5585999999999999e-05 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979764+0000", 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6321000000000002e-05 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166954 5.3 5:c22244f8:::benchmark_data_smithi017_288048_object55185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.184178666999998, 2023-09-23T15:11:37.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.279674321, 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166954 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092027+0000", 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092034+0000", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8050000000000001e-06 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092053+0000", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9315000000000001e-05 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092061+0000", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9319999999999996e-06 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092120+0000", 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8522e-05 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092275+0000", 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015473200000000001 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092403+0000", 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128739 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096986+0000", 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045829709999999999 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.099294+0000", 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023073640000000001 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371598+0000", 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27230407499999998 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371657+0000", 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9521000000000002e-05 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371703+0000", 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5927000000000003e-05 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.419 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166955 5.5 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 79.183717383000001, 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41495601799999998, 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166955 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.420 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092489+0000", 2023-09-23T15:11:37.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092494+0000", 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2989999999999998e-06 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092512+0000", 2023-09-23T15:11:37.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7952999999999999e-05 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092521+0000", 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1449999999999997e-06 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092566+0000", 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5461999999999999e-05 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092677+0000", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011045200000000001 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092782+0000", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096249+0000", 2023-09-23T15:11:37.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00346767 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507308+0000", 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41105842100000001 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507348+0000", 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0608999999999997e-05 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507446+0000", 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7705000000000001e-05 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.426 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:187780 5.5 5:a2bdf4dc:::benchmark_data_smithi017_288048_object76011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 50.925724535000001, 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21912024099999999, 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 187780 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:11:37.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350482+0000", 2023-09-23T15:11:37.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350487+0000", 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9520000000000002e-06 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350497+0000", 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0518e-05 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350506+0000", 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5459999999999992e-06 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350617+0000", 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011165100000000001 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350766+0000", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014886700000000001 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.351318+0000", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055183399999999996 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.362549+0000", 2023-09-23T15:11:37.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011230801 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569458+0000", 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20690950299999999 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:11:37.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569518+0000", 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9911999999999998e-05 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569603+0000", 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5008999999999999e-05 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:11:37.433 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:11:37.434 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:11:37.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: osdmap e169: 8 total, 7 up, 7 in 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: pgmap v415: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 175 KiB/s rd, 92 op/s; 3/84 objects degraded (3.571%); 335 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 60/333 objects degraded (18.018%), 12 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: osdmap e169: 8 total, 7 up, 7 in 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: pgmap v415: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 175 KiB/s rd, 92 op/s; 3/84 objects degraded (3.571%); 335 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:37.436 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 60/333 objects degraded (18.018%), 12 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:37.512 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:37.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:37.516 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:37.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: osdmap e169: 8 total, 7 up, 7 in 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: failed to encode map e169 with expected crc 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: pgmap v415: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 175 KiB/s rd, 92 op/s; 3/84 objects degraded (3.571%); 335 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 60/333 objects degraded (18.018%), 12 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:37.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:37.645 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:38.268 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:38] "GET /metrics HTTP/1.1" 200 36302 "" "Prometheus/2.43.0" 2023-09-23T15:11:38.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 25 pgs peering (PG_AVAILABILITY) 2023-09-23T15:11:38.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: osdmap e170: 8 total, 7 up, 7 in 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[215646]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 25 pgs peering (PG_AVAILABILITY) 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: osdmap e170: 8 total, 7 up, 7 in 2023-09-23T15:11:38.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:38 smithi017 ceph-mon[222878]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:11:38.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:38.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:38.344 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:38.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:38.384 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:38.404 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:38.429 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:38.429 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.430 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:11:38.448 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:38.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:38.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 25 pgs peering (PG_AVAILABILITY) 2023-09-23T15:11:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: osdmap e170: 8 total, 7 up, 7 in 2023-09-23T15:11:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.586 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:38 smithi027 ceph-mon[195233]: failed to encode map e170 with expected crc 2023-09-23T15:11:38.588 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:38.638 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:38.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:38.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:38.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:38.779 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:38.976 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:38.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:38.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:39.080 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:39.112 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:39 smithi017 ceph-mon[215646]: pgmap v417: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 170 KiB/s rd, 90 op/s; 3/84 objects degraded (3.571%); 326 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/749164322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:39.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:39 smithi017 ceph-mon[222878]: pgmap v417: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 170 KiB/s rd, 90 op/s; 3/84 objects degraded (3.571%); 326 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:39.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/749164322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:39 smithi027 ceph-mon[195233]: pgmap v417: 93 pgs: 39 remapped+peering, 3 active+undersized+degraded, 21 active+undersized, 30 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 170 KiB/s rd, 90 op/s; 3/84 objects degraded (3.571%); 326 KiB/s, 2 keys/s, 2 objects/s recovering 2023-09-23T15:11:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/749164322' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:39.661 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:39.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:39.765 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:39.792 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:39.934 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:11:39.934 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:11:39.962 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:39.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:40.064 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:40.064 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:40.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:40.099 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:40.131 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:40.150 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:40.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:40.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:40.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:40.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:40.334 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:40.340 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:40.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:40.468 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:40.513 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:40.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:40 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 25 pgs peering) 2023-09-23T15:11:40.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:40.645 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:40.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:40 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 25 pgs peering) 2023-09-23T15:11:40.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:40 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 25 pgs peering) 2023-09-23T15:11:40.688 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:40.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:40.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:40.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:40.827 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:40.992 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:40.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:40.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:41.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:41.125 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:41.170 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:41.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:41.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:41.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:41.304 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:41.348 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:41.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:41.362 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:11:41.364 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' created 2023-09-23T15:11:41.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:41.478 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:41 smithi027 ceph-mon[195233]: pgmap v418: 93 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 117 KiB/s rd, 61 op/s; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 906 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T15:11:41.478 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3007059329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.478 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.478 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T15:11:41.480 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:41.524 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:41.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:41.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:41.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[215646]: pgmap v418: 93 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 117 KiB/s rd, 61 op/s; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 906 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T15:11:41.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3007059329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.607 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.608 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T15:11:41.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[222878]: pgmap v418: 93 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 117 KiB/s rd, 61 op/s; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 906 KiB/s, 1 keys/s, 8 objects/s recovering 2023-09-23T15:11:41.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3007059329' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:11:41.608 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T15:11:41.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:41.657 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:42.055 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:42.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:42.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:42.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:11:42.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:42.191 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:42.241 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:42.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:42.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:42.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:42.390 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: osdmap e171: 8 total, 7 up, 7 in 2023-09-23T15:11:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:42.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:42.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.646 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: osdmap e171: 8 total, 7 up, 7 in 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:42.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:42.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/573703572' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: osdmap e171: 8 total, 7 up, 7 in 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: failed to encode map e171 with expected crc 2023-09-23T15:11:42.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:42.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T15:11:42.868 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:42.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:42.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:42.881 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:11:42.881 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:11:42.945 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:42.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:43.006 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:43.102 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:43.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:43.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:43.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:43.234 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:43.487 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:43.488 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: pgmap v420: 109 pgs: 16 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 675 KiB/s, 6 objects/s recovering 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: osdmap e172: 8 total, 7 up, 7 in 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: pgmap v420: 109 pgs: 16 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 675 KiB/s, 6 objects/s recovering 2023-09-23T15:11:43.489 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: osdmap e172: 8 total, 7 up, 7 in 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:43.490 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:43.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:43.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:43.621 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: pgmap v420: 109 pgs: 16 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 21 active+undersized, 32 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 675 KiB/s, 6 objects/s recovering 2023-09-23T15:11:43.621 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: osdmap e172: 8 total, 7 up, 7 in 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: failed to encode map e172 with expected crc 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:11:43.622 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:43.624 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:43.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:11:43.675 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:43.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:43.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:43.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:43.809 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:43.916 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:43.957 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:43.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:43.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:44.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:44.089 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:44.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:44 smithi017 ceph-mon[215646]: osdmap e173: 8 total, 7 up, 7 in 2023-09-23T15:11:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:44 smithi017 ceph-mon[222878]: osdmap e173: 8 total, 7 up, 7 in 2023-09-23T15:11:44.762 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:44.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:44.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:44 smithi027 ceph-mon[195233]: osdmap e173: 8 total, 7 up, 7 in 2023-09-23T15:11:44.859 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:11:44.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:44.930 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:45.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:45.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:45.602 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:45.642 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: pgmap v423: 109 pgs: 11 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 35 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 682 KiB/s, 6 objects/s recovering 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[222878]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.674 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: pgmap v423: 109 pgs: 11 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 35 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 682 KiB/s, 6 objects/s recovering 2023-09-23T15:11:45.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:45 smithi017 ceph-mon[215646]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.679 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:45.752 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:45.771 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:45.790 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:45.792 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:45.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:45.795 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:45.810 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:45.811 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:45.832 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:45.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: pgmap v423: 109 pgs: 11 unknown, 37 active+clean+remapped, 3 active+undersized+degraded, 23 active+undersized, 35 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%); 682 KiB/s, 6 objects/s recovering 2023-09-23T15:11:45.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:45 smithi027 ceph-mon[195233]: failed to encode map e173 with expected crc 2023-09-23T15:11:45.856 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:45.876 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:45.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.877 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:11:45.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:45.925 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:46.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1362865467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:46.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1362865467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:46.830 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:46.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:46.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:46.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1362865467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:46.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:46.962 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:46.963 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:46.968 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:47.019 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:47.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:47.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:47.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:47.156 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:47.657 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:47.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:47.660 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:47.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:47.793 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:47.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:47 smithi017 ceph-mon[222878]: pgmap v424: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:47.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:11:47.817 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T15:11:47.848 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:47.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:47.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:47.954 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:47.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:48.031 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:48.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:48.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:47 smithi027 ceph-mon[195233]: pgmap v424: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:48.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:11:48.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:48.165 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:48] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2023-09-23T15:11:48.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:47 smithi017 ceph-mon[215646]: pgmap v424: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:48.166 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:11:48.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:48.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:11:49.021 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:49.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:49.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:49.099 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:11:49.100 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:11:49.100 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:11:49.100 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:11:49.100 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:11:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:48 smithi027 ceph-mon[195233]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:48 smithi027 ceph-mon[195233]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:11:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:48 smithi027 ceph-mon[195233]: osdmap e174: 8 total, 7 up, 7 in 2023-09-23T15:11:49.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:49.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[222878]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[222878]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:11:49.131 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[222878]: osdmap e174: 8 total, 7 up, 7 in 2023-09-23T15:11:49.131 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[215646]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[215646]: failed to encode map e174 with expected crc 2023-09-23T15:11:49.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3938604535' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:11:49.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:48 smithi017 ceph-mon[215646]: osdmap e174: 8 total, 7 up, 7 in 2023-09-23T15:11:49.159 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:49.211 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:49.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:49.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:49.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:49.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:49.847 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:49.851 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:49.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:11:49.942 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T15:11:49.943 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_1 2023-09-23T15:11:49.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:11:49.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:11:49.972 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:11:49.972 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:11:49.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:50.033 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:50.037 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:11:50.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:50.042 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:11:50.049 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:11:50.049 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:11:50.049 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_350392 2023-09-23T15:11:50.061 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:11:50.061 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 15 29 14 75.2726 -1 0.00570817 0.0075328 2023-09-23T15:11:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:49 smithi027 ceph-mon[195233]: pgmap v426: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:50.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:11:50.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:49 smithi017 ceph-mon[215646]: pgmap v426: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:50.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:49 smithi017 ceph-mon[222878]: pgmap v426: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:50.224 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:50.385 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:11:50.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:50.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:11:50.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:50.562 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:50.773 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:50.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:50.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:50.834 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:50.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:50.854 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:11:50.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:50.923 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:51.003 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:51.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:51.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:51.061 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3251 3235 199.842 201.312 0.00500358 0.00498984 2023-09-23T15:11:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:11:51.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:51.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:52.061 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 7215 7199 223.612 247.75 0.00335042 0.00446358 2023-09-23T15:11:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: pgmap v427: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:52.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: osdmap e175: 8 total, 7 up, 7 in 2023-09-23T15:11:52.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:51 smithi027 ceph-mon[195233]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: pgmap v427: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: osdmap e175: 8 total, 7 up, 7 in 2023-09-23T15:11:52.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[215646]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: pgmap v427: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/191826653' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: osdmap e175: 8 total, 7 up, 7 in 2023-09-23T15:11:52.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:51 smithi017 ceph-mon[222878]: failed to encode map e175 with expected crc 2023-09-23T15:11:52.929 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:52.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:52.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:53.019 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:53.019 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:53.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:53.062 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9718 9702 201.261 156.438 0.00447743 0.0049614 2023-09-23T15:11:53.070 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:52 smithi027 ceph-mon[195233]: failed to encode map e175 with expected crc 2023-09-23T15:11:53.140 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:53.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:53.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:52 smithi017 ceph-mon[215646]: failed to encode map e175 with expected crc 2023-09-23T15:11:53.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:52 smithi017 ceph-mon[222878]: failed to encode map e175 with expected crc 2023-09-23T15:11:53.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:11:53.305 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:53.454 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:11:53.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:53.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:11:53.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:53.600 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:53.612 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:11:53.654 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:11:53.662 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:53.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:53.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:53.694 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:11:53.735 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:11:53.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:53.771 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:11:53.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:11:53.828 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:53.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:53 smithi027 ceph-mon[195233]: pgmap v429: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:53.857 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:11:53.857 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.857 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:11:53.857 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.857 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:11:53.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:11:54.063 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12417 12401 193.117 168.688 0.00404777 0.00517337 2023-09-23T15:11:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1512062955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:54.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:53 smithi017 ceph-mon[222878]: pgmap v429: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:54.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1512062955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:54.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:53 smithi017 ceph-mon[215646]: pgmap v429: 109 pgs: 37 active+clean+remapped, 3 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 3/84 objects degraded (3.571%); 6/84 objects misplaced (7.143%) 2023-09-23T15:11:54.144 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1512062955' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:11:55.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:11:55.063 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 15417 15401 191.993 187.5 0.00668013 0.00520281 2023-09-23T15:11:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:54 smithi017 ceph-mon[222878]: pgmap v430: 109 pgs: 37 active+clean+remapped, 5 active+undersized+degraded, 25 active+undersized, 42 active+clean; 238 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 1629/11289 objects degraded (14.430%); 6/11289 objects misplaced (0.053%) 2023-09-23T15:11:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:54 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 1629/11289 objects degraded (14.430%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:54 smithi017 ceph-mon[215646]: pgmap v430: 109 pgs: 37 active+clean+remapped, 5 active+undersized+degraded, 25 active+undersized, 42 active+clean; 238 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 1629/11289 objects degraded (14.430%); 6/11289 objects misplaced (0.053%) 2023-09-23T15:11:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:54 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 1629/11289 objects degraded (14.430%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:54 smithi027 ceph-mon[195233]: pgmap v430: 109 pgs: 37 active+clean+remapped, 5 active+undersized+degraded, 25 active+undersized, 42 active+clean; 238 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 29 MiB/s wr, 466 op/s; 1629/11289 objects degraded (14.430%); 6/11289 objects misplaced (0.053%) 2023-09-23T15:11:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:54 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 1629/11289 objects degraded (14.430%), 5 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:11:56.063 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 17411 17395 180.788 124.625 0.00549513 0.00552574 2023-09-23T15:11:56.405 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:56.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:56.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:56.557 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:56.601 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:56.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:56.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:11:56.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:56.740 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:56.820 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:56.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:56.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:56.926 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:56.966 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:57.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:57.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:57.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:57.069 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 19457 19441 173.09 127.875 0.00526503 0.00577 2023-09-23T15:11:57.110 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:11:57.111 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:11:57.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:11:57.190 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:57.267 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:11:57.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:57.273 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:11:57.339 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:11:57.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:11:57.418 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:57.468 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:11:57.468 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:11:57.485 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:11:57.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:57.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:11:57.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:57.646 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:57.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:57 smithi017 ceph-mon[215646]: pgmap v431: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 817 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5077/39090 objects degraded (12.988%); 6/39090 objects misplaced (0.015%) 2023-09-23T15:11:57.821 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:57 smithi017 ceph-mon[222878]: pgmap v431: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 817 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5077/39090 objects degraded (12.988%); 6/39090 objects misplaced (0.015%) 2023-09-23T15:11:57.821 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:57 smithi027 ceph-mon[195233]: pgmap v431: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 817 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 5077/39090 objects degraded (12.988%); 6/39090 objects misplaced (0.015%) 2023-09-23T15:11:58.069 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 21896 21880 170.513 152.438 0.00740839 0.00585776 2023-09-23T15:11:58.116 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:11:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:11:58] "GET /metrics HTTP/1.1" 200 36258 "" "Prometheus/2.43.0" 2023-09-23T15:11:58.463 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:11:58.465 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:11:58.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:58.954 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:58.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:58.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:59.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:11:59.070 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 15 24321 24306 168.404 151.625 0.0111093 0.00593234 2023-09-23T15:11:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:59.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:11:59.116 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:11:59.119 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:59.745 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:11:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:59.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:11:59.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:11:59.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:11:59.889 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:11:59.926 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":432,"stamp":"2023-09-23T15:11:58.250622+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":954144691,"num_objects":14514,"num_object_clones":0,"num_object_copies":43542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5077,"num_objects_misplaced":6,"num_objects_unfound":0,"num_objects_dirty":14514,"num_whiteouts":0,"num_read":6481,"num_read_kb":14124,"num_write":15632,"num_write_kb":942245,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":5970336,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18898,"ondisk_log_size":18898,"up":260,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5135492,"kb_used_data":4264268,"kb_used_omap":30,"kb_used_meta":871137,"kb_avail":651080572,"statfs":{"total":671965249536,"available":666706505728,"internally_reserved":0,"allocated":4366610432,"data_stored":4362163949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":892044888},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":949354496,"num_objects":14486,"num_object_clones":0,"num_object_copies":43458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14486,"num_write_kb":927104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004113"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":159,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.814452+0000","last_change":"2023-09-23T15:08:51.256433+0000","last_active":"2023-09-23T15:11:47.814452+0000","last_peered":"2023-09-23T15:11:47.814452+0000","last_clean":"2023-09-23T15:08:45.866113+0000","last_became_active":"2023-09-23T15:08:51.256433+0000","last_became_peered":"2023-09-23T15:08:51.256433+0000","last_unstale":"2023-09-23T15:11:47.814452+0000","last_undegraded":"2023-09-23T15:11:47.814452+0000","last_fullsized":"2023-09-23T15:08:51.253610+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:05.344878+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":102,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.914295+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:11:48.914295+0000","last_peered":"2023-09-23T15:11:48.914295+0000","last_clean":"2023-09-23T15:11:48.914295+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:11:48.914295+0000","last_undegraded":"2023-09-23T15:11:48.914295+0000","last_fullsized":"2023-09-23T15:11:48.914295+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915828+0000","last_change":"2023-09-23T15:08:51.260175+0000","last_active":"2023-09-23T15:11:48.915828+0000","last_peered":"2023-09-23T15:11:48.915828+0000","last_clean":"2023-09-23T15:05:44.711180+0000","last_became_active":"2023-09-23T15:08:51.260175+0000","last_became_peered":"2023-09-23T15:08:51.260175+0000","last_unstale":"2023-09-23T15:11:48.915828+0000","last_undegraded":"2023-09-23T15:11:48.915828+0000","last_fullsized":"2023-09-23T15:08:51.247674+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:36.101095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":128,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.024079+0000","last_change":"2023-09-23T15:11:35.318901+0000","last_active":"2023-09-23T15:11:48.024079+0000","last_peered":"2023-09-23T15:11:48.024079+0000","last_clean":"2023-09-23T15:11:48.024079+0000","last_became_active":"2023-09-23T15:11:35.302723+0000","last_became_peered":"2023-09-23T15:11:35.302723+0000","last_unstale":"2023-09-23T15:11:48.024079+0000","last_undegraded":"2023-09-23T15:11:48.024079+0000","last_fullsized":"2023-09-23T15:11:48.024079+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:22.744532+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":162,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.916496+0000","last_change":"2023-09-23T15:08:51.250676+0000","last_active":"2023-09-23T15:11:48.916496+0000","last_peered":"2023-09-23T15:11:48.916496+0000","last_clean":"2023-09-23T15:08:45.961410+0000","last_became_active":"2023-09-23T15:08:51.250676+0000","last_became_peered":"2023-09-23T15:08:51.250676+0000","last_unstale":"2023-09-23T15:11:48.916496+0000","last_undegraded":"2023-09-23T15:11:48.916496+0000","last_fullsized":"2023-09-23T15:08:51.246510+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:00:59.552169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915448+0000","last_change":"2023-09-23T15:08:51.268819+0000","last_active":"2023-09-23T15:11:48.915448+0000","last_peered":"2023-09-23T15:11:48.915448+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:08:51.268819+0000","last_became_peered":"2023-09-23T15:08:51.268819+0000","last_unstale":"2023-09-23T15:11:48.915448+0000","last_undegraded":"2023-09-23T15:11:48.915448+0000","last_fullsized":"2023-09-23T15:08:51.250280+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:25:42.891886+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":118,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:50.841485+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:11:50.841485+0000","last_peered":"2023-09-23T15:11:50.841485+0000","last_clean":"2023-09-23T15:11:50.841485+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:11:50.841485+0000","last_undegraded":"2023-09-23T15:11:50.841485+0000","last_fullsized":"2023-09-23T15:11:50.841485+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":144,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.916653+0000","last_change":"2023-09-23T15:08:51.251026+0000","last_active":"2023-09-23T15:11:48.916653+0000","last_peered":"2023-09-23T15:11:48.916653+0000","last_clean":"2023-09-23T15:08:45.961298+0000","last_became_active":"2023-09-23T15:08:51.251026+0000","last_became_peered":"2023-09-23T15:08:51.251026+0000","last_unstale":"2023-09-23T15:11:48.916653+0000","last_undegraded":"2023-09-23T15:11:48.916653+0000","last_fullsized":"2023-09-23T15:08:51.246433+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:12:14.320021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023822+0000","last_change":"2023-09-23T15:11:35.197432+0000","last_active":"2023-09-23T15:11:48.023822+0000","last_peered":"2023-09-23T15:11:48.023822+0000","last_clean":"2023-09-23T15:11:48.023822+0000","last_became_active":"2023-09-23T15:11:35.196022+0000","last_became_peered":"2023-09-23T15:11:35.196022+0000","last_unstale":"2023-09-23T15:11:48.023822+0000","last_undegraded":"2023-09-23T15:11:48.023822+0000","last_fullsized":"2023-09-23T15:11:48.023822+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:47.819230+0000","last_change":"2023-09-23T15:11:35.181890+0000","last_active":"2023-09-23T15:11:47.819230+0000","last_peered":"2023-09-23T15:11:47.819230+0000","last_clean":"2023-09-23T15:11:47.819230+0000","last_became_active":"2023-09-23T15:11:35.174866+0000","last_became_peered":"2023-09-23T15:11:35.174866+0000","last_unstale":"2023-09-23T15:11:47.819230+0000","last_undegraded":"2023-09-23T15:11:47.819230+0000","last_fullsized":"2023-09-23T15:11:47.819230+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:27.425238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3,7],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":96,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.915749+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:11:48.915749+0000","last_peered":"2023-09-23T15:11:48.915749+0000","last_clean":"2023-09-23T15:11:48.915749+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:11:48.915749+0000","last_undegraded":"2023-09-23T15:11:48.915749+0000","last_fullsized":"2023-09-23T15:11:48.915749+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":36,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.914219+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:11:48.914219+0000","last_peered":"2023-09-23T15:11:48.914219+0000","last_clean":"2023-09-23T15:11:48.914219+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:11:48.914219+0000","last_undegraded":"2023-09-23T15:11:48.914219+0000","last_fullsized":"2023-09-23T15:11:48.914219+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915079+0000","last_change":"2023-09-23T15:08:51.253311+0000","last_active":"2023-09-23T15:11:48.915079+0000","last_peered":"2023-09-23T15:11:48.915079+0000","last_clean":"2023-09-23T15:05:12.641908+0000","last_became_active":"2023-09-23T15:08:51.253311+0000","last_became_peered":"2023-09-23T15:08:51.253311+0000","last_unstale":"2023-09-23T15:11:48.915079+0000","last_undegraded":"2023-09-23T15:11:48.915079+0000","last_fullsized":"2023-09-23T15:08:51.250257+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:12:46.907705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":159,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.815341+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:11:47.815341+0000","last_peered":"2023-09-23T15:11:47.815341+0000","last_clean":"2023-09-23T15:11:47.815341+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:11:47.815341+0000","last_undegraded":"2023-09-23T15:11:47.815341+0000","last_fullsized":"2023-09-23T15:11:47.815341+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.814636+0000","last_change":"2023-09-23T15:08:51.258299+0000","last_active":"2023-09-23T15:11:47.814636+0000","last_peered":"2023-09-23T15:11:47.814636+0000","last_clean":"2023-09-23T15:08:45.865233+0000","last_became_active":"2023-09-23T15:08:51.258299+0000","last_became_peered":"2023-09-23T15:08:51.258299+0000","last_unstale":"2023-09-23T15:11:47.814636+0000","last_undegraded":"2023-09-23T15:11:47.814636+0000","last_fullsized":"2023-09-23T15:08:51.253870+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:04.473057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023690+0000","last_change":"2023-09-23T15:11:35.190857+0000","last_active":"2023-09-23T15:11:48.023690+0000","last_peered":"2023-09-23T15:11:48.023690+0000","last_clean":"2023-09-23T15:11:48.023690+0000","last_became_active":"2023-09-23T15:11:35.190147+0000","last_became_peered":"2023-09-23T15:11:35.190147+0000","last_unstale":"2023-09-23T15:11:48.023690+0000","last_undegraded":"2023-09-23T15:11:48.023690+0000","last_fullsized":"2023-09-23T15:11:48.023690+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:24.157332+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:24.157332+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":37,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814878+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:11:47.814878+0000","last_peered":"2023-09-23T15:11:47.814878+0000","last_clean":"2023-09-23T15:11:47.814878+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:11:47.814878+0000","last_undegraded":"2023-09-23T15:11:47.814878+0000","last_fullsized":"2023-09-23T15:11:47.814878+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":154,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.914990+0000","last_change":"2023-09-23T15:11:35.153638+0000","last_active":"2023-09-23T15:11:48.914990+0000","last_peered":"2023-09-23T15:11:48.914990+0000","last_clean":"2023-09-23T15:11:48.914990+0000","last_became_active":"2023-09-23T15:11:35.153286+0000","last_became_peered":"2023-09-23T15:11:35.153286+0000","last_unstale":"2023-09-23T15:11:48.914990+0000","last_undegraded":"2023-09-23T15:11:48.914990+0000","last_fullsized":"2023-09-23T15:11:48.914990+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:40.656355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023004+0000","last_change":"2023-09-23T15:11:35.191915+0000","last_active":"2023-09-23T15:11:48.023004+0000","last_peered":"2023-09-23T15:11:48.023004+0000","last_clean":"2023-09-23T15:11:48.023004+0000","last_became_active":"2023-09-23T15:11:35.187894+0000","last_became_peered":"2023-09-23T15:11:35.187894+0000","last_unstale":"2023-09-23T15:11:48.023004+0000","last_undegraded":"2023-09-23T15:11:48.023004+0000","last_fullsized":"2023-09-23T15:11:48.023004+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:21.135617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:21.135617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":128,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023231+0000","last_change":"2023-09-23T15:11:35.167728+0000","last_active":"2023-09-23T15:11:48.023231+0000","last_peered":"2023-09-23T15:11:48.023231+0000","last_clean":"2023-09-23T15:11:48.023231+0000","last_became_active":"2023-09-23T15:11:35.167146+0000","last_became_peered":"2023-09-23T15:11:35.167146+0000","last_unstale":"2023-09-23T15:11:48.023231+0000","last_undegraded":"2023-09-23T15:11:48.023231+0000","last_fullsized":"2023-09-23T15:11:48.023231+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:19.014386+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.819453+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:11:47.819453+0000","last_peered":"2023-09-23T15:11:47.819453+0000","last_clean":"2023-09-23T15:11:47.819453+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:11:47.819453+0000","last_undegraded":"2023-09-23T15:11:47.819453+0000","last_fullsized":"2023-09-23T15:11:47.819453+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023272+0000","last_change":"2023-09-23T15:11:35.335193+0000","last_active":"2023-09-23T15:11:48.023272+0000","last_peered":"2023-09-23T15:11:48.023272+0000","last_clean":"2023-09-23T15:11:48.023272+0000","last_became_active":"2023-09-23T15:11:35.289107+0000","last_became_peered":"2023-09-23T15:11:35.289107+0000","last_unstale":"2023-09-23T15:11:48.023272+0000","last_undegraded":"2023-09-23T15:11:48.023272+0000","last_fullsized":"2023-09-23T15:11:48.023272+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814684+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:11:47.814684+0000","last_peered":"2023-09-23T15:11:47.814684+0000","last_clean":"2023-09-23T15:11:47.814684+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:11:47.814684+0000","last_undegraded":"2023-09-23T15:11:47.814684+0000","last_fullsized":"2023-09-23T15:11:47.814684+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":127,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:50.841251+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:11:50.841251+0000","last_peered":"2023-09-23T15:11:50.841251+0000","last_clean":"2023-09-23T15:11:50.841251+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:11:50.841251+0000","last_undegraded":"2023-09-23T15:11:50.841251+0000","last_fullsized":"2023-09-23T15:11:50.841251+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023479+0000","last_change":"2023-09-23T15:11:35.190735+0000","last_active":"2023-09-23T15:11:48.023479+0000","last_peered":"2023-09-23T15:11:48.023479+0000","last_clean":"2023-09-23T15:11:48.023479+0000","last_became_active":"2023-09-23T15:11:35.189538+0000","last_became_peered":"2023-09-23T15:11:35.189538+0000","last_unstale":"2023-09-23T15:11:48.023479+0000","last_undegraded":"2023-09-23T15:11:48.023479+0000","last_fullsized":"2023-09-23T15:11:48.023479+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":141,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.914222+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:11:48.914222+0000","last_peered":"2023-09-23T15:11:48.914222+0000","last_clean":"2023-09-23T15:11:48.914222+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:11:48.914222+0000","last_undegraded":"2023-09-23T15:11:48.914222+0000","last_fullsized":"2023-09-23T15:11:48.914222+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":154,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.916430+0000","last_change":"2023-09-23T15:11:35.207462+0000","last_active":"2023-09-23T15:11:48.916430+0000","last_peered":"2023-09-23T15:11:48.916430+0000","last_clean":"2023-09-23T15:11:48.916430+0000","last_became_active":"2023-09-23T15:11:35.204333+0000","last_became_peered":"2023-09-23T15:11:35.204333+0000","last_unstale":"2023-09-23T15:11:48.916430+0000","last_undegraded":"2023-09-23T15:11:48.916430+0000","last_fullsized":"2023-09-23T15:11:48.916430+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:41.666300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2,7],"avail_no_missing":["3","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"175'891","reported_seq":903,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.646577+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:11:54.646577+0000","last_peered":"2023-09-23T15:11:54.646577+0000","last_clean":"2023-09-23T15:11:54.646577+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:11:54.646577+0000","last_undegraded":"2023-09-23T15:11:54.646577+0000","last_fullsized":"2023-09-23T15:11:54.646577+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":891,"log_dups_size":0,"ondisk_log_size":891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58392576,"num_objects":891,"num_object_clones":0,"num_object_copies":2673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":891,"num_write_kb":57024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023846+0000","last_change":"2023-09-23T15:11:35.274252+0000","last_active":"2023-09-23T15:11:48.023846+0000","last_peered":"2023-09-23T15:11:48.023846+0000","last_clean":"2023-09-23T15:11:48.023846+0000","last_became_active":"2023-09-23T15:11:35.252209+0000","last_became_peered":"2023-09-23T15:11:35.252209+0000","last_unstale":"2023-09-23T15:11:48.023846+0000","last_undegraded":"2023-09-23T15:11:48.023846+0000","last_fullsized":"2023-09-23T15:11:48.023846+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":33,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:47.815678+0000","last_change":"2023-09-23T15:11:35.198281+0000","last_active":"2023-09-23T15:11:47.815678+0000","last_peered":"2023-09-23T15:11:47.815678+0000","last_clean":"2023-09-23T15:11:47.815678+0000","last_became_active":"2023-09-23T15:11:35.189267+0000","last_became_peered":"2023-09-23T15:11:35.189267+0000","last_unstale":"2023-09-23T15:11:47.815678+0000","last_undegraded":"2023-09-23T15:11:47.815678+0000","last_fullsized":"2023-09-23T15:11:47.815678+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:47:09.475698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,7],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.818655+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:11:47.818655+0000","last_peered":"2023-09-23T15:11:47.818655+0000","last_clean":"2023-09-23T15:11:47.818655+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:11:47.818655+0000","last_undegraded":"2023-09-23T15:11:47.818655+0000","last_fullsized":"2023-09-23T15:11:47.818655+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"175'816","reported_seq":826,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.250937+0000","last_change":"2023-09-23T15:11:50.059433+0000","last_active":"2023-09-23T15:11:54.250937+0000","last_peered":"2023-09-23T15:11:54.250937+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.375391+0000","last_became_peered":"2023-09-23T15:11:42.375391+0000","last_unstale":"2023-09-23T15:11:54.250937+0000","last_undegraded":"2023-09-23T15:11:50.055498+0000","last_fullsized":"2023-09-23T15:11:42.369727+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":816,"log_dups_size":0,"ondisk_log_size":816,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:33.967506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53477376,"num_objects":816,"num_object_clones":0,"num_object_copies":2448,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":816,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":816,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":816,"num_write_kb":52224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":816}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023655+0000","last_change":"2023-09-23T15:11:35.197009+0000","last_active":"2023-09-23T15:11:48.023655+0000","last_peered":"2023-09-23T15:11:48.023655+0000","last_clean":"2023-09-23T15:11:48.023655+0000","last_became_active":"2023-09-23T15:11:35.195039+0000","last_became_peered":"2023-09-23T15:11:35.195039+0000","last_unstale":"2023-09-23T15:11:48.023655+0000","last_undegraded":"2023-09-23T15:11:48.023655+0000","last_fullsized":"2023-09-23T15:11:48.023655+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":37,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:47.815694+0000","last_change":"2023-09-23T15:11:35.323354+0000","last_active":"2023-09-23T15:11:47.815694+0000","last_peered":"2023-09-23T15:11:47.815694+0000","last_clean":"2023-09-23T15:11:47.815694+0000","last_became_active":"2023-09-23T15:11:35.190405+0000","last_became_peered":"2023-09-23T15:11:35.190405+0000","last_unstale":"2023-09-23T15:11:47.815694+0000","last_undegraded":"2023-09-23T15:11:47.815694+0000","last_fullsized":"2023-09-23T15:11:47.815694+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:12.478773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,7],"avail_no_missing":["4","0","7"],"object_location_counts":[{"shards":"0,4,7","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":157,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.814475+0000","last_change":"2023-09-23T15:08:51.259937+0000","last_active":"2023-09-23T15:11:47.814475+0000","last_peered":"2023-09-23T15:11:47.814475+0000","last_clean":"2023-09-23T15:08:45.867189+0000","last_became_active":"2023-09-23T15:08:51.259937+0000","last_became_peered":"2023-09-23T15:08:51.259937+0000","last_unstale":"2023-09-23T15:11:47.814475+0000","last_undegraded":"2023-09-23T15:11:47.814475+0000","last_fullsized":"2023-09-23T15:08:51.253537+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:57.015912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"175'1382","reported_seq":1394,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:58.239439+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:11:58.239439+0000","last_peered":"2023-09-23T15:11:58.239439+0000","last_clean":"2023-09-23T15:11:58.239439+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:11:58.239439+0000","last_undegraded":"2023-09-23T15:11:58.239439+0000","last_fullsized":"2023-09-23T15:11:58.239439+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":1382,"log_dups_size":0,"ondisk_log_size":1382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90570752,"num_objects":1382,"num_object_clones":0,"num_object_copies":4146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1382,"num_write_kb":88448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023730+0000","last_change":"2023-09-23T15:11:35.189649+0000","last_active":"2023-09-23T15:11:48.023730+0000","last_peered":"2023-09-23T15:11:48.023730+0000","last_clean":"2023-09-23T15:11:48.023730+0000","last_became_active":"2023-09-23T15:11:35.188746+0000","last_became_peered":"2023-09-23T15:11:35.188746+0000","last_unstale":"2023-09-23T15:11:48.023730+0000","last_undegraded":"2023-09-23T15:11:48.023730+0000","last_fullsized":"2023-09-23T15:11:48.023730+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":35,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.819396+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:11:47.819396+0000","last_peered":"2023-09-23T15:11:47.819396+0000","last_clean":"2023-09-23T15:11:47.819396+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:11:47.819396+0000","last_undegraded":"2023-09-23T15:11:47.819396+0000","last_fullsized":"2023-09-23T15:11:47.819396+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":151,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.914818+0000","last_change":"2023-09-23T15:11:35.292690+0000","last_active":"2023-09-23T15:11:48.914818+0000","last_peered":"2023-09-23T15:11:48.914818+0000","last_clean":"2023-09-23T15:11:48.914818+0000","last_became_active":"2023-09-23T15:11:35.285740+0000","last_became_peered":"2023-09-23T15:11:35.285740+0000","last_unstale":"2023-09-23T15:11:48.914818+0000","last_undegraded":"2023-09-23T15:11:48.914818+0000","last_fullsized":"2023-09-23T15:11:48.914818+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:18.699392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5,7],"avail_no_missing":["6","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"175'802","reported_seq":815,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.306875+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:11:54.306875+0000","last_peered":"2023-09-23T15:11:54.306875+0000","last_clean":"2023-09-23T15:11:54.306875+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:11:54.306875+0000","last_undegraded":"2023-09-23T15:11:54.306875+0000","last_fullsized":"2023-09-23T15:11:54.306875+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":802,"log_dups_size":0,"ondisk_log_size":802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52559872,"num_objects":802,"num_object_clones":0,"num_object_copies":2406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":802,"num_write_kb":51328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023555+0000","last_change":"2023-09-23T15:11:35.311545+0000","last_active":"2023-09-23T15:11:48.023555+0000","last_peered":"2023-09-23T15:11:48.023555+0000","last_clean":"2023-09-23T15:11:48.023555+0000","last_became_active":"2023-09-23T15:11:35.292953+0000","last_became_peered":"2023-09-23T15:11:35.292953+0000","last_unstale":"2023-09-23T15:11:48.023555+0000","last_undegraded":"2023-09-23T15:11:48.023555+0000","last_fullsized":"2023-09-23T15:11:48.023555+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.819084+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:11:47.819084+0000","last_peered":"2023-09-23T15:11:47.819084+0000","last_clean":"2023-09-23T15:11:47.819084+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:11:47.819084+0000","last_undegraded":"2023-09-23T15:11:47.819084+0000","last_fullsized":"2023-09-23T15:11:47.819084+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":124,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.024031+0000","last_change":"2023-09-23T15:08:51.267790+0000","last_active":"2023-09-23T15:11:48.024031+0000","last_peered":"2023-09-23T15:11:48.024031+0000","last_clean":"2023-09-23T15:08:45.955825+0000","last_became_active":"2023-09-23T15:08:51.267790+0000","last_became_peered":"2023-09-23T15:08:51.267790+0000","last_unstale":"2023-09-23T15:11:48.024031+0000","last_undegraded":"2023-09-23T15:11:48.024031+0000","last_fullsized":"2023-09-23T15:08:51.247802+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:40.856232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"175'845","reported_seq":854,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.311305+0000","last_change":"2023-09-23T15:11:50.080642+0000","last_active":"2023-09-23T15:11:54.311305+0000","last_peered":"2023-09-23T15:11:54.311305+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.383347+0000","last_became_peered":"2023-09-23T15:11:42.383347+0000","last_unstale":"2023-09-23T15:11:54.311305+0000","last_undegraded":"2023-09-23T15:11:50.069646+0000","last_fullsized":"2023-09-23T15:11:42.367078+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":845,"log_dups_size":0,"ondisk_log_size":845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:18.832926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":845,"num_write_kb":54080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":845}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023264+0000","last_change":"2023-09-23T15:11:35.190529+0000","last_active":"2023-09-23T15:11:48.023264+0000","last_peered":"2023-09-23T15:11:48.023264+0000","last_clean":"2023-09-23T15:11:48.023264+0000","last_became_active":"2023-09-23T15:11:35.183876+0000","last_became_peered":"2023-09-23T15:11:35.183876+0000","last_unstale":"2023-09-23T15:11:48.023264+0000","last_undegraded":"2023-09-23T15:11:48.023264+0000","last_fullsized":"2023-09-23T15:11:48.023264+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:25.167633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:25.167633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":136,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.914815+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:11:48.914815+0000","last_peered":"2023-09-23T15:11:48.914815+0000","last_clean":"2023-09-23T15:11:48.914815+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:11:48.914815+0000","last_undegraded":"2023-09-23T15:11:48.914815+0000","last_fullsized":"2023-09-23T15:11:48.914815+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":133,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.815320+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:11:47.815320+0000","last_peered":"2023-09-23T15:11:47.815320+0000","last_clean":"2023-09-23T15:11:47.815320+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:11:47.815320+0000","last_undegraded":"2023-09-23T15:11:47.815320+0000","last_fullsized":"2023-09-23T15:11:47.815320+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"175'1394","reported_seq":1407,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:58.209190+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:11:58.209190+0000","last_peered":"2023-09-23T15:11:58.209190+0000","last_clean":"2023-09-23T15:11:58.209190+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:11:58.209190+0000","last_undegraded":"2023-09-23T15:11:58.209190+0000","last_fullsized":"2023-09-23T15:11:58.209190+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":1394,"log_dups_size":0,"ondisk_log_size":1394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91357184,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1394,"num_write_kb":89216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023072+0000","last_change":"2023-09-23T15:11:35.192805+0000","last_active":"2023-09-23T15:11:48.023072+0000","last_peered":"2023-09-23T15:11:48.023072+0000","last_clean":"2023-09-23T15:11:48.023072+0000","last_became_active":"2023-09-23T15:11:35.192293+0000","last_became_peered":"2023-09-23T15:11:35.192293+0000","last_unstale":"2023-09-23T15:11:48.023072+0000","last_undegraded":"2023-09-23T15:11:48.023072+0000","last_fullsized":"2023-09-23T15:11:48.023072+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:30.198818+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:30.198818+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.914619+0000","last_change":"2023-09-23T15:08:51.258416+0000","last_active":"2023-09-23T15:11:48.914619+0000","last_peered":"2023-09-23T15:11:48.914619+0000","last_clean":"2023-09-23T15:08:45.875112+0000","last_became_active":"2023-09-23T15:08:51.258416+0000","last_became_peered":"2023-09-23T15:08:51.258416+0000","last_unstale":"2023-09-23T15:11:48.914619+0000","last_undegraded":"2023-09-23T15:11:48.914619+0000","last_fullsized":"2023-09-23T15:08:51.250989+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:24.713686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":157,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.815387+0000","last_change":"2023-09-23T15:08:51.256323+0000","last_active":"2023-09-23T15:11:47.815387+0000","last_peered":"2023-09-23T15:11:47.815387+0000","last_clean":"2023-09-23T15:08:45.865599+0000","last_became_active":"2023-09-23T15:08:51.256323+0000","last_became_peered":"2023-09-23T15:08:51.256323+0000","last_unstale":"2023-09-23T15:11:47.815387+0000","last_undegraded":"2023-09-23T15:11:47.815387+0000","last_fullsized":"2023-09-23T15:08:51.253608+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:46.006010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"175'798","reported_seq":811,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.250543+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:11:54.250543+0000","last_peered":"2023-09-23T15:11:54.250543+0000","last_clean":"2023-09-23T15:11:54.250543+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:11:54.250543+0000","last_undegraded":"2023-09-23T15:11:54.250543+0000","last_fullsized":"2023-09-23T15:11:54.250543+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":798,"log_dups_size":0,"ondisk_log_size":798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52297728,"num_objects":798,"num_object_clones":0,"num_object_copies":2394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":798,"num_write_kb":51072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023693+0000","last_change":"2023-09-23T15:11:35.335583+0000","last_active":"2023-09-23T15:11:48.023693+0000","last_peered":"2023-09-23T15:11:48.023693+0000","last_clean":"2023-09-23T15:11:48.023693+0000","last_became_active":"2023-09-23T15:11:35.302795+0000","last_became_peered":"2023-09-23T15:11:35.302795+0000","last_unstale":"2023-09-23T15:11:48.023693+0000","last_undegraded":"2023-09-23T15:11:48.023693+0000","last_fullsized":"2023-09-23T15:11:48.023693+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":146,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:47.814150+0000","last_change":"2023-09-23T15:11:35.179725+0000","last_active":"2023-09-23T15:11:47.814150+0000","last_peered":"2023-09-23T15:11:47.814150+0000","last_clean":"2023-09-23T15:11:47.814150+0000","last_became_active":"2023-09-23T15:11:35.179316+0000","last_became_peered":"2023-09-23T15:11:35.179316+0000","last_unstale":"2023-09-23T15:11:47.814150+0000","last_undegraded":"2023-09-23T15:11:47.814150+0000","last_fullsized":"2023-09-23T15:11:47.814150+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:10:07.161531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2,7],"avail_no_missing":["4","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":140,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.815808+0000","last_change":"2023-09-23T15:08:51.256872+0000","last_active":"2023-09-23T15:11:47.815808+0000","last_peered":"2023-09-23T15:11:47.815808+0000","last_clean":"2023-09-23T15:08:45.866359+0000","last_became_active":"2023-09-23T15:08:51.256872+0000","last_became_peered":"2023-09-23T15:08:51.256872+0000","last_unstale":"2023-09-23T15:11:47.815808+0000","last_undegraded":"2023-09-23T15:11:47.815808+0000","last_fullsized":"2023-09-23T15:08:51.254666+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:08.302388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"175'889","reported_seq":898,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.638538+0000","last_change":"2023-09-23T15:11:50.071485+0000","last_active":"2023-09-23T15:11:54.638538+0000","last_peered":"2023-09-23T15:11:54.638538+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.373151+0000","last_became_peered":"2023-09-23T15:11:42.373151+0000","last_unstale":"2023-09-23T15:11:54.638538+0000","last_undegraded":"2023-09-23T15:11:50.058729+0000","last_fullsized":"2023-09-23T15:11:42.370570+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":889,"log_dups_size":0,"ondisk_log_size":889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:04.009319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58261504,"num_objects":889,"num_object_clones":0,"num_object_copies":2667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":889,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":889,"num_write_kb":56896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":889}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023628+0000","last_change":"2023-09-23T15:11:35.190367+0000","last_active":"2023-09-23T15:11:48.023628+0000","last_peered":"2023-09-23T15:11:48.023628+0000","last_clean":"2023-09-23T15:11:48.023628+0000","last_became_active":"2023-09-23T15:11:35.189065+0000","last_became_peered":"2023-09-23T15:11:35.189065+0000","last_unstale":"2023-09-23T15:11:48.023628+0000","last_undegraded":"2023-09-23T15:11:48.023628+0000","last_fullsized":"2023-09-23T15:11:48.023628+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:49.276278+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:49.276278+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":155,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.916345+0000","last_change":"2023-09-23T15:11:35.244523+0000","last_active":"2023-09-23T15:11:48.916345+0000","last_peered":"2023-09-23T15:11:48.916345+0000","last_clean":"2023-09-23T15:11:48.916345+0000","last_became_active":"2023-09-23T15:11:35.243099+0000","last_became_peered":"2023-09-23T15:11:35.243099+0000","last_unstale":"2023-09-23T15:11:48.916345+0000","last_undegraded":"2023-09-23T15:11:48.916345+0000","last_fullsized":"2023-09-23T15:11:48.916345+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:22.936257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4,7],"avail_no_missing":["3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":90,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.916411+0000","last_change":"2023-09-23T15:08:51.257143+0000","last_active":"2023-09-23T15:11:48.916411+0000","last_peered":"2023-09-23T15:11:48.916411+0000","last_clean":"2023-09-23T15:05:44.712668+0000","last_became_active":"2023-09-23T15:08:51.257143+0000","last_became_peered":"2023-09-23T15:08:51.257143+0000","last_unstale":"2023-09-23T15:11:48.916411+0000","last_undegraded":"2023-09-23T15:11:48.916411+0000","last_fullsized":"2023-09-23T15:08:51.246234+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:10.736408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4],"acting":[3,4],"avail_no_missing":["3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"175'847","reported_seq":860,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.312803+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:11:54.312803+0000","last_peered":"2023-09-23T15:11:54.312803+0000","last_clean":"2023-09-23T15:11:54.312803+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:11:54.312803+0000","last_undegraded":"2023-09-23T15:11:54.312803+0000","last_fullsized":"2023-09-23T15:11:54.312803+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":847,"log_dups_size":0,"ondisk_log_size":847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55508992,"num_objects":847,"num_object_clones":0,"num_object_copies":2541,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":847,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":847,"num_write_kb":54208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023890+0000","last_change":"2023-09-23T15:11:35.190869+0000","last_active":"2023-09-23T15:11:48.023890+0000","last_peered":"2023-09-23T15:11:48.023890+0000","last_clean":"2023-09-23T15:11:48.023890+0000","last_became_active":"2023-09-23T15:11:35.189208+0000","last_became_peered":"2023-09-23T15:11:35.189208+0000","last_unstale":"2023-09-23T15:11:48.023890+0000","last_undegraded":"2023-09-23T15:11:48.023890+0000","last_fullsized":"2023-09-23T15:11:48.023890+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":387,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023440+0000","last_change":"2023-09-23T15:11:35.345351+0000","last_active":"2023-09-23T15:11:48.023440+0000","last_peered":"2023-09-23T15:11:48.023440+0000","last_clean":"2023-09-23T15:11:48.023440+0000","last_became_active":"2023-09-23T15:11:35.195520+0000","last_became_peered":"2023-09-23T15:11:35.195520+0000","last_unstale":"2023-09-23T15:11:48.023440+0000","last_undegraded":"2023-09-23T15:11:48.023440+0000","last_fullsized":"2023-09-23T15:11:48.023440+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:09.178870+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.819334+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:11:47.819334+0000","last_peered":"2023-09-23T15:11:47.819334+0000","last_clean":"2023-09-23T15:11:47.819334+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:11:47.819334+0000","last_undegraded":"2023-09-23T15:11:47.819334+0000","last_fullsized":"2023-09-23T15:11:47.819334+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":95,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:50.836725+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:11:50.836725+0000","last_peered":"2023-09-23T15:11:50.836725+0000","last_clean":"2023-09-23T15:11:50.836725+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:11:50.836725+0000","last_undegraded":"2023-09-23T15:11:50.836725+0000","last_fullsized":"2023-09-23T15:11:50.836725+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"175'810","reported_seq":823,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.238617+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:11:54.238617+0000","last_peered":"2023-09-23T15:11:54.238617+0000","last_clean":"2023-09-23T15:11:54.238617+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:11:54.238617+0000","last_undegraded":"2023-09-23T15:11:54.238617+0000","last_fullsized":"2023-09-23T15:11:54.238617+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":810,"log_dups_size":0,"ondisk_log_size":810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53084160,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":810,"num_write_kb":51840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023856+0000","last_change":"2023-09-23T15:11:35.191247+0000","last_active":"2023-09-23T15:11:48.023856+0000","last_peered":"2023-09-23T15:11:48.023856+0000","last_clean":"2023-09-23T15:11:48.023856+0000","last_became_active":"2023-09-23T15:11:35.184293+0000","last_became_peered":"2023-09-23T15:11:35.184293+0000","last_unstale":"2023-09-23T15:11:48.023856+0000","last_undegraded":"2023-09-23T15:11:48.023856+0000","last_fullsized":"2023-09-23T15:11:48.023856+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":683,"reported_epoch":174,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:47.818895+0000","last_change":"2023-09-23T15:08:51.252973+0000","last_active":"2023-09-23T15:11:47.818895+0000","last_peered":"2023-09-23T15:11:47.818895+0000","last_clean":"2023-09-23T15:08:45.872643+0000","last_became_active":"2023-09-23T15:08:51.252973+0000","last_became_peered":"2023-09-23T15:08:51.252973+0000","last_unstale":"2023-09-23T15:11:47.818895+0000","last_undegraded":"2023-09-23T15:08:51.247667+0000","last_fullsized":"2023-09-23T15:08:51.247548+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:26.001486+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814516+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:11:47.814516+0000","last_peered":"2023-09-23T15:11:47.814516+0000","last_clean":"2023-09-23T15:11:47.814516+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:11:47.814516+0000","last_undegraded":"2023-09-23T15:11:47.814516+0000","last_fullsized":"2023-09-23T15:11:47.814516+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":105,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915525+0000","last_change":"2023-09-23T15:08:51.249038+0000","last_active":"2023-09-23T15:11:48.915525+0000","last_peered":"2023-09-23T15:11:48.915525+0000","last_clean":"2023-09-23T15:05:44.719999+0000","last_became_active":"2023-09-23T15:08:51.249038+0000","last_became_peered":"2023-09-23T15:08:51.249038+0000","last_unstale":"2023-09-23T15:11:48.915525+0000","last_undegraded":"2023-09-23T15:11:48.915525+0000","last_fullsized":"2023-09-23T15:08:51.245952+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:28.251439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"175'817","reported_seq":829,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.154732+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:11:54.154732+0000","last_peered":"2023-09-23T15:11:54.154732+0000","last_clean":"2023-09-23T15:11:54.154732+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:11:54.154732+0000","last_undegraded":"2023-09-23T15:11:54.154732+0000","last_fullsized":"2023-09-23T15:11:54.154732+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":817,"log_dups_size":0,"ondisk_log_size":817,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53542912,"num_objects":817,"num_object_clones":0,"num_object_copies":2451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":817,"num_write_kb":52288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023294+0000","last_change":"2023-09-23T15:11:35.190009+0000","last_active":"2023-09-23T15:11:48.023294+0000","last_peered":"2023-09-23T15:11:48.023294+0000","last_clean":"2023-09-23T15:11:48.023294+0000","last_became_active":"2023-09-23T15:11:35.188452+0000","last_became_peered":"2023-09-23T15:11:35.188452+0000","last_unstale":"2023-09-23T15:11:48.023294+0000","last_undegraded":"2023-09-23T15:11:48.023294+0000","last_fullsized":"2023-09-23T15:11:48.023294+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:55.364609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:55.364609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":786,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:50.836625+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:11:50.836625+0000","last_peered":"2023-09-23T15:11:50.836625+0000","last_clean":"2023-09-23T15:11:50.836625+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:11:50.836625+0000","last_undegraded":"2023-09-23T15:11:50.836625+0000","last_fullsized":"2023-09-23T15:11:50.836625+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.913605+0000","last_change":"2023-09-23T15:08:51.251021+0000","last_active":"2023-09-23T15:11:48.913605+0000","last_peered":"2023-09-23T15:11:48.913605+0000","last_clean":"2023-09-23T15:05:12.641245+0000","last_became_active":"2023-09-23T15:08:51.251021+0000","last_became_peered":"2023-09-23T15:08:51.251021+0000","last_unstale":"2023-09-23T15:11:48.913605+0000","last_undegraded":"2023-09-23T15:11:48.913605+0000","last_fullsized":"2023-09-23T15:08:51.247874+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:57.497456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.819475+0000","last_change":"2023-09-23T15:08:51.268494+0000","last_active":"2023-09-23T15:11:47.819475+0000","last_peered":"2023-09-23T15:11:47.819475+0000","last_clean":"2023-09-23T15:00:28.018761+0000","last_became_active":"2023-09-23T15:08:51.268494+0000","last_became_peered":"2023-09-23T15:08:51.268494+0000","last_unstale":"2023-09-23T15:11:47.819475+0000","last_undegraded":"2023-09-23T15:11:47.819475+0000","last_fullsized":"2023-09-23T15:08:51.248109+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:08.330407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"175'894","reported_seq":906,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.638478+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:11:54.638478+0000","last_peered":"2023-09-23T15:11:54.638478+0000","last_clean":"2023-09-23T15:11:54.638478+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:11:54.638478+0000","last_undegraded":"2023-09-23T15:11:54.638478+0000","last_fullsized":"2023-09-23T15:11:54.638478+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":894,"log_dups_size":0,"ondisk_log_size":894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58589184,"num_objects":894,"num_object_clones":0,"num_object_copies":2682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":894,"num_write_kb":57216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":462,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.818897+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:11:47.818897+0000","last_peered":"2023-09-23T15:11:47.818897+0000","last_clean":"2023-09-23T15:11:47.818897+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:11:47.818897+0000","last_undegraded":"2023-09-23T15:11:47.818897+0000","last_fullsized":"2023-09-23T15:11:47.818897+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023236+0000","last_change":"2023-09-23T15:11:35.196697+0000","last_active":"2023-09-23T15:11:48.023236+0000","last_peered":"2023-09-23T15:11:48.023236+0000","last_clean":"2023-09-23T15:11:48.023236+0000","last_became_active":"2023-09-23T15:11:35.187866+0000","last_became_peered":"2023-09-23T15:11:35.187866+0000","last_unstale":"2023-09-23T15:11:48.023236+0000","last_undegraded":"2023-09-23T15:11:48.023236+0000","last_fullsized":"2023-09-23T15:11:48.023236+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:18.860128+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:18.860128+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":87,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.913886+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:11:48.913886+0000","last_peered":"2023-09-23T15:11:48.913886+0000","last_clean":"2023-09-23T15:11:48.913886+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:11:48.913886+0000","last_undegraded":"2023-09-23T15:11:48.913886+0000","last_fullsized":"2023-09-23T15:11:48.913886+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":125,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.914943+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:11:48.914943+0000","last_peered":"2023-09-23T15:11:48.914943+0000","last_clean":"2023-09-23T15:11:48.914943+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:11:48.914943+0000","last_undegraded":"2023-09-23T15:11:48.914943+0000","last_fullsized":"2023-09-23T15:11:48.914943+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"175'898","reported_seq":906,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.642563+0000","last_change":"2023-09-23T15:11:50.056050+0000","last_active":"2023-09-23T15:11:54.642563+0000","last_peered":"2023-09-23T15:11:54.642563+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.373784+0000","last_became_peered":"2023-09-23T15:11:42.373784+0000","last_unstale":"2023-09-23T15:11:54.642563+0000","last_undegraded":"2023-09-23T15:11:50.055907+0000","last_fullsized":"2023-09-23T15:11:42.369779+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":898,"log_dups_size":0,"ondisk_log_size":898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:44.452791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58851328,"num_objects":898,"num_object_clones":0,"num_object_copies":2694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":898,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":898,"num_write_kb":57472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"3,5","objects":898}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023856+0000","last_change":"2023-09-23T15:11:35.335836+0000","last_active":"2023-09-23T15:11:48.023856+0000","last_peered":"2023-09-23T15:11:48.023856+0000","last_clean":"2023-09-23T15:11:48.023856+0000","last_became_active":"2023-09-23T15:11:35.310380+0000","last_became_peered":"2023-09-23T15:11:35.310380+0000","last_unstale":"2023-09-23T15:11:48.023856+0000","last_undegraded":"2023-09-23T15:11:48.023856+0000","last_fullsized":"2023-09-23T15:11:48.023856+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":297,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023424+0000","last_change":"2023-09-23T15:11:35.376284+0000","last_active":"2023-09-23T15:11:48.023424+0000","last_peered":"2023-09-23T15:11:48.023424+0000","last_clean":"2023-09-23T15:11:48.023424+0000","last_became_active":"2023-09-23T15:11:35.337589+0000","last_became_peered":"2023-09-23T15:11:35.337589+0000","last_unstale":"2023-09-23T15:11:48.023424+0000","last_undegraded":"2023-09-23T15:11:48.023424+0000","last_fullsized":"2023-09-23T15:11:48.023424+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:45:38.290414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":118,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.023586+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:11:48.023586+0000","last_peered":"2023-09-23T15:11:48.023586+0000","last_clean":"2023-09-23T15:11:48.023586+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:11:48.023586+0000","last_undegraded":"2023-09-23T15:11:48.023586+0000","last_fullsized":"2023-09-23T15:11:48.023586+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":139,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.023726+0000","last_change":"2023-09-23T15:08:51.253152+0000","last_active":"2023-09-23T15:11:48.023726+0000","last_peered":"2023-09-23T15:11:48.023726+0000","last_clean":"2023-09-23T15:08:45.956281+0000","last_became_active":"2023-09-23T15:08:51.253152+0000","last_became_peered":"2023-09-23T15:08:51.253152+0000","last_unstale":"2023-09-23T15:11:48.023726+0000","last_undegraded":"2023-09-23T15:11:48.023726+0000","last_fullsized":"2023-09-23T15:08:51.249511+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:06:27.618096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"175'781","reported_seq":790,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.151799+0000","last_change":"2023-09-23T15:11:50.067531+0000","last_active":"2023-09-23T15:11:54.151799+0000","last_peered":"2023-09-23T15:11:54.151799+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.377520+0000","last_became_peered":"2023-09-23T15:11:42.377520+0000","last_unstale":"2023-09-23T15:11:54.151799+0000","last_undegraded":"2023-09-23T15:11:50.056977+0000","last_fullsized":"2023-09-23T15:11:42.372185+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":781,"log_dups_size":0,"ondisk_log_size":781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:36.002206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51183616,"num_objects":781,"num_object_clones":0,"num_object_copies":2343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":781,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":781,"num_write_kb":49984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":781}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.024306+0000","last_change":"2023-09-23T15:11:35.332321+0000","last_active":"2023-09-23T15:11:48.024306+0000","last_peered":"2023-09-23T15:11:48.024306+0000","last_clean":"2023-09-23T15:11:48.024306+0000","last_became_active":"2023-09-23T15:11:35.309922+0000","last_became_peered":"2023-09-23T15:11:35.309922+0000","last_unstale":"2023-09-23T15:11:48.024306+0000","last_undegraded":"2023-09-23T15:11:48.024306+0000","last_fullsized":"2023-09-23T15:11:48.024306+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":756,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023647+0000","last_change":"2023-09-23T15:11:35.263896+0000","last_active":"2023-09-23T15:11:48.023647+0000","last_peered":"2023-09-23T15:11:48.023647+0000","last_clean":"2023-09-23T15:11:48.023647+0000","last_became_active":"2023-09-23T15:11:35.221717+0000","last_became_peered":"2023-09-23T15:11:35.221717+0000","last_unstale":"2023-09-23T15:11:48.023647+0000","last_undegraded":"2023-09-23T15:11:48.023647+0000","last_fullsized":"2023-09-23T15:11:48.023647+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:23:31.485769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915171+0000","last_change":"2023-09-23T15:08:51.252916+0000","last_active":"2023-09-23T15:11:48.915171+0000","last_peered":"2023-09-23T15:11:48.915171+0000","last_clean":"2023-09-23T15:05:12.647697+0000","last_became_active":"2023-09-23T15:08:51.252916+0000","last_became_peered":"2023-09-23T15:08:51.252916+0000","last_unstale":"2023-09-23T15:11:48.915171+0000","last_undegraded":"2023-09-23T15:11:48.915171+0000","last_fullsized":"2023-09-23T15:08:51.248971+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:15.465497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":113,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.024254+0000","last_change":"2023-09-23T15:11:35.336252+0000","last_active":"2023-09-23T15:11:48.024254+0000","last_peered":"2023-09-23T15:11:48.024254+0000","last_clean":"2023-09-23T15:11:48.024254+0000","last_became_active":"2023-09-23T15:11:35.324595+0000","last_became_peered":"2023-09-23T15:11:35.324595+0000","last_unstale":"2023-09-23T15:11:48.024254+0000","last_undegraded":"2023-09-23T15:11:48.024254+0000","last_fullsized":"2023-09-23T15:11:48.024254+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:03.567960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5,7],"avail_no_missing":["0","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"175'777","reported_seq":789,"reported_epoch":175,"state":"active+clean","last_fresh":"2023-09-23T15:11:54.306444+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:11:54.306444+0000","last_peered":"2023-09-23T15:11:54.306444+0000","last_clean":"2023-09-23T15:11:54.306444+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:11:54.306444+0000","last_undegraded":"2023-09-23T15:11:54.306444+0000","last_fullsized":"2023-09-23T15:11:54.306444+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":777,"log_dups_size":0,"ondisk_log_size":777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50921472,"num_objects":777,"num_object_clones":0,"num_object_copies":2331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":777,"num_write_kb":49728,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.023810+0000","last_change":"2023-09-23T15:11:35.273919+0000","last_active":"2023-09-23T15:11:48.023810+0000","last_peered":"2023-09-23T15:11:48.023810+0000","last_clean":"2023-09-23T15:11:48.023810+0000","last_became_active":"2023-09-23T15:11:35.242983+0000","last_became_peered":"2023-09-23T15:11:35.242983+0000","last_unstale":"2023-09-23T15:11:48.023810+0000","last_undegraded":"2023-09-23T15:11:48.023810+0000","last_fullsized":"2023-09-23T15:11:48.023810+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":149,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.916361+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:11:48.916361+0000","last_peered":"2023-09-23T15:11:48.916361+0000","last_clean":"2023-09-23T15:11:48.916361+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:11:48.916361+0000","last_undegraded":"2023-09-23T15:11:48.916361+0000","last_fullsized":"2023-09-23T15:11:48.916361+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":669,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915200+0000","last_change":"2023-09-23T15:08:51.258518+0000","last_active":"2023-09-23T15:11:48.915200+0000","last_peered":"2023-09-23T15:11:48.915200+0000","last_clean":"2023-09-23T14:57:56.384336+0000","last_became_active":"2023-09-23T15:08:51.258518+0000","last_became_peered":"2023-09-23T15:08:51.258518+0000","last_unstale":"2023-09-23T15:11:48.915200+0000","last_undegraded":"2023-09-23T15:11:48.915200+0000","last_fullsized":"2023-09-23T15:08:51.249016+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:28.474890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.916810+0000","last_change":"2023-09-23T15:11:35.244248+0000","last_active":"2023-09-23T15:11:48.916810+0000","last_peered":"2023-09-23T15:11:48.916810+0000","last_clean":"2023-09-23T15:11:48.916810+0000","last_became_active":"2023-09-23T15:11:35.240312+0000","last_became_peered":"2023-09-23T15:11:35.240312+0000","last_unstale":"2023-09-23T15:11:48.916810+0000","last_undegraded":"2023-09-23T15:11:48.916810+0000","last_fullsized":"2023-09-23T15:11:48.916810+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:00.676311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0,7],"avail_no_missing":["3","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"175'845","reported_seq":854,"reported_epoch":175,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:54.157154+0000","last_change":"2023-09-23T15:11:50.069857+0000","last_active":"2023-09-23T15:11:54.157154+0000","last_peered":"2023-09-23T15:11:54.157154+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:11:42.383348+0000","last_became_peered":"2023-09-23T15:11:42.383348+0000","last_unstale":"2023-09-23T15:11:54.157154+0000","last_undegraded":"2023-09-23T15:11:50.069643+0000","last_fullsized":"2023-09-23T15:11:42.371914+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":845,"log_dups_size":0,"ondisk_log_size":845,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:59.247104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":845,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":845,"num_write_kb":54080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":845}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":433,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.024030+0000","last_change":"2023-09-23T15:11:35.178809+0000","last_active":"2023-09-23T15:11:48.024030+0000","last_peered":"2023-09-23T15:11:48.024030+0000","last_clean":"2023-09-23T15:11:48.024030+0000","last_became_active":"2023-09-23T15:11:35.178365+0000","last_became_peered":"2023-09-23T15:11:35.178365+0000","last_unstale":"2023-09-23T15:11:48.024030+0000","last_undegraded":"2023-09-23T15:11:48.024030+0000","last_fullsized":"2023-09-23T15:11:48.024030+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,7],"avail_no_missing":["0","6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":651,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.915376+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:11:48.915376+0000","last_peered":"2023-09-23T15:11:48.915376+0000","last_clean":"2023-09-23T15:11:48.915376+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:11:48.915376+0000","last_undegraded":"2023-09-23T15:11:48.915376+0000","last_fullsized":"2023-09-23T15:11:48.915376+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":127,"reported_epoch":174,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:48.023911+0000","last_change":"2023-09-23T15:08:51.264748+0000","last_active":"2023-09-23T15:11:48.023911+0000","last_peered":"2023-09-23T15:11:48.023911+0000","last_clean":"2023-09-23T15:08:45.956207+0000","last_became_active":"2023-09-23T15:08:51.264748+0000","last_became_peered":"2023-09-23T15:08:51.264748+0000","last_unstale":"2023-09-23T15:11:48.023911+0000","last_undegraded":"2023-09-23T15:08:51.249297+0000","last_fullsized":"2023-09-23T15:08:51.249189+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:07.409724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":116,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814354+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:11:47.814354+0000","last_peered":"2023-09-23T15:11:47.814354+0000","last_clean":"2023-09-23T15:11:47.814354+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:11:47.814354+0000","last_undegraded":"2023-09-23T15:11:47.814354+0000","last_fullsized":"2023-09-23T15:11:47.814354+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":115,"reported_epoch":174,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:11:48.024172+0000","last_change":"2023-09-23T15:11:35.336055+0000","last_active":"2023-09-23T15:11:48.024172+0000","last_peered":"2023-09-23T15:11:48.024172+0000","last_clean":"2023-09-23T15:11:48.024172+0000","last_became_active":"2023-09-23T15:11:35.318924+0000","last_became_peered":"2023-09-23T15:11:35.318924+0000","last_unstale":"2023-09-23T15:11:48.024172+0000","last_undegraded":"2023-09-23T15:11:48.024172+0000","last_fullsized":"2023-09-23T15:11:48.024172+0000","mapping_epoch":167,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:53.294507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":164,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:48.915339+0000","last_change":"2023-09-23T15:08:51.252017+0000","last_active":"2023-09-23T15:11:48.915339+0000","last_peered":"2023-09-23T15:11:48.915339+0000","last_clean":"2023-09-23T15:08:45.982818+0000","last_became_active":"2023-09-23T15:08:51.252017+0000","last_became_peered":"2023-09-23T15:08:51.252017+0000","last_unstale":"2023-09-23T15:11:48.915339+0000","last_undegraded":"2023-09-23T15:11:48.915339+0000","last_fullsized":"2023-09-23T15:08:51.246298+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:53.322639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":143,"reported_epoch":174,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:11:47.819255+0000","last_change":"2023-09-23T15:08:51.259971+0000","last_active":"2023-09-23T15:11:47.819255+0000","last_peered":"2023-09-23T15:11:47.819255+0000","last_clean":"2023-09-23T15:08:45.872826+0000","last_became_active":"2023-09-23T15:08:51.259971+0000","last_became_peered":"2023-09-23T15:08:51.259971+0000","last_unstale":"2023-09-23T15:11:47.819255+0000","last_undegraded":"2023-09-23T15:08:51.247910+0000","last_fullsized":"2023-09-23T15:08:51.247814+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":101,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:46.740727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":157,"reported_epoch":174,"state":"active+undersized","last_fresh":"2023-09-23T15:11:47.815004+0000","last_change":"2023-09-23T15:08:51.256456+0000","last_active":"2023-09-23T15:11:47.815004+0000","last_peered":"2023-09-23T15:11:47.815004+0000","last_clean":"2023-09-23T15:08:45.865354+0000","last_became_active":"2023-09-23T15:08:51.256456+0000","last_became_peered":"2023-09-23T15:08:51.256456+0000","last_unstale":"2023-09-23T15:11:47.815004+0000","last_undegraded":"2023-09-23T15:11:47.815004+0000","last_fullsized":"2023-09-23T15:08:51.253711+0000","mapping_epoch":132,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:07.673880+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":119,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.819240+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:11:47.819240+0000","last_peered":"2023-09-23T15:11:47.819240+0000","last_clean":"2023-09-23T15:11:47.819240+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:11:47.819240+0000","last_undegraded":"2023-09-23T15:11:47.819240+0000","last_fullsized":"2023-09-23T15:11:47.819240+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":150,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814276+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:11:47.814276+0000","last_peered":"2023-09-23T15:11:47.814276+0000","last_clean":"2023-09-23T15:11:47.814276+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:11:47.814276+0000","last_undegraded":"2023-09-23T15:11:47.814276+0000","last_fullsized":"2023-09-23T15:11:47.814276+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":144,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.916162+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:11:48.916162+0000","last_peered":"2023-09-23T15:11:48.916162+0000","last_clean":"2023-09-23T15:11:48.916162+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:11:48.916162+0000","last_undegraded":"2023-09-23T15:11:48.916162+0000","last_fullsized":"2023-09-23T15:11:48.916162+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":159,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.915689+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:11:48.915689+0000","last_peered":"2023-09-23T15:11:48.915689+0000","last_clean":"2023-09-23T15:11:48.915689+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:11:48.915689+0000","last_undegraded":"2023-09-23T15:11:48.915689+0000","last_fullsized":"2023-09-23T15:11:48.915689+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":121,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:48.024150+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:11:48.024150+0000","last_peered":"2023-09-23T15:11:48.024150+0000","last_clean":"2023-09-23T15:11:48.024150+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:11:48.024150+0000","last_undegraded":"2023-09-23T15:11:48.024150+0000","last_fullsized":"2023-09-23T15:11:48.024150+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":143,"reported_epoch":174,"state":"active+clean","last_fresh":"2023-09-23T15:11:47.814075+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:11:47.814075+0000","last_peered":"2023-09-23T15:11:47.814075+0000","last_clean":"2023-09-23T15:11:47.814075+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:11:47.814075+0000","last_undegraded":"2023-09-23T15:11:47.814075+0000","last_fullsized":"2023-09-23T15:11:47.814075+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":949354496,"num_objects":14486,"num_object_clones":0,"num_object_copies":43458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14486,"num_write_kb":927104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2605187072,"data_stored":2605187072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":14486,"ondisk_log_size":14486,"up":42,"acting":42,"num_store_stats":6},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":76,"acting":82,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":4928,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":81,"acting":88,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":42,"acting":63,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":19,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201486,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463122,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":494360,"kb_used_data":410256,"kb_used_omap":0,"kb_used_meta":84096,"kb_avail":93250792,"statfs":{"total":95995035648,"available":95488811008,"internally_reserved":0,"allocated":420102144,"data_stored":419532453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":86114304},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":2.218,"5min":5.7859999999999996,"15min":2.2890000000000001},"min":{"1min":0.32500000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":14.957000000000001,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.38},{"interface":"front","average":{"1min":1.782,"5min":5.9749999999999996,"15min":2.3809999999999998},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":15.047000000000001,"5min":108.452,"15min":108.452},"last":2.464}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":1.2410000000000001,"5min":0.69799999999999995,"15min":0.61499999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.0540000000000003,"5min":6.0540000000000003,"15min":6.0540000000000003},"last":6.1989999999999998},{"interface":"front","average":{"1min":1.0840000000000001,"5min":0.69799999999999995,"15min":0.63300000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4390000000000001,"5min":2.4390000000000001,"15min":2.4390000000000001},"last":0.80600000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":3.899,"5min":12.573,"15min":4.6029999999999998},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":47.362000000000002,"5min":180.374,"15min":180.374},"last":0.55600000000000005},{"interface":"front","average":{"1min":4.3399999999999999,"5min":12.521000000000001,"15min":4.5869999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":48.619999999999997,"5min":181.685,"15min":181.685},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:11:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3889999999999998,"5min":1.762,"15min":0.98999999999999999},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.5969999999999995,"5min":17.084,"15min":17.084},"last":0.436},{"interface":"front","average":{"1min":2.9510000000000001,"5min":1.647,"15min":0.95599999999999996},"min":{"1min":0.36199999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.6129999999999995,"5min":8.6129999999999995,"15min":8.6129999999999995},"last":1.3}]},{"osd":4,"last update":"Sat Sep 23 15:11:04 2023","interfaces":[{"interface":"back","average":{"1min":2.879,"5min":1.544,"15min":0.86199999999999999},"min":{"1min":0.28000000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":12.942,"5min":12.942,"15min":12.942},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.008,"5min":1.669,"15min":0.92200000000000004},"min":{"1min":0.33200000000000002,"5min":0.30399999999999999,"15min":0.25600000000000001},"max":{"1min":9.7840000000000007,"5min":17.120000000000001,"15min":17.120000000000001},"last":0.36199999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:11:04 2023","interfaces":[{"interface":"back","average":{"1min":3.327,"5min":1.675,"15min":0.92700000000000005},"min":{"1min":0.34899999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":12.800000000000001,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.75600000000000001},{"interface":"front","average":{"1min":3.1829999999999998,"5min":1.736,"15min":0.96599999999999997},"min":{"1min":0.32300000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":12.852,"5min":17.224,"15min":17.224},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:11:22 2023","interfaces":[{"interface":"back","average":{"1min":2.794,"5min":1.6220000000000001,"15min":0.94199999999999995},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":7.5910000000000002,"5min":17,"15min":17},"last":0.60499999999999998},{"interface":"front","average":{"1min":2.5739999999999998,"5min":1.5049999999999999,"15min":0.93600000000000005},"min":{"1min":0.38400000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.7809999999999997,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.54400000000000004}]}]},{"osd":1,"up_from":100,"seq":429496729645,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1917576,"kb_used_data":1871936,"kb_used_omap":6,"kb_used_meta":45625,"kb_avail":91827576,"statfs":{"total":95995035648,"available":94031437824,"internally_reserved":0,"allocated":1916862464,"data_stored":1916271201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":46720075},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":0.47299999999999998,"15min":0.437},"min":{"1min":0.222,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.27,"5min":2.27,"15min":2.27},"last":1.8939999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.51300000000000001,"15min":0.49299999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.2480000000000002,"5min":3.2480000000000002,"15min":3.2480000000000002},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:07:49 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.45300000000000001,"15min":0.44900000000000001},"min":{"1min":0.29499999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":1.0269999999999999,"5min":1.621,"15min":1.621},"last":1.169},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.45100000000000001,"15min":0.45700000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":1.099,"5min":1.099,"15min":1.099},"last":1.2549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5049999999999999,"5min":0.82999999999999996,"15min":0.55100000000000005},"min":{"1min":0.35299999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":30.890999999999998,"5min":30.890999999999998,"15min":30.890999999999998},"last":1.873},{"interface":"front","average":{"1min":2.589,"5min":0.871,"15min":0.58499999999999996},"min":{"1min":0.35999999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":30.652999999999999,"5min":30.652999999999999,"15min":30.652999999999999},"last":2.2090000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.1679999999999999,"5min":0.72899999999999998,"15min":0.66500000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":3.8730000000000002,"5min":3.8730000000000002,"15min":3.8730000000000002},"last":1.1339999999999999},{"interface":"front","average":{"1min":1.226,"5min":0.76000000000000001,"15min":0.68600000000000005},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.5090000000000003,"5min":4.5090000000000003,"15min":4.5090000000000003},"last":1.796}]},{"osd":5,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.69099999999999995,"15min":0.65400000000000003},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":3.194,"5min":3.194,"15min":3.194},"last":0.65900000000000003},{"interface":"front","average":{"1min":1.3819999999999999,"5min":0.79400000000000004,"15min":0.70399999999999996},"min":{"1min":0.48999999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.9660000000000002,"5min":4.9660000000000002,"15min":4.9660000000000002},"last":1.2110000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":0.97499999999999998,"5min":0.745,"15min":0.70999999999999996},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":2.5449999999999999,"5min":2.5449999999999999,"15min":2.5449999999999999},"last":1.8280000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.68600000000000005,"15min":0.69999999999999996},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.4910000000000001,"5min":1.4910000000000001,"15min":1.4910000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.165,"5min":0.78900000000000003,"15min":0.73499999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":2.609,"5min":2.609,"15min":2.609},"last":1.8500000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.76000000000000001,"15min":0.73699999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":3.5699999999999998,"5min":3.5699999999999998,"15min":3.5699999999999998},"last":0.63200000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893209,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":397776,"kb_used_data":206728,"kb_used_omap":8,"kb_used_meta":191031,"kb_avail":93347376,"statfs":{"total":95995035648,"available":95587713024,"internally_reserved":0,"allocated":211689472,"data_stored":210956072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":195616340},"hb_peers":[2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:08:16 2023","interfaces":[{"interface":"back","average":{"1min":1.839,"5min":0.65600000000000003,"15min":0.44400000000000001},"min":{"1min":0.255,"5min":0.214,"15min":0.214},"max":{"1min":9.9039999999999999,"5min":9.9039999999999999,"15min":9.9039999999999999},"last":2.347},{"interface":"front","average":{"1min":1.157,"5min":0.53300000000000003,"15min":0.42199999999999999},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":5.1429999999999998,"5min":5.1429999999999998,"15min":5.1429999999999998},"last":3.6269999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":6.0949999999999998,"5min":4.2649999999999997,"15min":1.6899999999999999},"min":{"1min":0.20799999999999999,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":103.81,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":5.7380000000000004,"5min":3.8559999999999999,"15min":1.5580000000000001},"min":{"1min":0.23999999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":108.19499999999999,"5min":122.071,"15min":122.071},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 15:11:21 2023","interfaces":[{"interface":"back","average":{"1min":1.786,"5min":1.506,"15min":0.81399999999999995},"min":{"1min":0.20499999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":9.5069999999999997,"5min":9.5069999999999997,"15min":9.5069999999999997},"last":1.528},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.8200000000000001,"15min":1.278},"min":{"1min":0.25700000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.883,"5min":11.978,"15min":11.978},"last":1.5680000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8080000000000001,"5min":1.909,"15min":1.101},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.247,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.50700000000000001},{"interface":"front","average":{"1min":1.6559999999999999,"5min":1.4410000000000001,"15min":0.90600000000000003},"min":{"1min":0.34100000000000003,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.5429999999999993,"5min":9.5429999999999993,"15min":9.5429999999999993},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.7310000000000001,"15min":1.0409999999999999},"min":{"1min":0.39600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.7589999999999999,"5min":13.49,"15min":13.49},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.698,"15min":1.0840000000000001},"min":{"1min":0.42199999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.3290000000000006,"5min":11.929,"15min":11.929},"last":0.54500000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.6899999999999999,"5min":1.7689999999999999,"15min":1.052},"min":{"1min":0.39200000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.3469999999999995,"5min":11.641999999999999,"15min":11.641999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.2150000000000001,"5min":1.595,"15min":0.98499999999999999},"min":{"1min":0.38800000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":6.0039999999999996,"5min":6.3940000000000001,"15min":6.3940000000000001},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:11:53 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.7390000000000001,"15min":1.0289999999999999},"min":{"1min":0.43099999999999999,"5min":0.374,"15min":0.374},"max":{"1min":7.4619999999999997,"5min":8.2989999999999995,"15min":8.2989999999999995},"last":0.432},{"interface":"front","average":{"1min":1.383,"5min":1.7170000000000001,"15min":1.079},"min":{"1min":0.40600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":7.2149999999999999,"5min":9.109,"15min":9.109},"last":0.52700000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566157,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":935064,"kb_used_data":773008,"kb_used_omap":11,"kb_used_meta":162036,"kb_avail":92810088,"statfs":{"total":95995035648,"available":95037530112,"internally_reserved":0,"allocated":791560192,"data_stored":790823101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":165925648},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":2.944,"5min":1.2709999999999999,"15min":0.69099999999999995},"min":{"1min":0.20000000000000001,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":30.402000000000001,"5min":30.402000000000001,"15min":30.402000000000001},"last":4.1580000000000004},{"interface":"front","average":{"1min":2.976,"5min":1.2609999999999999,"15min":0.67700000000000005},"min":{"1min":0.26200000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":30.550000000000001,"5min":30.550000000000001,"15min":30.550000000000001},"last":2.1589999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":0.20499999999999999},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 15:11:21 2023","interfaces":[{"interface":"back","average":{"1min":1.629,"5min":1.681,"15min":0.81999999999999995},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":4.9699999999999998,"5min":57.695,"15min":57.695},"last":1.1220000000000001},{"interface":"front","average":{"1min":1.373,"5min":1.8280000000000001,"15min":0.876},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":5.4779999999999998,"5min":58.387999999999998,"15min":58.387999999999998},"last":1.5629999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":4.3360000000000003,"5min":1.714,"15min":1.0049999999999999},"min":{"1min":0.308,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":52.795999999999999,"5min":52.795999999999999,"15min":52.795999999999999},"last":1.7050000000000001},{"interface":"front","average":{"1min":7.0339999999999998,"5min":2.5270000000000001,"15min":1.2849999999999999},"min":{"1min":0.36099999999999999,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":68.894999999999996,"5min":68.894999999999996,"15min":68.894999999999996},"last":1.1579999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":5.2309999999999999,"5min":2.2730000000000001,"15min":1.181},"min":{"1min":0.42499999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":41.723999999999997,"5min":41.723999999999997,"15min":41.723999999999997},"last":1.6060000000000001},{"interface":"front","average":{"1min":5.5410000000000004,"5min":2.2040000000000002,"15min":1.1899999999999999},"min":{"1min":0.35099999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":39.822000000000003,"5min":39.822000000000003,"15min":39.822000000000003},"last":1.7330000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":6.008,"5min":2.2610000000000001,"15min":1.2410000000000001},"min":{"1min":0.35299999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":37.834000000000003,"5min":37.834000000000003,"15min":37.834000000000003},"last":1.7729999999999999},{"interface":"front","average":{"1min":4.3339999999999996,"5min":1.9970000000000001,"15min":1.139},"min":{"1min":0.437,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":37.749000000000002,"5min":37.749000000000002,"15min":37.749000000000002},"last":1.1930000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6539999999999999,"5min":1.6850000000000001,"15min":1.0329999999999999},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.1280000000000001,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":1.8049999999999999},{"interface":"front","average":{"1min":3.0179999999999998,"5min":1.655,"15min":1.0169999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":8.7319999999999993,"5min":8.7319999999999993,"15min":8.7319999999999993},"last":1.667}]}]},{"osd":3,"up_from":110,"seq":472446402630,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":639648,"kb_used_data":558040,"kb_used_omap":3,"kb_used_meta":81596,"kb_avail":93105504,"statfs":{"total":95995035648,"available":95340036096,"internally_reserved":0,"allocated":571432960,"data_stored":570742103,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":83554473},"hb_peers":[0,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":5.0839999999999996,"5min":3.2890000000000001,"15min":1.347},"min":{"1min":0.191,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":61.744,"5min":79.611000000000004,"15min":79.611000000000004},"last":0.68300000000000005},{"interface":"front","average":{"1min":4.8490000000000002,"5min":3.3660000000000001,"15min":1.3799999999999999},"min":{"1min":0.17599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":61.686,"5min":79.656000000000006,"15min":79.656000000000006},"last":1.8340000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":9.0030000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":6.6150000000000002,"5min":7.0490000000000004,"15min":2.7629999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":63.009999999999998,"5min":109.077,"15min":109.077},"last":0.41399999999999998},{"interface":"front","average":{"1min":7.7160000000000002,"5min":7.774,"15min":2.8730000000000002},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":69.715999999999994,"5min":119.72799999999999,"15min":119.72799999999999},"last":1.8169999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.3500000000000001,"15min":0.84899999999999998},"min":{"1min":0.35699999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.4740000000000002,"5min":8.9700000000000006,"15min":8.9700000000000006},"last":1.7370000000000001},{"interface":"front","average":{"1min":2.1110000000000002,"5min":1.738,"15min":1.073},"min":{"1min":0.32900000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.2249999999999996,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":2.548,"5min":2.0670000000000002,"15min":1.115},"min":{"1min":0.38200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":7.085,"5min":56.960999999999999,"15min":56.960999999999999},"last":1.7909999999999999},{"interface":"front","average":{"1min":3.254,"5min":2.552,"15min":1.3859999999999999},"min":{"1min":0.38400000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":7.2619999999999996,"5min":56.981999999999999,"15min":56.981999999999999},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1579999999999999,"5min":1.7270000000000001,"15min":1.0249999999999999},"min":{"1min":0.39400000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":24.318000000000001,"5min":24.318000000000001,"15min":24.318000000000001},"last":1.8540000000000001},{"interface":"front","average":{"1min":3.3879999999999999,"5min":2.2330000000000001,"15min":1.3009999999999999},"min":{"1min":0.371,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":19.355,"5min":19.355,"15min":19.355},"last":0.503}]},{"osd":7,"last update":"Sat Sep 23 15:11:24 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":2.3570000000000002,"15min":1.3129999999999999},"min":{"1min":0.40999999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.6799999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.679,"5min":1.4259999999999999,"15min":0.91000000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":5.6849999999999996,"5min":13.403,"15min":13.403},"last":0.45100000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986395,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":371668,"kb_used_data":217100,"kb_used_omap":0,"kb_used_meta":154560,"kb_avail":93373484,"statfs":{"total":95995035648,"available":95614447616,"internally_reserved":0,"allocated":222310400,"data_stored":221744284,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":158269440},"hb_peers":[0,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.7470000000000001,"5min":4.9820000000000002,"15min":2.0190000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":7.7910000000000004,"5min":122.69199999999999,"15min":122.69199999999999},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.78600000000000003,"5min":5.0739999999999998,"15min":2.0529999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":3.4390000000000001,"5min":122.745,"15min":122.745},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:17 2023","interfaces":[{"interface":"back","average":{"1min":4.8099999999999996,"5min":1.3919999999999999,"15min":0.81899999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":67.542000000000002,"5min":67.542000000000002,"15min":67.542000000000002},"last":0.82899999999999996},{"interface":"front","average":{"1min":4.6150000000000002,"5min":1.4339999999999999,"15min":0.88500000000000001},"min":{"1min":0.36399999999999999,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":67.808999999999997,"5min":67.808999999999997,"15min":67.808999999999997},"last":0.53300000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:11:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":4.6589999999999998,"15min":1.9470000000000001},"min":{"1min":0.309,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.4940000000000002,"5min":109.541,"15min":109.541},"last":0.72799999999999998},{"interface":"front","average":{"1min":0.92100000000000004,"5min":4.976,"15min":2.1499999999999999},"min":{"1min":0.34200000000000003,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":3.46,"5min":109.619,"15min":109.619},"last":0.67000000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:11:23 2023","interfaces":[{"interface":"back","average":{"1min":2.7000000000000002,"5min":1.8959999999999999,"15min":1.006},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":13.946999999999999,"5min":13.946999999999999,"15min":13.946999999999999},"last":0.68999999999999995},{"interface":"front","average":{"1min":1.2390000000000001,"5min":1.133,"15min":0.78500000000000003},"min":{"1min":0.316,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":9.0670000000000002,"5min":9.0670000000000002,"15min":9.0670000000000002},"last":2.0699999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:11:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2080000000000002,"5min":2.5310000000000001,"15min":1.1990000000000001},"min":{"1min":0.28699999999999998,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":14.073,"5min":28.891999999999999,"15min":28.891999999999999},"last":1.034},{"interface":"front","average":{"1min":2.2410000000000001,"5min":1.992,"15min":1.0369999999999999},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":8.9920000000000009,"5min":32.100999999999999,"15min":32.100999999999999},"last":1.859}]},{"osd":6,"last update":"Sat Sep 23 15:11:11 2023","interfaces":[{"interface":"back","average":{"1min":3.7330000000000001,"5min":1.9219999999999999,"15min":1.0389999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":13.974,"5min":13.974,"15min":13.974},"last":1.417},{"interface":"front","average":{"1min":2.2789999999999999,"5min":1.542,"15min":0.90900000000000003},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":9.6760000000000002,"5min":14.711,"15min":14.711},"last":1.897}]},{"osd":7,"last update":"Sat Sep 23 15:11:17 2023","interfaces":[{"interface":"back","average":{"1min":2.0110000000000001,"5min":2.1040000000000001,"15min":1.113},"min":{"1min":0.36599999999999999,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":7.7389999999999999,"5min":18.23,"15min":18.23},"last":0.70099999999999996},{"interface":"front","average":{"1min":3.4079999999999999,"5min":2.048,"15min":1.1319999999999999},"min":{"1min":0.38600000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":14.044,"5min":14.653,"15min":14.653},"last":0.879}]}]},{"osd":5,"up_from":39,"seq":167503724758,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":379400,"kb_used_data":227200,"kb_used_omap":0,"kb_used_meta":152192,"kb_avail":93365752,"statfs":{"total":95995035648,"available":95606530048,"internally_reserved":0,"allocated":232652800,"data_stored":232094735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":155844608},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:10:51 2023","interfaces":[{"interface":"back","average":{"1min":3.48,"5min":1.7350000000000001,"15min":0.94299999999999995},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":17.044,"5min":24.962,"15min":24.962},"last":0.377},{"interface":"front","average":{"1min":2.6949999999999998,"5min":1.671,"15min":0.91400000000000003},"min":{"1min":0.23499999999999999,"5min":0.222,"15min":0.222},"max":{"1min":14.529,"5min":25.317,"15min":25.317},"last":0.63800000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:08:22 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":0.87,"15min":0.66200000000000003},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":9.2870000000000008,"5min":9.2870000000000008,"15min":9.2870000000000008},"last":0.25700000000000001},{"interface":"front","average":{"1min":2.0219999999999998,"5min":0.86899999999999999,"15min":0.66700000000000004},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.2759999999999998,"5min":9.2759999999999998,"15min":9.2759999999999998},"last":0.312}]},{"osd":2,"last update":"Sat Sep 23 15:10:54 2023","interfaces":[{"interface":"back","average":{"1min":10.397,"5min":3.556,"15min":1.5600000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":97.087999999999994,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.61199999999999999},{"interface":"front","average":{"1min":10.038,"5min":3.3159999999999998,"15min":1.498},"min":{"1min":0.26800000000000002,"5min":0.254,"15min":0.254},"max":{"1min":97.305999999999997,"5min":97.305999999999997,"15min":97.305999999999997},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:11:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":2.226,"15min":1.1459999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":9.6950000000000003,"5min":15.265000000000001,"15min":15.265000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":3.3650000000000002,"5min":1.8180000000000001,"15min":1.0229999999999999},"min":{"1min":0.34599999999999997,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":16.954000000000001,"5min":16.954000000000001,"15min":16.954000000000001},"last":0.56799999999999995}]},{"osd":4,"last update":"Sat Sep 23 15:11:40 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":1.72,"15min":0.89400000000000002},"min":{"1min":0.311,"5min":0.26800000000000002,"15min":0.26300000000000001},"max":{"1min":9.5600000000000005,"5min":11.539999999999999,"15min":11.539999999999999},"last":0.66600000000000004},{"interface":"front","average":{"1min":3.5550000000000002,"5min":2.0089999999999999,"15min":1.026},"min":{"1min":0.33900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":16.763999999999999,"5min":16.763999999999999,"15min":16.763999999999999},"last":0.76800000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:10:59 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":1.9550000000000001,"15min":1.008},"min":{"1min":0.40100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":11.076000000000001,"5min":15.196,"15min":15.196},"last":0.71599999999999997},{"interface":"front","average":{"1min":3.3290000000000002,"5min":1.5669999999999999,"15min":0.92900000000000005},"min":{"1min":0.33300000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":14.965999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.81899999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:11:29 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":2.3759999999999999,"15min":1.1879999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.28299999999999997},"max":{"1min":9.6560000000000006,"5min":15.301,"15min":15.301},"last":0.68899999999999995},{"interface":"front","average":{"1min":3.5779999999999998,"5min":2.347,"15min":1.1839999999999999},"min":{"1min":0.41699999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":15.762,"5min":15.762,"15min":15.762},"last":0.79400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":216006656,"data_stored":216006656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":822673408,"data_stored":822673408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":585498624,"data_stored":585498624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":270794752,"data_stored":270794752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":291831808,"data_stored":291831808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":418381824,"data_stored":418381824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:11:59.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_1 pgp_num 16 2023-09-23T15:11:59.946 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:11:59.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:11:59.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:00.008 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:59 smithi017 ceph-mon[215646]: pgmap v432: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 910 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5077/43542 objects degraded (11.660%); 6/43542 objects misplaced (0.014%) 2023-09-23T15:12:00.008 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:11:59 smithi017 ceph-mon[215646]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:12:00.009 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:59 smithi017 ceph-mon[222878]: pgmap v432: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 910 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5077/43542 objects degraded (11.660%); 6/43542 objects misplaced (0.014%) 2023-09-23T15:12:00.010 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:11:59 smithi017 ceph-mon[222878]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:12:00.051 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:00.070 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 26349 26333 164.233 126.688 0.00723859 0.00608376 2023-09-23T15:12:00.087 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:59 smithi027 ceph-mon[195233]: pgmap v432: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 910 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 91 MiB/s wr, 1.45k op/s; 5077/43542 objects degraded (11.660%); 6/43542 objects misplaced (0.014%) 2023-09-23T15:12:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:11:59 smithi027 ceph-mon[195233]: from='client.54964 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:12:00.155 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:00.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:00.261 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:00.292 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:00.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:00.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:00.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:00.444 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:00.478 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:00.560 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:00.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:00.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:00.742 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:00.817 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:00.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:00.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:00.958 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:00.985 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:01.018 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:01.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:00 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 9537/73827 objects degraded (12.918%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:01.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:01.074 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 28001 27985 158.669 103.25 0.00962215 0.00629718 2023-09-23T15:12:01.095 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:01.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:01.165 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:01.179 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:01.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:00 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 9537/73827 objects degraded (12.918%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:01.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:00 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 9537/73827 objects degraded (12.918%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:01.230 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:01.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:01.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:01.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:01.411 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:01.582 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:01.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:01.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:01.784 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:02.073 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 29939 29923 155.545 121.125 0.00680826 0.00642266 2023-09-23T15:12:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:01 smithi017 ceph-mon[215646]: pgmap v433: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:01 smithi017 ceph-mon[222878]: pgmap v433: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:01 smithi027 ceph-mon[195233]: pgmap v433: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:02.928 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:12:03.073 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 32095 32079 153.947 134.75 0.00934434 0.00649101 2023-09-23T15:12:03.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:03.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:03.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:03.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: pgmap v434: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:03.900 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:03.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:03.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: osdmap e176: 8 total, 7 up, 7 in 2023-09-23T15:12:03.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: failed to encode map e176 with expected crc 2023-09-23T15:12:03.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[222878]: failed to encode map e176 with expected crc 2023-09-23T15:12:04.073 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 34206 34190 152.377 131.938 0.00383137 0.00653915 2023-09-23T15:12:04.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: pgmap v434: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: osdmap e176: 8 total, 7 up, 7 in 2023-09-23T15:12:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: failed to encode map e176 with expected crc 2023-09-23T15:12:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:03 smithi017 ceph-mon[215646]: failed to encode map e176 with expected crc 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: pgmap v434: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.5 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 135 MiB/s wr, 2.15k op/s; 9537/73827 objects degraded (12.918%); 6/73827 objects misplaced (0.008%) 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4052277804' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: osdmap e176: 8 total, 7 up, 7 in 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: failed to encode map e176 with expected crc 2023-09-23T15:12:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:03 smithi027 ceph-mon[195233]: failed to encode map e176 with expected crc 2023-09-23T15:12:05.073 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 35808 35792 148.898 100.125 0.0106369 0.00671024 2023-09-23T15:12:05.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:04 smithi017 ceph-mon[215646]: pgmap v436: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 150 MiB/s wr, 2.39k op/s; 10709/83103 objects degraded (12.886%); 6/83103 objects misplaced (0.007%) 2023-09-23T15:12:05.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:04 smithi017 ceph-mon[222878]: pgmap v436: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 150 MiB/s wr, 2.39k op/s; 10709/83103 objects degraded (12.886%); 6/83103 objects misplaced (0.007%) 2023-09-23T15:12:05.194 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:05.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:05.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:05.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:05.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:04 smithi027 ceph-mon[195233]: pgmap v436: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 1.7 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 150 MiB/s wr, 2.39k op/s; 10709/83103 objects degraded (12.886%); 6/83103 objects misplaced (0.007%) 2023-09-23T15:12:05.379 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:05.563 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:05.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:05.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:05.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:05.738 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:05.920 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:05.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:06.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:06.076 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 37356 37340 145.621 96.75 0.0104524 0.00686106 2023-09-23T15:12:06.094 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:06.203 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:06.207 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:06.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:06.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:06.341 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:06.351 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:06.387 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:06.402 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:06.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:06.429 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:06.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/255894924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:06.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/255894924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:06.476 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:37950 6.5 e176/171 6:aca1a138:::benchmark_data_smithi017_350392_object37949:head v 176'2388, mlcod=176'2386)", 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:06.486005+0000", 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0040760019999999996, 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041002130000000001, 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:12:06.494 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486005+0000", 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486005+0000", 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486004+0000", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486084+0000", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0506999999999993e-05 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486095+0000", 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0723999999999999e-05 2023-09-23T15:12:06.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486102+0000", 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9789999999999996e-06 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486152+0000", 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9716999999999998e-05 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486175+0000", 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.266e-05 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:37941 6.0 e176/171 6:054f6d9f:::benchmark_data_smithi017_350392_object37940:head v 176'2411, mlcod=176'2408)", 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:06.486556+0000", 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0035245200000000002, 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003591014, 2023-09-23T15:12:06.498 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486556+0000", 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486556+0000", 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486555+0000", 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486578+0000", 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2714e-05 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:12:06.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486593+0000", 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5092000000000001e-05 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486600+0000", 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6980000000000001e-06 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.489152+0000", 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025521099999999998 2023-09-23T15:12:06.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.489177+0000", 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4618e-05 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:37948 6.9 e176/171 6:9f844241:::benchmark_data_smithi017_350392_object37947:head v 176'2361, mlcod=176'2359)", 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:06.486609+0000", 2023-09-23T15:12:06.502 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0034713050000000001, 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035743390000000002, 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486609+0000", 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:12:06.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486609+0000", 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486609+0000", 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486627+0000", 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8096000000000001e-05 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486633+0000", 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3470000000000004e-06 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486638+0000", 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3769999999999998e-06 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:12:06.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486685+0000", 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7435000000000002e-05 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:06.486707+0000", 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1985999999999999e-05 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 3 2023-09-23T15:12:06.506 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:12:06.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:06.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:12:06.585 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/255894924' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:07.076 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 39009 38993 143.135 103.312 0.00539059 0.00698232 2023-09-23T15:12:07.403 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 6 pgp_num to 16 2023-09-23T15:12:07.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:12:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:07 smithi027 ceph-mon[195233]: pgmap v437: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 13087/103194 objects degraded (12.682%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[215646]: pgmap v437: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 13087/103194 objects degraded (12.682%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[222878]: pgmap v437: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 133 MiB/s wr, 2.13k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 13087/103194 objects degraded (12.682%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:12:08.076 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 40507 40491 140.388 93.625 0.0467333 0.00711353 2023-09-23T15:12:08.332 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:08] "GET /metrics HTTP/1.1" 200 36301 "" "Prometheus/2.43.0" 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: 1.11 scrub starts 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: 1.11 scrub ok 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: osdmap e177: 8 total, 7 up, 7 in 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.334 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[222878]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.551 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:08.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:08.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: 1.11 scrub starts 2023-09-23T15:12:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: 1.11 scrub ok 2023-09-23T15:12:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:12:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: osdmap e177: 8 total, 7 up, 7 in 2023-09-23T15:12:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:08 smithi027 ceph-mon[195233]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:08.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: 1.11 scrub starts 2023-09-23T15:12:08.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: 1.11 scrub ok 2023-09-23T15:12:08.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2482022244' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_1", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:12:08.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: osdmap e177: 8 total, 7 up, 7 in 2023-09-23T15:12:08.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:08 smithi017 ceph-mon[215646]: failed to encode map e177 with expected crc 2023-09-23T15:12:08.684 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:08.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:12:08.980 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:08.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:08.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:09.077 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 41771 41755 137.156 79 0.00914193 0.00728525 2023-09-23T15:12:09.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:09.132 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:12:09.177 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:12:09.197 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:09.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:09.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:09.278 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:09.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:09.339 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:09 smithi017 ceph-mon[222878]: pgmap v439: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:09.346 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:09.413 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:09.418 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:09 smithi027 ceph-mon[195233]: pgmap v439: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:09.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:09.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:09.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:09.578 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:09.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:09 smithi017 ceph-mon[215646]: pgmap v439: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.1 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s; 13087/103194 objects degraded (12.682%); 6/103194 objects misplaced (0.006%) 2023-09-23T15:12:09.725 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:09.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:09.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:09.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:09.876 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:09.893 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:10.065 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:10.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:10.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:10.077 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:12:10.078211+0000 min lat: 0.00114865 max lat: 0.217231 avg lat: 0.00739896 2023-09-23T15:12:10.077 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:12:10.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 43301 43285 135.077 95.625 0.00591159 0.00739896 2023-09-23T15:12:10.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:10.212 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:10.299 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:10.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:10.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:10.461 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:10.521 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:10.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:10.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:10.670 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:10.754 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:10.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:10.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:10.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:10.903 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:10.980 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:10.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:10.985 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:11.077 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 44881 44865 133.348 98.75 0.0112889 0.00749198 2023-09-23T15:12:11.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:11.172 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:11.178 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:12:11.302 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:11.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:11.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:11.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:11.453 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:11.517 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:11.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:11.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:11.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:11.653 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:11 smithi027 ceph-mon[195233]: pgmap v440: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:11.657 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:11.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:11 smithi017 ceph-mon[215646]: pgmap v440: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:11.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:11 smithi017 ceph-mon[222878]: pgmap v440: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:11.701 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:11.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:11.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:11.806 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:11.878 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:12.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 45981 45965 130.415 68.75 0.00287207 0.00766234 2023-09-23T15:12:12.236 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:12.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:12.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:12.430 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:12.604 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 15949/126051 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 15949/126051 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:12.767 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:12.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:12.772 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:12.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 15949/126051 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:12.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:12.909 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:12.977 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:12.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:12.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:13.084 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:12:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:12:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:12:13.084 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:12:13.084 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:12:13.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 47435 47419 128.695 90.875 0.0167471 0.00776604 2023-09-23T15:12:13.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:13.179 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:13.851 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: pgmap v441: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: 1.2 deep-scrub starts 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: 1.2 deep-scrub ok 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1643861588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:13.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: pgmap v441: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: 1.2 deep-scrub starts 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: 1.2 deep-scrub ok 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1643861588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:13.877 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:13.981 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:12:13.981 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:12:13.981 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T15:12:13.981 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:12:13.981 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:12:13.982 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T15:12:14.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 48852 48836 127.024 88.5625 0.102812 0.00784323 2023-09-23T15:12:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: pgmap v441: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.6 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 116 MiB/s wr, 1.86k op/s; 15949/126051 objects degraded (12.653%); 5/126051 objects misplaced (0.004%) 2023-09-23T15:12:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: 1.2 deep-scrub starts 2023-09-23T15:12:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: 1.2 deep-scrub ok 2023-09-23T15:12:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1643861588' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:14.500 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:15.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 50280 50264 125.514 89.25 0.00665869 0.00796251 2023-09-23T15:12:15.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:15.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[215646]: failed to encode map e178 with expected crc 2023-09-23T15:12:15.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:15.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[215646]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T15:12:15.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:15.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[222878]: failed to encode map e178 with expected crc 2023-09-23T15:12:15.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:15.093 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:14 smithi017 ceph-mon[222878]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T15:12:15.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:14 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:15.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:14 smithi027 ceph-mon[195233]: failed to encode map e178 with expected crc 2023-09-23T15:12:15.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:15.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:14 smithi027 ceph-mon[195233]: osdmap e178: 8 total, 7 up, 7 in 2023-09-23T15:12:15.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:12:15.497 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:15.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:15.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:15.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:15.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:15.786 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:15.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:15.794 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:15.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:15.979 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:16.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:15 smithi017 ceph-mon[215646]: pgmap v443: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 16847/132654 objects degraded (12.700%); 5/132654 objects misplaced (0.004%) 2023-09-23T15:12:16.046 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:15 smithi017 ceph-mon[222878]: pgmap v443: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 16847/132654 objects degraded (12.700%); 5/132654 objects misplaced (0.004%) 2023-09-23T15:12:16.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 51623 51607 123.916 83.9375 0.024185 0.00806534 2023-09-23T15:12:16.081 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:16.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:16.090 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:15 smithi027 ceph-mon[195233]: pgmap v443: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 16847/132654 objects degraded (12.700%); 5/132654 objects misplaced (0.004%) 2023-09-23T15:12:16.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:16.211 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:16.241 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:16.291 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:16.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:16.293 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:16.340 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:16.360 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:16.379 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:16.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:16.397 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:16.418 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:16.421 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:16.464 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:16.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:16.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:16.568 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:16.610 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:16.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.644 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:12:16.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1969381740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:16.861 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:16 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 16847/132654 objects degraded (12.700%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:16.861 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1969381740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:16.861 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:16 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 16847/132654 objects degraded (12.700%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:16.871 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:16.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:16.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:17.037 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:17.079 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 52685 52669 121.787 66.375 0.00620293 0.00820641 2023-09-23T15:12:17.190 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:17.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1969381740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:16 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 16847/132654 objects degraded (12.700%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:17.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:17.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:17.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:17.330 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:17.377 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:17.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:17.420 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:17.420 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:12:17.420 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:17.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:17.515 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:17.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:12:17.649 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:17.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:17.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:17.798 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:17.842 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:17.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:17.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:17 smithi017 ceph-mon[215646]: pgmap v444: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:17.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:17 smithi017 ceph-mon[222878]: pgmap v444: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:17.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:17.972 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:18.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:18.048 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:17 smithi027 ceph-mon[195233]: pgmap v444: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:18.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:18.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:18.078 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 53587 53571 119.453 56.375 0.00904728 0.00836673 2023-09-23T15:12:18.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:18.243 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:18.245 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:12:18.246 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:12:18.276 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:18] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-09-23T15:12:18.331 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:18.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:18.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:18.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:18.454 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:18.479 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:18.549 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:18.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:18.555 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:18.655 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:18.698 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:18.768 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:18.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:18.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:18.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:18.949 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:19.079 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 54985 54969 118.348 87.375 0.00662199 0.00844445 2023-09-23T15:12:19.112 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:19.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:19.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:19.131 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:19.131 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:19.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:19.262 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:19.339 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:19.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:19.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:19.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:19.501 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:20.080 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 56154 56138 116.839 73.0625 0.163464 0.00853448 2023-09-23T15:12:20.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:19 smithi017 ceph-mon[215646]: pgmap v445: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:20.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:19 smithi017 ceph-mon[222878]: pgmap v445: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:20.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:12:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:19 smithi027 ceph-mon[195233]: pgmap v445: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.0 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 92 MiB/s wr, 1.47k op/s; 18665/147477 objects degraded (12.656%); 5/147477 objects misplaced (0.003%) 2023-09-23T15:12:20.462 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:20.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:20.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:20.595 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:20.993 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:21.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:21.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:21.081 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 57392 57376 115.557 77.375 0.0136411 0.00864773 2023-09-23T15:12:21.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:12:21.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:21.160 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:21.417 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:21.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:21.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:21.539 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:21.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:21.616 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:21.828 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:21.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:21.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:21.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:22.017 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:22.082 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 58690 58674 114.482 81.125 0.0150008 0.00873042 2023-09-23T15:12:22.160 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:21 smithi027 ceph-mon[195233]: pgmap v446: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:22.160 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:21 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 20960/165675 objects degraded (12.651%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:21 smithi017 ceph-mon[215646]: pgmap v446: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:21 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 20960/165675 objects degraded (12.651%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:22.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:21 smithi017 ceph-mon[222878]: pgmap v446: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:22.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:21 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 20960/165675 objects degraded (12.651%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:22.228 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:22.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:22.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:22.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:22.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:22.474 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:22.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:22.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:22.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:22.641 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:22.711 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:22.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:22.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:22.814 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:22.874 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:22.886 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1757849341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:22.887 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:22.909 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:12:22.937 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:22.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:23.040 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:23.082 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 60041 60025 113.572 84.4375 0.00912904 0.00879946 2023-09-23T15:12:23.101 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:23.174 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:23.176 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1757849341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:23.176 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:23.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:23.177 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:23.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1757849341' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:23.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:23.278 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:23.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:23.416 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:12:23.416 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:12:23.416 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T15:12:23.416 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:12:23.417 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:12:23.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T15:12:23.461 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:23.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:23.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:23.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:23.592 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:23.709 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:23.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:23.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:23.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:23.872 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:23.961 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:23.970 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: pgmap v447: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:23.970 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: failed to encode map e179 with expected crc 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: failed to encode map e179 with expected crc 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: failed to encode map e179 with expected crc 2023-09-23T15:12:23.971 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:23 smithi027 ceph-mon[195233]: failed to encode map e179 with expected crc 2023-09-23T15:12:23.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:23.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:24.006 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:24.006 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:24.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: pgmap v447: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:24.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:24.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[222878]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: pgmap v447: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.4 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 83 MiB/s wr, 1.32k op/s; 20960/165675 objects degraded (12.651%); 5/165675 objects misplaced (0.003%) 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:24.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:24.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: osdmap e179: 8 total, 7 up, 7 in 2023-09-23T15:12:24.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:23 smithi017 ceph-mon[215646]: failed to encode map e179 with expected crc 2023-09-23T15:12:24.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:24.082 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 61081 61065 112.145 65 0.0100325 0.00891222 2023-09-23T15:12:24.144 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:24.240 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:24.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:24.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:24.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:24.365 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:24.411 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:24.509 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:24.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:24.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:24.534 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:24.574 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:24.609 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:24.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:24.631 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:24.653 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:24.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:24.730 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:24.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:24.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:24.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:24.772 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.773 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:12:24.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:24.887 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:24.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/621317508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:24.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/621317508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:25.018 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:25.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:12:25.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:25.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:25.086 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 62242 62226 111.015 72.5625 0.00656216 0.00900296 2023-09-23T15:12:25.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:25.202 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:25.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/621317508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:25.535 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:26.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:12:26.083 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 63446 63430 110.017 75.25 0.0109625 0.0090846 2023-09-23T15:12:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:25 smithi027 ceph-mon[195233]: pgmap v449: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 21713/171609 objects degraded (12.653%); 5/171609 objects misplaced (0.003%) 2023-09-23T15:12:26.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:25 smithi017 ceph-mon[222878]: pgmap v449: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 21713/171609 objects degraded (12.653%); 5/171609 objects misplaced (0.003%) 2023-09-23T15:12:26.111 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:25 smithi017 ceph-mon[215646]: pgmap v449: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 21713/171609 objects degraded (12.653%); 5/171609 objects misplaced (0.003%) 2023-09-23T15:12:26.224 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:26.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:26.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:26.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:12:26.333 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:12:26.466 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:26.575 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:26.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:26.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:26.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:26.745 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:26.908 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:26.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:26.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:26.961 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:26.984 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:12:26.985 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:12:26.985 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:12:27.015 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:27.018 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:26 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21713/171609 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:27.019 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:26 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21713/171609 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:27.080 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:27.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 64451 64435 108.741 62.8125 0.0170903 0.00918997 2023-09-23T15:12:27.233 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:27.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:27.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:27.244 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:26 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21713/171609 objects degraded (12.653%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:27.336 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:27.396 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:27.449 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:27.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:27.494 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:27.495 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:12:27.495 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:12:27.495 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:12:27.495 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:27.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:27.591 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:28.048 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:28.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:28.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:28.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 65337 65321 107.338 55.375 0.00256825 0.00927141 2023-09-23T15:12:28.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:28.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:28] "GET /metrics HTTP/1.1" 200 36300 "" "Prometheus/2.43.0" 2023-09-23T15:12:28.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:27 smithi017 ceph-mon[215646]: pgmap v450: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:28.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:27 smithi017 ceph-mon[222878]: pgmap v450: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:28.196 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:27 smithi027 ceph-mon[195233]: pgmap v450: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:28.201 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:28.333 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:12:28.404 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:28.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:28.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:28.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:12:28.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:28.580 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:28.696 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:28.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:28.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:28.804 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:28.847 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:28.931 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:28.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:28.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:29.007 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:29.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:29.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 66574 66558 106.569 77.3125 0.00921205 0.00937809 2023-09-23T15:12:29.119 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:29.227 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:29.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:29.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:29.237 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:29.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:29.382 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:29.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:29.442 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:29.442 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:29 smithi027 ceph-mon[195233]: pgmap v451: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:29.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[215646]: pgmap v451: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:29 smithi017 ceph-mon[222878]: pgmap v451: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 76 MiB/s wr, 1.21k op/s; 23236/183864 objects degraded (12.638%); 5/183864 objects misplaced (0.003%) 2023-09-23T15:12:29.663 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:29.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:29.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:29.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:29.815 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:29.886 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:29.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:29.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:29.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:30.017 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:30.061 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:30.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:30.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:30.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:12:30.085014+0000 min lat: 0.00114865 max lat: 0.325082 avg lat: 0.00943597 2023-09-23T15:12:30.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:12:30.085 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 67629 67613 105.553 65.9375 0.105084 0.00943597 2023-09-23T15:12:30.135 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:12:30.139 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:12:30.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:30.186 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:12:30.246 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:30.317 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:30.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:30.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:12:30.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:30.470 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:31.036 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:31.036 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:12:31.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:12:31.036 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106167 5.c 5:3b2646bb:::benchmark_data_smithi017_288048_object106166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.32345224400001, 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26369647800000001, 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106167 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.037 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710224+0000", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710278+0000", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4126000000000003e-05 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710288+0000", 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4380000000000001e-06 2023-09-23T15:12:31.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710293+0000", 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9969999999999998e-06 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712898+0000", 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026053460000000001 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713002+0000", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103749 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713127+0000", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012499599999999999 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719754+0000", 2023-09-23T15:12:31.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066268890000000004 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973752+0000", 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25399865999999999 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973807+0000", 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4224999999999997e-05 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973921+0000", 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114744 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106172 5.c 5:3a30f215:::benchmark_data_smithi017_288048_object106171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.32070737000001, 2023-09-23T15:12:31.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26115257600000003, 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106172 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712969+0000", 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713027+0000", 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7998999999999998e-05 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713038+0000", 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1790999999999999e-05 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713046+0000", 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.323e-06 2023-09-23T15:12:31.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716305+0000", 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032597160000000002 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716430+0000", 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124528 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716566+0000", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135735 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723883+0000", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073173500000000002 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973975+0000", 2023-09-23T15:12:31.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.250091695 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974026+0000", 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1768000000000003e-05 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974122+0000", 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5847999999999994e-05 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106173 5.9 e134/114 5:9fa66f55:::benchmark_data_smithi017_288048_object106172:head v 134'6688, mlcod=134'6685)", 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.718615+0000", 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.315061824, 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25512913799999998, 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718615+0000", 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718615+0000", 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718615+0000", 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718634+0000", 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8856999999999999e-05 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718641+0000", 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2880000000000004e-06 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718645+0000", 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8600000000000003e-06 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718685+0000", 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0538999999999999e-05 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718705+0000", 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0038999999999999e-05 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728189+0000", 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094838909999999995 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973744+0000", 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.245555262 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.054 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106171 5.a e134/114 5:591ee17c:::benchmark_data_smithi017_288048_object106170:head v 134'6617, mlcod=134'6615)", 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.314132538, 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.264097527, 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:12:31.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719544+0000", 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719581+0000", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6968999999999998e-05 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719591+0000", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9680000000000001e-06 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719596+0000", 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4990000000000002e-06 2023-09-23T15:12:31.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719639+0000", 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2762000000000002e-05 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719657+0000", 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8224999999999999e-05 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983596+0000", 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26393868399999998 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983642+0000", 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6028e-05 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106179 5.c 5:3b72c5c1:::benchmark_data_smithi017_288048_object106178:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.720399+0000", 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.313277935, 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25409145900000002, 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106179 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720399+0000", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720399+0000", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720398+0000", 2023-09-23T15:12:31.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720457+0000", 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8791e-05 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720467+0000", 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0448999999999999e-05 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720473+0000", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8440000000000003e-06 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720569+0000", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5654000000000003e-05 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720692+0000", 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012330800000000001 2023-09-23T15:12:31.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720836+0000", 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014451199999999999 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974359+0000", 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25352306499999999 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974368+0000", 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4130000000000006e-06 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974406+0000", 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8115999999999999e-05 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974491+0000", 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4652000000000002e-05 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.065 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106180 5.e 5:746d511a:::benchmark_data_smithi017_288048_object106179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.720484+0000", 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.313193391, 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25578967400000002, 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106180 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720484+0000", 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720484+0000", 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720483+0000", 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720517+0000", 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4088000000000003e-05 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720524+0000", 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9739999999999996e-06 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.720529+0000", 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4880000000000001e-06 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723999+0000", 2023-09-23T15:12:31.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034699449999999999 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.724115+0000", 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011618 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.724255+0000", 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014027899999999999 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973751+0000", 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24949587500000001 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973895+0000", 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000143955 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.976194+0000", 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022988040000000002 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.976237+0000", 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3265e-05 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.976273+0000", 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6300999999999997e-05 2023-09-23T15:12:31.072 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106177 5.a e134/114 5:561ec9dc:::benchmark_data_smithi017_288048_object106176:head v 134'6618, mlcod=134'6615)", 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.30976560100001, 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25975826499999999, 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723910+0000", 2023-09-23T15:12:31.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723946+0000", 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6072000000000003e-05 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723957+0000", 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0461e-05 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723963+0000", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9569999999999999e-06 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725450+0000", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001486769 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725471+0000", 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1826000000000001e-05 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983647+0000", 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25817548600000001 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983670+0000", 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2768e-05 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106178 5.a e134/114 5:56b4f3e4:::benchmark_data_smithi017_288048_object106177:head v 134'6619, mlcod=134'6615)", 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 210.30843599100001, 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25845162399999999, 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725276+0000", 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5089000000000002e-05 2023-09-23T15:12:31.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725287+0000", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1552999999999999e-05 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725293+0000", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8449999999999999e-06 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977314+0000", 2023-09-23T15:12:31.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.252021461 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977334+0000", 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9160999999999998e-05 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983673+0000", 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006339028 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983693+0000", 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0047e-05 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166941 5.1 5:8b30db7e:::benchmark_data_smithi017_288048_object55172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.94732489, 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40214831299999998, 2023-09-23T15:12:31.082 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166941 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086351+0000", 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086356+0000", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1039999999999998e-06 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086368+0000", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1618e-05 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086374+0000", 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5490000000000003e-06 2023-09-23T15:12:31.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086536+0000", 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161743 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086652+0000", 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011594 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086770+0000", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011798600000000001 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091315+0000", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045446310000000004 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488382+0000", 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39706780000000003 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488406+0000", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3218e-05 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488454+0000", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8301999999999999e-05 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488501+0000", 2023-09-23T15:12:31.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6573000000000003e-05 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166942 5.e 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.94729055600001, 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40238844899999998, 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.089 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166942 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086386+0000", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086389+0000", 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5830000000000001e-06 2023-09-23T15:12:31.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086395+0000", 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086399+0000", 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2300000000000002e-06 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086603+0000", 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020372899999999999 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086701+0000", 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8332000000000007e-05 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086804+0000", 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103294 2023-09-23T15:12:31.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093768+0000", 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006964034 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488259+0000", 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39449091200000003 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.094 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488702+0000", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00044246200000000002 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488734+0000", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2069000000000001e-05 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488775+0000", 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1278e-05 2023-09-23T15:12:31.095 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166947 5.e 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.946826894, 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40205635099999998, 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.096 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166947 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086849+0000", 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086854+0000", 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6290000000000001e-06 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086866+0000", 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2478e-05 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086873+0000", 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1299999999999998e-06 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090167+0000", 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032944319999999999 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090246+0000", 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9369999999999994e-05 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090328+0000", 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1706999999999995e-05 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488279+0000", 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397950631 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488321+0000", 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2725999999999997e-05 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488813+0000", 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000491991 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488869+0000", 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5350999999999999e-05 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488907+0000", 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7762999999999997e-05 2023-09-23T15:12:31.102 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166944 5.0 e147/132 5:0733dbd2:::benchmark_data_smithi017_288048_object55175:head v 147'10299, mlcod=147'10297)", 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.94659649600001, 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40144166599999997, 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087080+0000", 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087086+0000", 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5890000000000002e-06 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087096+0000", 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0434999999999999e-05 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087101+0000", 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4890000000000003e-06 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087141+0000", 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9147999999999999e-05 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087162+0000", 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0928e-05 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099400+0000", 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012238123 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488522+0000", 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38912261599999998 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.107 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166953 5.c 5:3faa0b3c:::benchmark_data_smithi017_288048_object55184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.942550122, 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39750094200000002, 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:31.108 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166953 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091126+0000", 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091130+0000", 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.014e-06 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091144+0000", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4717e-05 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091149+0000", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9590000000000003e-06 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091258+0000", 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108806 2023-09-23T15:12:31.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091386+0000", 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012736 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091496+0000", 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109987 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488306+0000", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39681003300000001 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488328+0000", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2175999999999999e-05 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488380+0000", 2023-09-23T15:12:31.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1913999999999998e-05 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488628+0000", 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024825900000000002 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166945 5.0 e147/132 5:0c8902eb:::benchmark_data_smithi017_288048_object55176:head v 147'10300, mlcod=147'10297)", 2023-09-23T15:12:31.114 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.94149369900001, 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39637069699999999, 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092188+0000", 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9760000000000003e-06 2023-09-23T15:12:31.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092197+0000", 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2019999999999993e-06 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092203+0000", 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8019999999999997e-06 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092271+0000", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8647999999999997e-05 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092292+0000", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0990999999999999e-05 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488527+0000", 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39623427999999999 2023-09-23T15:12:31.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488554+0000", 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7458e-05 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166939 5.f e147/114 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head v 147'10605, mlcod=147'10602)", 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:12:31.119 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.93732192300001, 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40201396099999998, 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:12:31.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096354+0000", 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096363+0000", 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2139999999999996e-06 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096374+0000", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1022000000000001e-05 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096379+0000", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488409+0000", 2023-09-23T15:12:31.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39202973600000002 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488494+0000", 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5271000000000004e-05 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498336+0000", 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098418459999999996 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498369+0000", 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2783000000000001e-05 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166950 5.f e147/114 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head v 147'10606, mlcod=147'10602)", 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.937274129, 2023-09-23T15:12:31.124 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40582205900000001, 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096406+0000", 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4649999999999999e-06 2023-09-23T15:12:31.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096412+0000", 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7450000000000001e-06 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096415+0000", 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5650000000000002e-06 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493677+0000", 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397262008 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493697+0000", 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9083e-05 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502182+0000", 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084849629999999999 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502225+0000", 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.35e-05 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166955 5.5 e147/132 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head v 147'10623, mlcod=147'10621)", 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.937236915, 2023-09-23T15:12:31.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.410760976, 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:12:31.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096443+0000", 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1700000000000001e-06 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096449+0000", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096453+0000", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4520000000000002e-06 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488252+0000", 2023-09-23T15:12:31.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39179915599999998 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488294+0000", 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2344e-05 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507165+0000", 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018870505999999999 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507201+0000", 2023-09-23T15:12:31.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6368999999999997e-05 2023-09-23T15:12:31.135 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.135 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.135 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166947 5.e e147/114 ondisk, result = 0)", 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.93721121199999, 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39185894700000001, 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096468+0000", 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.357e-06 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096473+0000", 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2220000000000001e-06 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096477+0000", 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.399e-06 2023-09-23T15:12:31.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488307+0000", 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39183000600000001 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488315+0000", 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488325+0000", 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0294e-05 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166953 5.c e147/132 ondisk, result = 0)", 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.93719315000001, 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39182831099999998, 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.140 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096485+0000", 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6139999999999999e-06 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096490+0000", 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8550000000000001e-06 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096493+0000", 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9909999999999998e-06 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488288+0000", 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.391795069 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488298+0000", 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4709999999999996e-06 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488312+0000", 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4516e-05 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166941 5.1 e147/114 ondisk, result = 0)", 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 132.93330047800001, 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38801138200000002, 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:12:31.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100376+0000", 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100382+0000", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0780000000000001e-06 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488291+0000", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38790906000000003 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488299+0000", 2023-09-23T15:12:31.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8660000000000006e-06 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488367+0000", 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8133000000000006e-05 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488376+0000", 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.935e-06 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:31.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488388+0000", 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2139e-05 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:31.149 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:31.150 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 68710 68694 104.627 67.5625 0.00927593 0.00955322 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: 1.a scrub starts 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: pgmap v453: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:31.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[215646]: 1.a scrub ok 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: 1.a scrub starts 2023-09-23T15:12:31.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.311 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.311 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: pgmap v453: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:31.311 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:31 smithi017 ceph-mon[222878]: 1.a scrub ok 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1187011048' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: osdmap e180: 8 total, 7 up, 7 in 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: 1.a scrub starts 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: failed to encode map e180 with expected crc 2023-09-23T15:12:31.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: pgmap v453: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:31.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:31 smithi027 ceph-mon[195233]: 1.a scrub ok 2023-09-23T15:12:31.374 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:31.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:31.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:31.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:31.595 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:31.717 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:31.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:31.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:31.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:31.875 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:32.002 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:32.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:32.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:32.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:12:32.040 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:32.075 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:32.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 69648 69632 103.533 58.625 0.00733615 0.00965341 2023-09-23T15:12:32.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:32.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:32.149 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:32.157 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25269/200298 objects degraded (12.616%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:32.157 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4209488289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:32.167 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:32.181 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:32.216 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:32.257 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:32.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.257 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:12:32.257 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.258 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.258 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.258 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.258 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.258 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:12:32.267 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:32.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:32.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:32.374 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:32.440 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25269/200298 objects degraded (12.616%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4209488289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25269/200298 objects degraded (12.616%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4209488289' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:32.535 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:32.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:32.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:32.647 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:32.687 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:32.763 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:32.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:32.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:32.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:32.929 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:33.028 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:33.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:33.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:33.084 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 70928 70912 102.985 80 0.0100584 0.0097058 2023-09-23T15:12:33.105 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [1] live_osds: [0, 2, 3, 4, 5, 6, 7] 2023-09-23T15:12:33.105 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T15:12:33.105 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T15:12:33.105 DEBUG:teuthology.orchestra.run.smithi017:osd.1> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T15:12:33.113 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl start ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T15:12:33.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:33.183 INFO:journalctl@ceph.osd.1.smithi017.stdout:-- Logs begin at Sat 2023-09-23 14:28:47 UTC. -- 2023-09-23T15:12:33.232 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:12:33.233 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:12:33.238 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:33.297 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:33 smithi027 ceph-mon[195233]: pgmap v454: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:33.398 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:33.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:33 smithi017 ceph-mon[215646]: pgmap v454: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:33.423 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:33 smithi017 systemd[1]: Starting Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:12:33.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:33 smithi017 ceph-mon[222878]: pgmap v454: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 77 MiB/s wr, 1.23k op/s; 25269/200298 objects degraded (12.616%); 5/200298 objects misplaced (0.002%) 2023-09-23T15:12:33.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:33.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:33.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:12:33.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:33.563 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:33.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:33.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:33.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:33.762 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:33.883 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:33.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:33.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:33.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:34.087 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 71914 71898 102.039 61.625 0.0151919 0.00979489 2023-09-23T15:12:34.094 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:34.203 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:34.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:34.276 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:34.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:34.337 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:34 smithi027 ceph-mon[195233]: 1.10 scrub starts 2023-09-23T15:12:34.337 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:34 smithi027 ceph-mon[195233]: 1.10 scrub ok 2023-09-23T15:12:34.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:34 smithi017 ceph-mon[222878]: 1.10 scrub starts 2023-09-23T15:12:34.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:34 smithi017 ceph-mon[222878]: 1.10 scrub ok 2023-09-23T15:12:34.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:34 smithi017 ceph-mon[215646]: 1.10 scrub starts 2023-09-23T15:12:34.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:34 smithi017 ceph-mon[215646]: 1.10 scrub ok 2023-09-23T15:12:34.391 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:34.554 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:34.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:34.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:34.623 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:34.624 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:34.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:34.730 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:34.890 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:34.895 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:34.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T15:12:35.056 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:34 smithi017 bash[359296]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T15:12:35.072 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:35.087 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 73000 72984 101.281 67.875 0.00746095 0.00986876 2023-09-23T15:12:35.234 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:12:35.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:35 smithi017 ceph-mon[215646]: pgmap v455: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 25954/205383 objects degraded (12.637%); 5/205383 objects misplaced (0.002%) 2023-09-23T15:12:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:35 smithi017 ceph-mon[222878]: pgmap v455: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 25954/205383 objects degraded (12.637%); 5/205383 objects misplaced (0.002%) 2023-09-23T15:12:35.555 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:35 smithi027 ceph-mon[195233]: pgmap v455: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 25954/205383 objects degraded (12.637%); 5/205383 objects misplaced (0.002%) 2023-09-23T15:12:35.576 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:12:36.088 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 73973 73957 100.401 60.8125 0.0187277 0.00995538 2023-09-23T15:12:36.147 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:36.436 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:36.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:36.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:36.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:12:36.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:12:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:36.558 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:12:36.558 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.558 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:36.650 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:36.932 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:36.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:36.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:37.037 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:37.066 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:37.091 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 74763 74747 99.3107 49.375 0.018259 0.0100606 2023-09-23T15:12:37.112 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:37.187 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:37.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:37.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:37.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:37.332 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:37.584 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: 1.9 scrub starts 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: 1.9 scrub ok 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: pgmap v456: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25954/205383 objects degraded (12.637%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:37.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:37.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:37.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:37.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: 1.9 scrub starts 2023-09-23T15:12:37.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: 1.9 scrub ok 2023-09-23T15:12:37.715 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: pgmap v456: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25954/205383 objects degraded (12.637%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: 1.9 scrub starts 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: 1.9 scrub ok 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: pgmap v456: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:37.716 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25954/205383 objects degraded (12.637%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:37.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:12:37.750 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:37.878 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:37.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:37.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:37.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:37.998 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:37 smithi017 bash[360838]: 5922ab9bf4a26db51d2e36e3fc0ff3c815bef536210e27fca85e75cce4904bf3 2023-09-23T15:12:37.998 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:37 smithi017 systemd[1]: Started Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:12:37.998 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:37 smithi017 conmon[360957]: 2023-09-23T15:12:37.993+0000 7f1b183ac640 -1 Falling back to public interface 2023-09-23T15:12:38.022 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:38.090 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 75619 75603 98.357 53.5 0.0225783 0.0101615 2023-09-23T15:12:38.191 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:38.191 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:38.191 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:38.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:38.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:38.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:38.329 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:38] "GET /metrics HTTP/1.1" 200 36303 "" "Prometheus/2.43.0" 2023-09-23T15:12:38.363 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:38.507 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:38.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:38.546 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:12:38.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:38.657 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:38.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[215646]: 1.12 scrub starts 2023-09-23T15:12:38.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[215646]: 1.12 scrub ok 2023-09-23T15:12:38.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[222878]: 1.12 scrub starts 2023-09-23T15:12:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[222878]: 1.12 scrub ok 2023-09-23T15:12:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:38.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:38 smithi027 ceph-mon[195233]: 1.12 scrub starts 2023-09-23T15:12:38.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:38 smithi027 ceph-mon[195233]: 1.12 scrub ok 2023-09-23T15:12:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:39.091 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 76493 76477 97.465 54.625 0.00357525 0.0102315 2023-09-23T15:12:39.399 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:39.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:39.403 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:39.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:39.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:12:39.585 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:39.817 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:39.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:39.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: pgmap v457: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:39.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.824 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[222878]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.825 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: pgmap v457: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.825 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.826 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:39 smithi017 ceph-mon[215646]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: pgmap v457: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 27319/216411 objects degraded (12.624%); 5/216411 objects misplaced (0.002%) 2023-09-23T15:12:39.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2402332122' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: osdmap e181: 8 total, 7 up, 7 in 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:39 smithi027 ceph-mon[195233]: failed to encode map e181 with expected crc 2023-09-23T15:12:39.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:39.997 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:40.092 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 77350 77334 96.5843 53.5625 0.0224423 0.0103476 2023-09-23T15:12:40.131 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:40.133 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:40.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:40.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:40.221 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:40.241 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:40.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:40.401 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:40.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:40.407 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:40.443 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:40 smithi017 conmon[360957]: 2023-09-23T15:12:40.372+0000 7f1b183ac640 -1 osd.1 131 log_to_monitors true 2023-09-23T15:12:40.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:40.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:40 smithi027 ceph-mon[195233]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.556 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:40.630 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:40.716 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:40.759 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:40.815 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:40.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:40 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.858 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:40.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:40 smithi017 ceph-mon[222878]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:40 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:40 smithi017 ceph-mon[215646]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:40 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T15:12:40.907 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.942 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.943 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.943 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:12:40.943 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:12:41.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 78133 78117 95.6507 48.9375 0.00712198 0.0104464 2023-09-23T15:12:41.119 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:41.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:41.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:41.192 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:12:40 smithi017 conmon[360957]: 2023-09-23T15:12:40.916+0000 7f1b09819700 -1 osd.1 182 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:12:41.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:41.290 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: pgmap v459: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 980 op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: 1.8 deep-scrub starts 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: 1.8 deep-scrub ok 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2335853820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:41.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: pgmap v459: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 980 op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: 1.8 deep-scrub starts 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: 1.8 deep-scrub ok 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2335853820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: pgmap v459: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 980 op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: 1.8 deep-scrub starts 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: 1.8 deep-scrub ok 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: osdmap e182: 8 total, 7 up, 7 in 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: failed to encode map e182 with expected crc 2023-09-23T15:12:41.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: from='osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi017", "root=default"]}]: dispatch 2023-09-23T15:12:41.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2335853820' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:42.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 78647 78631 94.43 32.125 0.00303075 0.0104841 2023-09-23T15:12:42.229 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166939 5.f 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.14821423500001, 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.42086912300000001, 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166939 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079752+0000", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079756+0000", 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7440000000000001e-06 2023-09-23T15:12:42.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079765+0000", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9819999999999997e-06 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079770+0000", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6439999999999999e-06 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086956+0000", 2023-09-23T15:12:42.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071860470000000001 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087060+0000", 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104073 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087174+0000", 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114411 2023-09-23T15:12:42.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371249+0000", 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28407477599999997 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371500+0000", 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025060700000000002 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500542+0000", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.129042504 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500587+0000", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4487000000000003e-05 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500622+0000", 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5672e-05 2023-09-23T15:12:42.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166948 5.b e147/114 5:dfd8a27b:::benchmark_data_smithi017_288048_object55179:head v 147'10455, mlcod=147'10453)", 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13730257700001, 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28076174100000001, 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090664+0000", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090672+0000", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3639999999999999e-06 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090684+0000", 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1531e-05 2023-09-23T15:12:42.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090690+0000", 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.229e-06 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090801+0000", 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011069900000000001 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090821+0000", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0027e-05 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371402+0000", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28058097599999998 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371427+0000", 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4839999999999999e-05 2023-09-23T15:12:42.241 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166950 5.f 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.137253845, 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.41173430799999999, 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:42.242 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166950 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090712+0000", 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090716+0000", 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.296e-06 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090725+0000", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3890000000000005e-06 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090730+0000", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.792e-06 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091549+0000", 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00081936800000000001 2023-09-23T15:12:42.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091663+0000", 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011382700000000001 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091780+0000", 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117632 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371458+0000", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27967720699999998 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371548+0000", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0597e-05 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502366+0000", 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13081746899999999 2023-09-23T15:12:42.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502413+0000", 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6909000000000001e-05 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502448+0000", 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5142000000000003e-05 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.248 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166942 5.e e147/114 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head v 147'10334, mlcod=147'10332)", 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.135773742, 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27917522900000002, 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:12:42.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092193+0000", 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092201+0000", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8779999999999998e-06 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092211+0000", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0341e-05 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092216+0000", 2023-09-23T15:12:42.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3469999999999998e-06 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092399+0000", 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018310200000000001 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092418+0000", 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8604e-05 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371292+0000", 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27887434100000003 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371369+0000", 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6413000000000002e-05 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166947 5.e e147/114 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head v 147'10335, mlcod=147'10332)", 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13571755699999, 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.279469315, 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092253+0000", 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2320000000000001e-06 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092258+0000", 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9749999999999998e-06 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092261+0000", 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1719999999999999e-06 2023-09-23T15:12:42.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096022+0000", 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037614689999999999 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096042+0000", 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9712999999999999e-05 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371373+0000", 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27533052099999999 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371719+0000", 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000346409 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166949 5.2 e147/114 5:4bbd8647:::benchmark_data_smithi017_288048_object55180:head v 147'10607, mlcod=147'10605)", 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13564953299999, 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27920739500000002, 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092317+0000", 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092323+0000", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1909999999999999e-06 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092331+0000", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0409999999999998e-06 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092335+0000", 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2050000000000004e-06 2023-09-23T15:12:42.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096032+0000", 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036968489999999999 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096049+0000", 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7271000000000001e-05 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371369+0000", 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27532035900000001 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371525+0000", 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015582500000000001 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166954 5.3 e147/114 5:c22244f8:::benchmark_data_smithi017_288048_object55185:head v 147'10165, mlcod=147'10163)", 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13409342599999, 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27758365299999999, 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093873+0000", 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093881+0000", 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3590000000000008e-06 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093894+0000", 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2184000000000001e-05 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093912+0000", 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8244999999999999e-05 2023-09-23T15:12:42.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.094053+0000", 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141366 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.094074+0000", 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0511999999999999e-05 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371431+0000", 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27735731899999999 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371458+0000", 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6616000000000001e-05 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166946 5.4 e147/114 5:2ea27988:::benchmark_data_smithi017_288048_object55177:head v 147'10343, mlcod=147'10340)", 2023-09-23T15:12:42.268 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.132794061, 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27613599900000002, 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095172+0000", 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095181+0000", 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3909999999999998e-06 2023-09-23T15:12:42.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095192+0000", 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1131999999999999e-05 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095198+0000", 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0769999999999996e-06 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095238+0000", 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0147e-05 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095257+0000", 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8853000000000001e-05 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371269+0000", 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27601161800000001 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371309+0000", 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0626e-05 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166951 5.4 e147/114 5:254d5c06:::benchmark_data_smithi017_288048_object55182:head v 147'10344, mlcod=147'10341)", 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13275304300001, 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27622131900000002, 2023-09-23T15:12:42.273 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095218+0000", 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8010000000000001e-06 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095223+0000", 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5629999999999999e-06 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095227+0000", 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5149999999999998e-06 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098400+0000", 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031727639999999998 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098417+0000", 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7609999999999999e-05 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371408+0000", 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27299110700000001 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371436+0000", 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7199000000000002e-05 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166952 5.b e147/114 5:da70994f:::benchmark_data_smithi017_288048_object55183:head v 147'10456, mlcod=147'10453)", 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13272614300001, 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.276238488, 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.278 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095245+0000", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7129999999999999e-06 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095250+0000", 2023-09-23T15:12:42.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8470000000000001e-06 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095254+0000", 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3139999999999999e-06 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095675+0000", 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042140999999999999 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095691+0000", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5928000000000001e-05 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371460+0000", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27576851099999999 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371480+0000", 2023-09-23T15:12:42.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0111000000000001e-05 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166956 5.d e147/114 5:bebbfc4b:::benchmark_data_smithi017_288048_object55187:head v 147'10394, mlcod=147'10392)", 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.097303+0000", 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.13066451200001, 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27411231800000002, 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097303+0000", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097303+0000", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097302+0000", 2023-09-23T15:12:42.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097309+0000", 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6989999999999997e-06 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097320+0000", 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0868e-05 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097325+0000", 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2009999999999998e-06 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097367+0000", 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.244e-05 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.097386+0000", 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8430000000000001e-05 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371379+0000", 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27399296299999998 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371415+0000", 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6511000000000003e-05 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.287 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166939 5.f e147/114 ondisk, result = 0)", 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.104449+0000", 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 144.12351824800001, 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26705311100000001, 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.104449+0000", 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.104449+0000", 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.104448+0000", 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.104456+0000", 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6149999999999999e-06 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371216+0000", 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26676007000000002 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371228+0000", 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2668e-05 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371484+0000", 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025535599999999999 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371493+0000", 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371502+0000", 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4199999999999996e-06 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.292 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200838 5.b e156/114 5:d7ba4b77:::benchmark_data_smithi017_288048_object89069:head v 156'12612, mlcod=156'12606)", 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 98.094056774999999, 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31912336400000002, 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133910+0000", 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133917+0000", 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7519999999999999e-06 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133928+0000", 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.151e-05 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133934+0000", 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6740000000000002e-06 2023-09-23T15:12:42.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137529+0000", 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035948030000000001 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137544+0000", 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5594999999999999e-05 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.452976+0000", 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31543138300000001 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453034+0000", 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8356000000000002e-05 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200843 5.3 e156/114 5:c8957fbc:::benchmark_data_smithi017_288048_object89074:head v 156'12232, mlcod=156'12230)", 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:12:42.297 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 98.093970658000003, 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31897129600000002, 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133996+0000", 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134004+0000", 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5989999999999996e-06 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134015+0000", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1121000000000001e-05 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134020+0000", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4670000000000003e-06 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137524+0000", 2023-09-23T15:12:42.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035037229999999998 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137543+0000", 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8983000000000001e-05 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.452968+0000", 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31542512 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.301 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200840 5.b e156/114 5:d0cff80c:::benchmark_data_smithi017_288048_object89071:head v 156'12613, mlcod=156'12606)", 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 98.088896228999999, 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31398826699999999, 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139078+0000", 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3420000000000002e-06 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139086+0000", 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3799999999999994e-06 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139090+0000", 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1230000000000002e-06 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296790+0000", 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15769914900000001 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.298031+0000", 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001241942 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453041+0000", 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15500998099999999 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453059+0000", 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7944999999999998e-05 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.306 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200853 5.7 5:ef3ce6d5:::benchmark_data_smithi017_288048_object89084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.931423541000001, 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.36030263699999998, 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:42.307 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200853 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296542+0000", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296548+0000", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4489999999999998e-06 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296560+0000", 2023-09-23T15:12:42.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2655e-05 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296567+0000", 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1650000000000003e-06 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.297205+0000", 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063859400000000003 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.297306+0000", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100739 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.298062+0000", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00075588599999999999 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632346+0000", 2023-09-23T15:12:42.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33428428199999999 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.656846+0000", 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024500299 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200858 5.7 5:e5e3a1c9:::benchmark_data_smithi017_288048_object89089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:12:42.312 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.931391949000002, 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.36027551899999999, 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200858 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:12:42.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296578+0000", 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.965e-06 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296584+0000", 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2720000000000003e-06 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296589+0000", 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3800000000000004e-06 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304686+0000", 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080975849999999992 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304839+0000", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015270600000000001 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304987+0000", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014832699999999999 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632380+0000", 2023-09-23T15:12:42.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32739249399999998 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.656851+0000", 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024471211999999999 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200860 5.6 e156/114 5:63b36d05:::benchmark_data_smithi017_288048_object89091:head v 156'12511, mlcod=156'12508)", 2023-09-23T15:12:42.319 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.302057+0000", 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.925910789, 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26411258199999998, 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302057+0000", 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302057+0000", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302056+0000", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302063+0000", 2023-09-23T15:12:42.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.063e-06 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302077+0000", 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4007e-05 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302083+0000", 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.19e-06 2023-09-23T15:12:42.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302352+0000", 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026861799999999998 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.302372+0000", 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0347999999999999e-05 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.566169+0000", 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26379716399999997 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:200853 5.7 e156/132)", 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.919369634000006, 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32381470600000001, 2023-09-23T15:12:42.324 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308597+0000", 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308604+0000", 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1910000000000003e-06 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308617+0000", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3037000000000001e-05 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308624+0000", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9380000000000003e-06 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632400+0000", 2023-09-23T15:12:42.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32377659600000003 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632412+0000", 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1933000000000001e-05 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200861 5.e e157/114 5:7029cdc5:::benchmark_data_smithi017_288048_object89092:head v 157'12376, mlcod=156'12374)", 2023-09-23T15:12:42.328 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 97.899103507000007, 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28701705900000002, 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:42.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328863+0000", 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328871+0000", 2023-09-23T15:12:42.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1450000000000002e-06 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328883+0000", 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2225000000000001e-05 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328890+0000", 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.934e-06 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.565469+0000", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236579123 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.565513+0000", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3636e-05 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.615832+0000", 2023-09-23T15:12:42.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.050319323999999999 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.615881+0000", 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9051999999999997e-05 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:42.333 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:42.530 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:42.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:42.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:12:42.627 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:12:42.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:42.663 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:42.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: 1.0 deep-scrub starts 2023-09-23T15:12:42.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: 1.0 deep-scrub ok 2023-09-23T15:12:42.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:12:42.787 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320] boot 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29014/229914 objects degraded (12.620%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: 1.0 deep-scrub starts 2023-09-23T15:12:42.788 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: 1.0 deep-scrub ok 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320] boot 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29014/229914 objects degraded (12.620%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T15:12:42.789 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:42 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: 1.0 deep-scrub starts 2023-09-23T15:12:42.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: 1.0 deep-scrub ok 2023-09-23T15:12:42.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:12:42.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: osd.1 [v2:172.21.15.17:6810/689323320,v1:172.21.15.17:6811/689323320] boot 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: osdmap e183: 8 total, 8 up, 7 in 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: failed to encode map e183 with expected crc 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29014/229914 objects degraded (12.620%), 9 pgs degraded, 24 pgs undersized (PG_DEGRADED) 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: osdmap e184: 8 total, 8 up, 7 in 2023-09-23T15:12:42.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:42 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:42.900 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:43.014 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:12:43.014 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:12:43.015 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:12:43.015 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:12:43.048 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:43.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:43.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 78647 78631 92.6496 0 - 0.0104841 2023-09-23T15:12:43.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:12:43.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:43.202 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:43.746 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:43.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:43.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:43.758 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: pgmap v463: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:43.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:12:43.896 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: pgmap v463: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e184 with expected crc 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: pgmap v463: 109 pgs: 37 active+clean+remapped, 9 active+undersized+degraded, 21 active+undersized, 42 active+clean; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 70 MiB/s wr, 1.12k op/s; 29014/229914 objects degraded (12.620%); 4/229914 objects misplaced (0.002%) 2023-09-23T15:12:43.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: osdmap e185: 8 total, 8 up, 7 in 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:43.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:44.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 79142 79126 91.5075 15.4688 0.0109041 0.0109211 2023-09-23T15:12:44.102 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:12:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:44.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:12:44.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:44.298 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:44.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:12:44.556 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.572 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: failed to encode map e186 with expected crc 2023-09-23T15:12:44.572 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T15:12:44.572 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.572 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:44.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:44.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:12:44.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:44.758 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: failed to encode map e186 with expected crc 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.759 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: failed to encode map e185 with expected crc 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: failed to encode map e186 with expected crc 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: osdmap e186: 8 total, 8 up, 7 in 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.760 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:44.827 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:45.093 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 79656 79640 90.4285 32.125 0.0365656 0.0110446 2023-09-23T15:12:45.360 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:45.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:45.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:45.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:45.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:45 smithi027 ceph-mon[195233]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:45 smithi027 ceph-mon[195233]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:45 smithi027 ceph-mon[195233]: pgmap v466: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 36 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 46 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 222 op/s; 29262/232422 objects degraded (12.590%); 4/232422 objects misplaced (0.002%) 2023-09-23T15:12:45.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:45 smithi027 ceph-mon[195233]: failed to encode map e187 with expected crc 2023-09-23T15:12:45.593 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:45 smithi027 ceph-mon[195233]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T15:12:45.596 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[215646]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[215646]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[215646]: pgmap v466: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 36 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 46 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 222 op/s; 29262/232422 objects degraded (12.590%); 4/232422 objects misplaced (0.002%) 2023-09-23T15:12:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[215646]: failed to encode map e187 with expected crc 2023-09-23T15:12:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[215646]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T15:12:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[222878]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[222878]: failed to encode map e186 with expected crc 2023-09-23T15:12:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[222878]: pgmap v466: 109 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 36 active+clean+remapped, 7 active+undersized+degraded, 18 active+undersized, 46 active+clean; 4.7 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 14 MiB/s wr, 222 op/s; 29262/232422 objects degraded (12.590%); 4/232422 objects misplaced (0.002%) 2023-09-23T15:12:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[222878]: failed to encode map e187 with expected crc 2023-09-23T15:12:45.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:45 smithi017 ceph-mon[222878]: osdmap e187: 8 total, 8 up, 7 in 2023-09-23T15:12:45.706 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:45.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:45.715 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:45.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:80124 6.3 6:ce9619af:::benchmark_data_smithi017_350392_object80123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:45.836526+0000", 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012014253000000001, 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012043111, 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 80124 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836526+0000", 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836526+0000", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836524+0000", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836557+0000", 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2571999999999999e-05 2023-09-23T15:12:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836574+0000", 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7027000000000001e-05 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836580+0000", 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1029999999999998e-06 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836627+0000", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6610000000000003e-05 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836777+0000", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015023799999999999 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.836979+0000", 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020226200000000001 2023-09-23T15:12:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.842153+0000", 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051736150000000003 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.845579+0000", 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034264270000000001 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:80129 6.5 6:acc88aa7:::benchmark_data_smithi017_350392_object80128:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e187)", 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:45.847586+0000", 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00095425900000000005, 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010690249999999999, 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 80129 2023-09-23T15:12:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847586+0000", 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847586+0000", 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847584+0000", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847663+0000", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8455999999999996e-05 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847682+0000", 2023-09-23T15:12:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8851e-05 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847689+0000", 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1380000000000001e-06 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847746+0000", 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6832000000000001e-05 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.847924+0000", 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017874000000000001 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:45.848101+0000", 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176742 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:12:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 2 2023-09-23T15:12:45.865 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:45.871 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:45.930 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:46.087 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:46.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:46.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:46.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 80292 80276 89.5205 39.75 0.0326607 0.0111636 2023-09-23T15:12:46.160 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:46.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:46.225 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:46.284 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:46.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:46.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:46.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:46.413 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:46.457 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:46.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:46.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:46.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:46.592 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:46.654 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:46 smithi017 ceph-mon[215646]: failed to encode map e187 with expected crc 2023-09-23T15:12:46.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:46 smithi017 ceph-mon[222878]: failed to encode map e187 with expected crc 2023-09-23T15:12:46.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:46 smithi027 ceph-mon[195233]: failed to encode map e187 with expected crc 2023-09-23T15:12:46.896 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:46.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:46.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:46.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:12:47.006 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:47.053 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:47.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 80819 80803 88.5285 32.9375 0.0201015 0.0112905 2023-09-23T15:12:47.134 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:47.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:47.250 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:47.319 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:47.469 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:47.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[222878]: pgmap v468: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 576 op/s; 29658/237687 objects degraded (12.478%); 653 KiB/s, 11 objects/s recovering 2023-09-23T15:12:47.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:47.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29658/237687 objects degraded (12.478%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:47.673 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:12:47.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[215646]: pgmap v468: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 576 op/s; 29658/237687 objects degraded (12.478%); 653 KiB/s, 11 objects/s recovering 2023-09-23T15:12:47.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:47.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29658/237687 objects degraded (12.478%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:48.037 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:47 smithi027 ceph-mon[195233]: pgmap v468: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 36 MiB/s wr, 576 op/s; 29658/237687 objects degraded (12.478%); 653 KiB/s, 11 objects/s recovering 2023-09-23T15:12:48.037 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:12:48.037 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29658/237687 objects degraded (12.478%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:48.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 81455 81439 87.688 39.75 0.0314233 0.0113984 2023-09-23T15:12:48.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:48] "GET /metrics HTTP/1.1" 200 36163 "" "Prometheus/2.43.0" 2023-09-23T15:12:48.664 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:48.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:48.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:48.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:48.800 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:48.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: failed to encode map e188 with expected crc 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:48 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:49.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:49.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: failed to encode map e188 with expected crc 2023-09-23T15:12:49.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: failed to encode map e188 with expected crc 2023-09-23T15:12:49.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3834414739' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:12:49.014 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: osdmap e188: 8 total, 8 up, 7 in 2023-09-23T15:12:49.015 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: failed to encode map e188 with expected crc 2023-09-23T15:12:49.015 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:49.015 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:48 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:49.023 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:49.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:49.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:49.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 81950 81934 86.7268 30.9375 0.0298604 0.0115246 2023-09-23T15:12:49.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:49.173 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:49.267 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:49.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:49.371 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:49.487 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:49.633 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:12:49.633 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:12:49.633 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:12:49.633 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:12:49.633 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:12:49.812 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:49.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:49.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[215646]: pgmap v470: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s; 29658/237687 objects degraded (12.478%); 533 KiB/s, 9 objects/s recovering 2023-09-23T15:12:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:12:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:12:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[222878]: pgmap v470: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s; 29658/237687 objects degraded (12.478%); 533 KiB/s, 9 objects/s recovering 2023-09-23T15:12:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:12:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:12:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:49.965 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:50.038 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:50.056 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:12:50.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:12:50.096368+0000 min lat: 0.00114865 max lat: 1.78741 avg lat: 0.0116291 2023-09-23T15:12:50.095 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:12:50.096 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 82583 82567 85.9412 39.5625 0.0155269 0.0116291 2023-09-23T15:12:50.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:49 smithi027 ceph-mon[195233]: pgmap v470: 109 pgs: 81 active+clean, 20 peering, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s; 29658/237687 objects degraded (12.478%); 533 KiB/s, 9 objects/s recovering 2023-09-23T15:12:50.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:12:50.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:12:50.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:50.122 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:12:50.180 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:12:50.211 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:50.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:50.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:50.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:12:50.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:50.342 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:12:50.390 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:50.399 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:12:50.453 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:12:50.504 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:50.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:50.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:50.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:50.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3076557621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:50.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1919711689' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:50.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:50.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:12:50.730 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:12:50.732 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:50.861 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:50.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:50.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3076557621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1919711689' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:50 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:50.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:51.059 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:51.099 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 83201 83185 85.1621 38.625 0.0391471 0.0117352 2023-09-23T15:12:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3076557621' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:12:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1919711689' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:50 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:12:51.207 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:51.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:12:51.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:51 smithi027 ceph-mon[195233]: pgmap v471: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29484/246090 objects degraded (11.981%); 2.1 MiB/s, 35 objects/s recovering 2023-09-23T15:12:51.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:51 smithi027 ceph-mon[195233]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:51 smithi027 ceph-mon[195233]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:12:51.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:51 smithi027 ceph-mon[195233]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[215646]: pgmap v471: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29484/246090 objects degraded (11.981%); 2.1 MiB/s, 35 objects/s recovering 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[215646]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[215646]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[215646]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[222878]: pgmap v471: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29484/246090 objects degraded (11.981%); 2.1 MiB/s, 35 objects/s recovering 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[222878]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[222878]: failed to encode map e189 with expected crc 2023-09-23T15:12:51.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:12:51.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:51 smithi017 ceph-mon[222878]: osdmap e189: 8 total, 8 up, 7 in 2023-09-23T15:12:51.755 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:51.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:51.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:51.894 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:52.098 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 83761 83745 84.3536 35 0.0214051 0.0118498 2023-09-23T15:12:52.256 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:52.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:52.282 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:52.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:12:52.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:52.470 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:52.612 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:52.612 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:52.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:52.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:52.722 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:52.801 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:52.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29484/246090 objects degraded (11.981%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29484/246090 objects degraded (11.981%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:52.964 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:52.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:52.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:53.025 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29484/246090 objects degraded (11.981%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:53.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:53.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 84389 84373 83.6357 39.25 0.0193736 0.011951 2023-09-23T15:12:53.143 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:53.309 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:53.309 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:53.414 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:53.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:53.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:53.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:53.596 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:53.656 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:53.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:53.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:53.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:53.786 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:53.854 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:53.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:53.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:53.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:54.053 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:54.100 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 84936 84920 82.8635 34.1875 0.0193792 0.0120619 2023-09-23T15:12:54.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:12:54.174 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:53 smithi017 ceph-mon[215646]: pgmap v473: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 29484/246090 objects degraded (11.981%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:12:54.179 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:53 smithi017 ceph-mon[222878]: pgmap v473: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 29484/246090 objects degraded (11.981%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:12:54.218 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:54.229 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:54.232 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:53 smithi027 ceph-mon[195233]: pgmap v473: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.0 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 24 MiB/s wr, 378 op/s; 29484/246090 objects degraded (11.981%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:12:54.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:54.246 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:54.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:12:54.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:54.441 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:54.736 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:54.755 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:54.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:54.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:54.971 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:55.108 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 85552 85536 82.1724 38.5 0.017969 0.0121644 2023-09-23T15:12:55.180 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:55.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:55.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:55.287 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:55.330 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:55.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:55.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:55.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:55.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:55.549 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:55.612 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:12:55.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:55.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:12:55.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:55.743 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:55.926 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:12:56.045 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:56.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:56.058 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:55 smithi017 ceph-mon[215646]: pgmap v474: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 29484/248913 objects degraded (11.845%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:12:56.058 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:56.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:56.059 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:55 smithi017 ceph-mon[222878]: pgmap v474: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 29484/248913 objects degraded (11.845%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:12:56.059 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:56.109 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 86114 86098 81.4579 35.125 0.0191839 0.0122703 2023-09-23T15:12:56.153 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:55 smithi027 ceph-mon[195233]: pgmap v474: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 29 MiB/s wr, 467 op/s; 29484/248913 objects degraded (11.845%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:12:56.154 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:12:56.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:12:56.254 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:56.715 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:12:56.715 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:12:56.717 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:12:56.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:56.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:12:56.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:56.830 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:56.869 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:56.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:56.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:56.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:56.972 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: failed to encode map e190 with expected crc 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: failed to encode map e190 with expected crc 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29484/248913 objects degraded (11.845%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: failed to encode map e190 with expected crc 2023-09-23T15:12:56.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: failed to encode map e190 with expected crc 2023-09-23T15:12:56.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:56.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T15:12:56.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:56 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29484/248913 objects degraded (11.845%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:57.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:12:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:12:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: failed to encode map e190 with expected crc 2023-09-23T15:12:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: failed to encode map e190 with expected crc 2023-09-23T15:12:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/714551410' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:12:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: osdmap e190: 8 total, 8 up, 7 in 2023-09-23T15:12:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:56 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29484/248913 objects degraded (11.845%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:12:57.111 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 86797 86781 80.8773 42.6875 0.0226819 0.012357 2023-09-23T15:12:57.139 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:57.327 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:12:57.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:57.330 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:12:57.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:57.549 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:57.750 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:12:57.759 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:57.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:57.864 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:57.909 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:57.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[215646]: pgmap v476: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 722 op/s; 29296/255051 objects degraded (11.486%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:12:57.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2090226871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:57.954 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:57.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[222878]: pgmap v476: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 722 op/s; 29296/255051 objects degraded (11.486%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:12:57.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2090226871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:57.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:57.963 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:12:58.025 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:58.032 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:57 smithi027 ceph-mon[195233]: pgmap v476: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 722 op/s; 29296/255051 objects degraded (11.486%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:12:58.032 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2090226871' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:58.032 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:12:58.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:58.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:58.112 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 87421 87405 80.2619 39 0.0198072 0.0124543 2023-09-23T15:12:58.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:12:58.161 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:58.260 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:12:58.280 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:12:58.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:58.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:12:58.292 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:12:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:12:58] "GET /metrics HTTP/1.1" 200 36163 "" "Prometheus/2.43.0" 2023-09-23T15:12:58.393 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:12:58.442 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:58.522 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:12:58.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:58.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:12:58.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:58.696 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:58.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: failed to encode map e191 with expected crc 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: failed to encode map e191 with expected crc 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: failed to encode map e191 with expected crc 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: failed to encode map e191 with expected crc 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:58.945 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:12:58 smithi027 ceph-mon[195233]: pgmap v478: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 392 op/s; 29296/255051 objects degraded (11.486%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:12:59.116 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 88018 88002 79.6353 37.3125 0.0493338 0.0125498 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:59.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[215646]: pgmap v478: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 392 op/s; 29296/255051 objects degraded (11.486%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: osdmap e191: 8 total, 8 up, 7 in 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: failed to encode map e191 with expected crc 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:12:59.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:12:58 smithi017 ceph-mon[222878]: pgmap v478: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.2 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 25 MiB/s wr, 392 op/s; 29296/255051 objects degraded (11.486%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:12:59.214 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:59.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:59.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:59.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:12:59.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:12:59.360 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:59.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:12:59.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:59.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:12:59.513 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:12:59.618 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:12:59.722 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:12:59.807 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:12:59.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:12:59.814 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:12:59.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:12:59.923 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:12:59.956 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:12:59.970 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:00.010 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:00.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:00.071 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:00.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:00.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:00.112 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:00.116 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 88555 88539 78.9776 33.5625 0.0230484 0.0126561 2023-09-23T15:13:00.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:00.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:00.194 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:00.228 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:00.230 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:00.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.238 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.239 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:13:00.348 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:00.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:00.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:00.433 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:13:00.433 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:13:00.433 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:13:00.433 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:13:00.433 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:13:00.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2600743963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:00.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2600743963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:00.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:00.456 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2600743963' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:00.496 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:00.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:00.625 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:00.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:00.770 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:00.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:00.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:00.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:00.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:00.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:01.021 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:01.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:01.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:01.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 89204 89188 78.4359 40.5625 0.0400676 0.0127427 2023-09-23T15:13:01.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:01.213 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:01 smithi027 ceph-mon[195233]: pgmap v479: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:01 smithi017 ceph-mon[215646]: pgmap v479: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:01.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:01 smithi017 ceph-mon[222878]: pgmap v479: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 753 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:01.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:13:01.764 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:01.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:01.821 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:01.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:01.925 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:01.972 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:13:02.010 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:02.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:02.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:02.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 89805 89789 77.8687 37.5625 0.0139233 0.0128372 2023-09-23T15:13:02.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:02.152 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29115/264192 objects degraded (11.020%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:02.204 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:02.230 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29115/264192 objects degraded (11.020%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:02.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29115/264192 objects degraded (11.020%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:02.483 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:02.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:02.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:02.637 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:02.707 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:02.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:02.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:02.850 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:02.910 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:02.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:02.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:03.017 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:03.044 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:03.102 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:03.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:03.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:03.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 90402 90386 77.3135 37.3125 0.0145542 0.0129274 2023-09-23T15:13:03.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:03.212 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:13:03.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[215646]: pgmap v480: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:03.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/272147133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[222878]: pgmap v480: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:03.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/272147133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.475 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:03 smithi027 ceph-mon[195233]: pgmap v480: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 40 MiB/s wr, 635 op/s; 29115/264192 objects degraded (11.020%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:13:03.475 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/272147133' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.475 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:03.492 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:03.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:03.605 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:03.649 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:03.770 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:13:03.819 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:03.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:03.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:03.923 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:03.967 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18560 5.4 5:24ff48df:::benchmark_data_smithi017_288048_object18559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:13:04.041 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 318.873598937, 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20308895499999999, 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18560 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.042 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163378+0000", 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163445+0000", 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6580000000000003e-05 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163462+0000", 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7041000000000002e-05 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163469+0000", 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3359999999999997e-06 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163522+0000", 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2321000000000003e-05 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163644+0000", 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012266800000000001 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163794+0000", 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015003099999999999 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165622+0000", 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00182759 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165900+0000", 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027825799999999998 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366389+0000", 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20048898100000001 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366440+0000", 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0593000000000002e-05 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366469+0000", 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9448e-05 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18562 5.4 5:263b20e9:::benchmark_data_smithi017_288048_object18561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 318.87349426100002, 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.203517166, 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.050 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18562 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163484+0000", 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163517+0000", 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2676999999999999e-05 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163526+0000", 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4999999999999999e-06 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163531+0000", 2023-09-23T15:13:04.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9010000000000002e-06 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164584+0000", 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001053165 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164706+0000", 2023-09-23T15:13:04.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012267800000000001 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164859+0000", 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152922 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165907+0000", 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001047926 2023-09-23T15:13:04.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.166956+0000", 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001048542 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366932+0000", 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.199975919 2023-09-23T15:13:04.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366968+0000", 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5896000000000003e-05 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.367002+0000", 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4619000000000002e-05 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.057 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18578 5.4 5:24fb8b16:::benchmark_data_smithi017_288048_object18577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 318.86263754499998, 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.1939555, 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.058 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18578 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174341+0000", 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174411+0000", 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9587999999999998e-05 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174422+0000", 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1327e-05 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174427+0000", 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6149999999999999e-06 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174467+0000", 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0268000000000001e-05 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174552+0000", 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4720000000000002e-05 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174641+0000", 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9091999999999993e-05 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175376+0000", 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073561399999999999 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175539+0000", 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016279399999999999 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368206+0000", 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19266702899999999 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368260+0000", 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3872999999999997e-05 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368297+0000", 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7327999999999999e-05 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.065 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:46777 5.4 5:2f7b1ca2:::benchmark_data_smithi017_288048_object46776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 302.15295799400002, 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19520114399999999, 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 46777 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.066 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.067 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884020+0000", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884079+0000", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9199e-05 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884093+0000", 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3969e-05 2023-09-23T15:13:04.068 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884099+0000", 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8220000000000003e-06 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884150+0000", 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1026000000000002e-05 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.069 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884247+0000", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6985999999999996e-05 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884345+0000", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8879e-05 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885695+0000", 2023-09-23T15:13:04.070 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00134965 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885893+0000", 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019828700000000001 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079139+0000", 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.193245841 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.071 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079186+0000", 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7052000000000003e-05 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079222+0000", 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6183999999999998e-05 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.072 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106173 5.9 5:9fa66f55:::benchmark_data_smithi017_288048_object106172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 243.32241669000001, 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26060794199999998, 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106173 2023-09-23T15:13:04.073 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.074 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714562+0000", 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714598+0000", 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5794999999999999e-05 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.075 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714607+0000", 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2240000000000001e-06 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714612+0000", 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1070000000000004e-06 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718260+0000", 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036480330000000002 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.076 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718380+0000", 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120047 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718533+0000", 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015235699999999999 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.077 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.724954+0000", 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064215239999999996 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728531+0000", 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003576606 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975076+0000", 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24654499199999999 2023-09-23T15:13:04.078 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975123+0000", 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7216999999999999e-05 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975171+0000", 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7536e-05 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.079 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106182 5.9 5:932e2a56:::benchmark_data_smithi017_288048_object106181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 243.31542291900001, 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26155082800000001, 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.080 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106182 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.081 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721554+0000", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721601+0000", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7094e-05 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721617+0000", 2023-09-23T15:13:04.082 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.588e-05 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721625+0000", 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6120000000000001e-06 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721968+0000", 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034316400000000002 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.083 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722089+0000", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121103 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722229+0000", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139791 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728355+0000", 2023-09-23T15:13:04.084 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061265089999999996 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728751+0000", 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039578699999999998 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983023+0000", 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25427224599999998 2023-09-23T15:13:04.085 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983068+0000", 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5055000000000001e-05 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983107+0000", 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8952999999999998e-05 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.086 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126389 5.4 5:26968410:::benchmark_data_smithi017_288048_object14620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 217.79604230699999, 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236534309, 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.087 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126389 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.088 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240936+0000", 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240942+0000", 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.212e-06 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.089 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240953+0000", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1722999999999999e-05 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240961+0000", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2060000000000001e-06 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241222+0000", 2023-09-23T15:13:04.090 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026150199999999999 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241334+0000", 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111864 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241432+0000", 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8293000000000002e-05 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.091 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253717+0000", 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012284982999999999 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259156+0000", 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054391600000000002 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.092 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477373+0000", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218216728 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477429+0000", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5939000000000001e-05 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477471+0000", 2023-09-23T15:13:04.093 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2169000000000002e-05 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126392 5.4 5:22950050:::benchmark_data_smithi017_288048_object14623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 217.795770772, 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236365035, 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.094 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126392 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.095 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241208+0000", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241213+0000", 2023-09-23T15:13:04.096 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2070000000000003e-06 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241223+0000", 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6930000000000002e-06 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241229+0000", 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8490000000000002e-06 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.097 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249366+0000", 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081375960000000004 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249518+0000", 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015237200000000001 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.098 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249654+0000", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013526400000000001 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253846+0000", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041924930000000003 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259182+0000", 2023-09-23T15:13:04.099 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053355219999999997 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477502+0000", 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21831988299999999 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477543+0000", 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1448e-05 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.100 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477574+0000", 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0459000000000001e-05 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126405 5.4 5:2fffe5cb:::benchmark_data_smithi017_288048_object14636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 217.77509134300001, 2023-09-23T15:13:04.101 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21577380400000001, 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126405 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.102 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261886+0000", 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:04.103 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261894+0000", 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0989999999999999e-06 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261912+0000", 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8020999999999999e-05 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.104 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261918+0000", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6069999999999998e-06 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261956+0000", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7832e-05 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262070+0000", 2023-09-23T15:13:04.105 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011383399999999999 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262173+0000", 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103539 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.267122+0000", 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049492349999999997 2023-09-23T15:13:04.106 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.269803+0000", 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002680375 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477601+0000", 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.207798129 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.107 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477633+0000", 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2582000000000001e-05 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477662+0000", 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8342000000000001e-05 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.108 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154970 5.4 5:2dd36d32:::benchmark_data_smithi017_288048_object43201:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 181.629071055, 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.201665601, 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154970 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.109 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.110 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407907+0000", 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407914+0000", 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2860000000000002e-06 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.111 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407934+0000", 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9840999999999999e-05 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407942+0000", 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1790000000000008e-06 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407997+0000", 2023-09-23T15:13:04.112 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5386000000000001e-05 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408150+0000", 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015222799999999999 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408276+0000", 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621600000000001 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.113 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414981+0000", 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067046579999999996 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414993+0000", 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2656e-05 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.114 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609483+0000", 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19448981700000001 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609532+0000", 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8523000000000003e-05 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609574+0000", 2023-09-23T15:13:04.115 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2299999999999998e-05 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154974 5.4 5:2790b49c:::benchmark_data_smithi017_288048_object43205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 181.626441847, 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19913220500000001, 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.116 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154974 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:13:04.117 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410536+0000", 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410542+0000", 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.417e-06 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.118 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410554+0000", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1855000000000001e-05 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410559+0000", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6640000000000003e-06 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411338+0000", 2023-09-23T15:13:04.119 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077873099999999995 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411463+0000", 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124938 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411583+0000", 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119839 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.120 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414997+0000", 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034144940000000001 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.417807+0000", 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028097830000000002 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.121 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609605+0000", 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.191798363 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609639+0000", 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3667999999999997e-05 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609670+0000", 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0574999999999998e-05 2023-09-23T15:13:04.122 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166946 5.4 5:2ea27988:::benchmark_data_smithi017_288048_object55177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 165.95038640499999, 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28508372300000001, 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.123 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166946 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.124 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086592+0000", 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086600+0000", 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4349999999999997e-06 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.125 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086612+0000", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2339e-05 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086619+0000", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5060000000000001e-06 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087518+0000", 2023-09-23T15:13:04.126 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000898762 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087616+0000", 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8196999999999995e-05 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087683+0000", 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6988000000000003e-05 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.127 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091939+0000", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042566660000000001 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095384+0000", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003444866 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371564+0000", 2023-09-23T15:13:04.128 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27617948799999997 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371623+0000", 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9219999999999999e-05 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371677+0000", 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3529999999999997e-05 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.129 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166951 5.4 5:254d5c06:::benchmark_data_smithi017_288048_object55182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 165.94531280300001, 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28024638499999999, 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166951 2023-09-23T15:13:04.130 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.131 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091665+0000", 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091672+0000", 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.132 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091685+0000", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3312000000000001e-05 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091692+0000", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4989999999999999e-06 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091958+0000", 2023-09-23T15:13:04.133 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026625599999999998 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092057+0000", 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9019999999999997e-05 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092171+0000", 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114345 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098893+0000", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067218640000000001 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100313+0000", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014193369999999999 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371840+0000", 2023-09-23T15:13:04.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27152736500000002 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371876+0000", 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6191e-05 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371913+0000", 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6745999999999997e-05 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.136 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:175544 5.4 5:26fea254:::benchmark_data_smithi017_288048_object63775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 154.31410298599999, 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18268135499999999, 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.137 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 175544 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722874+0000", 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722882+0000", 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6249999999999998e-06 2023-09-23T15:13:04.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722903+0000", 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1369000000000002e-05 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722911+0000", 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3560000000000004e-06 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722956+0000", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5698000000000002e-05 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723091+0000", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013489599999999999 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723244+0000", 2023-09-23T15:13:04.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152387 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.728221+0000", 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049771060000000002 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.732170+0000", 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039488830000000003 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905471+0000", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17330090500000001 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905520+0000", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9104999999999998e-05 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905558+0000", 2023-09-23T15:13:04.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8028999999999998e-05 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:176231 5.9 5:95f16d04:::benchmark_data_smithi017_288048_object64462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 153.35224660700001, 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17770085299999999, 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.144 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 176231 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684732+0000", 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684735+0000", 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1430000000000001e-06 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684738+0000", 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5710000000000002e-06 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684742+0000", 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1459999999999998e-06 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688383+0000", 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036404229999999998 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688504+0000", 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121162 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688627+0000", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123273 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691418+0000", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002790837 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691423+0000", 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2460000000000003e-06 2023-09-23T15:13:04.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862340+0000", 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17091711200000001 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862391+0000", 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0390999999999999e-05 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862434+0000", 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.286e-05 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187773 5.9 5:98d8c468:::benchmark_data_smithi017_288048_object76004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 137.69946564700001, 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20575924800000001, 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.151 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187773 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:13:04.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337512+0000", 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337520+0000", 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8520000000000004e-06 2023-09-23T15:13:04.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337539+0000", 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9249000000000002e-05 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337547+0000", 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3399999999999998e-06 2023-09-23T15:13:04.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337601+0000", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4172000000000002e-05 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337728+0000", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126198 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337851+0000", 2023-09-23T15:13:04.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012301799999999999 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.341076+0000", 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003225136 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342051+0000", 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097561399999999997 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543167+0000", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20111568099999999 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543222+0000", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4877000000000003e-05 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543273+0000", 2023-09-23T15:13:04.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0952000000000003e-05 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187774 5.9 5:9a77e4c2:::benchmark_data_smithi017_288048_object76005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 137.694438542, 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20974965300000001, 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.160 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187774 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:13:04.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342539+0000", 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342545+0000", 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342558+0000", 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2813000000000001e-05 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342569+0000", 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0916e-05 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342603+0000", 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4600000000000001e-05 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342700+0000", 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6618000000000002e-05 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342775+0000", 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5092000000000005e-05 2023-09-23T15:13:04.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.345881+0000", 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031061130000000002 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.346007+0000", 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012578200000000001 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552158+0000", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20615127 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552231+0000", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2809e-05 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552290+0000", 2023-09-23T15:13:04.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9240000000000002e-05 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197695 5.9 5:9f6a849a:::benchmark_data_smithi017_288048_object85926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 124.006278958, 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.204857551, 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.167 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197695 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:13:04.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030699+0000", 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030707+0000", 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8480000000000001e-06 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030717+0000", 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414999999999999e-05 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030724+0000", 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9360000000000002e-06 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031186+0000", 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046191899999999998 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031291+0000", 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010539000000000001 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031395+0000", 2023-09-23T15:13:04.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010343199999999999 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.034429+0000", 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030340480000000001 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039920+0000", 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054914489999999998 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235467+0000", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19554693500000001 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235519+0000", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1835000000000001e-05 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235558+0000", 2023-09-23T15:13:04.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8813000000000003e-05 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197697 5.9 5:93334a62:::benchmark_data_smithi017_288048_object85928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 124.006246508, 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.216589857, 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:04.174 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197697 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030732+0000", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030735+0000", 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6709999999999999e-06 2023-09-23T15:13:04.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030741+0000", 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0909999999999998e-06 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030745+0000", 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2579999999999997e-06 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.036909+0000", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061638029999999998 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037024+0000", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011513 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037142+0000", 2023-09-23T15:13:04.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118171 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039953+0000", 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002810707 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.040002+0000", 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8758999999999999e-05 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247229+0000", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20722660500000001 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247277+0000", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8078999999999998e-05 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247323+0000", 2023-09-23T15:13:04.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6004000000000002e-05 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200855 5.4 5:2c8e5777:::benchmark_data_smithi017_288048_object89086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 119.888936156, 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37976222399999998, 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:13:04.181 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200855 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148042+0000", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148048+0000", 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2020000000000001e-06 2023-09-23T15:13:04.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148058+0000", 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5910000000000002e-06 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148063+0000", 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8760000000000004e-06 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148114+0000", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1851999999999998e-05 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148231+0000", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011667699999999999 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148336+0000", 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010487799999999999 2023-09-23T15:13:04.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.151117+0000", 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027812620000000001 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.152434+0000", 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013172259999999999 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527791+0000", 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37535686699999998 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527805+0000", 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4005999999999999e-05 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:13:04.187 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:13:04.188 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:13:04.188 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 91004 90988 76.7776 37.625 0.0194679 0.0130181 2023-09-23T15:13:04.188 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:04.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[215646]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:04.189 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[215646]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T15:13:04.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:04.189 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:04.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:04.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:04 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:04 smithi027 ceph-mon[195233]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:04 smithi027 ceph-mon[195233]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T15:13:04.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:04 smithi027 ceph-mon[195233]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.357 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:04.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[215646]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:04.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[222878]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[222878]: osdmap e192: 8 total, 8 up, 7 in 2023-09-23T15:13:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:04 smithi017 ceph-mon[222878]: failed to encode map e192 with expected crc 2023-09-23T15:13:04.526 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:04.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:04.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:04.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:04.676 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:04.743 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:04.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:04.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:04.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:04.888 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:04.898 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:04.986 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:04.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:05.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:13:05.089 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:05.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 91625 91609 76.2718 38.8125 0.033442 0.0131062 2023-09-23T15:13:05.128 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:05.188 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:05.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:05.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:05.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:05.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:05 smithi027 ceph-mon[195233]: pgmap v482: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 31 MiB/s wr, 499 op/s; 29115/267057 objects degraded (10.902%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:13:05.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:05 smithi017 ceph-mon[215646]: pgmap v482: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 31 MiB/s wr, 499 op/s; 29115/267057 objects degraded (10.902%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:13:05.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:05 smithi017 ceph-mon[222878]: pgmap v482: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 31 MiB/s wr, 499 op/s; 29115/267057 objects degraded (10.902%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:13:05.464 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:05.753 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:05.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:05.763 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:05.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:05.956 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:06.105 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:06.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:06.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:06.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 92248 92232 75.7809 38.9375 0.00923724 0.0131909 2023-09-23T15:13:06.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:06.216 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:13:06.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:13:06.245 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:06.303 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:06.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:06.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:06.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:13:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:13:06.560 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:13:06.727 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:06.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:06.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:06.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:06.945 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:07.127 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 92845 92829 75.2731 37.3125 0.0209164 0.0132793 2023-09-23T15:13:07.262 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:07.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:07.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:07.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:07.408 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:07.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:13:07.469 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: pgmap v484: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 28906/273216 objects degraded (10.580%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:13:07.477 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29115/267057 objects degraded (10.902%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:07.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:07.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: pgmap v484: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 28906/273216 objects degraded (10.580%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29115/267057 objects degraded (10.902%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4209001070' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: osdmap e193: 8 total, 8 up, 7 in 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: failed to encode map e193 with expected crc 2023-09-23T15:13:07.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: pgmap v484: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 28906/273216 objects degraded (10.580%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:13:07.533 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29115/267057 objects degraded (10.902%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:07.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:07.608 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:07.664 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:07.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:07.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:07.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:07.755 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:07.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:07.877 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:07.945 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:13:07.985 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:13:08.050 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:08.109 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:08.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 93412 93396 74.761 35.4375 0.0138507 0.0133684 2023-09-23T15:13:08.151 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:08.174 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:08] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-09-23T15:13:08.183 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:08.201 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:08.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:08.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:08.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:08.273 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:08.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.301 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:13:08.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:08.347 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:08.408 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:08.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:08.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:08.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2669113085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:08.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2669113085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:08.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2669113085' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:08.627 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:09.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 94010 93994 74.2882 37.375 0.0449016 0.0134543 2023-09-23T15:13:09.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:09 smithi017 ceph-mon[215646]: pgmap v485: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s; 28906/273216 objects degraded (10.580%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T15:13:09.313 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:09 smithi017 ceph-mon[222878]: pgmap v485: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s; 28906/273216 objects degraded (10.580%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T15:13:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:09 smithi027 ceph-mon[195233]: pgmap v485: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 23 MiB/s wr, 375 op/s; 28906/273216 objects degraded (10.580%); 1.6 MiB/s, 26 objects/s recovering 2023-09-23T15:13:09.863 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:09.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:09.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:09.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:10.060 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:10.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:13:10.129185+0000 min lat: 0.00114865 max lat: 1.78741 avg lat: 0.0135267 2023-09-23T15:13:10.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:13:10.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 94692 94676 73.8927 42.625 0.0322464 0.0135267 2023-09-23T15:13:10.294 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:10.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:10.401 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:10.428 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:10.487 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:10.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:10.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:10.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:10.659 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:10.889 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:11.001 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:11.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:11.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:11.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:11.131 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 95253 95237 73.4118 35.0625 0.0294441 0.0136154 2023-09-23T15:13:11.206 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:11.439 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:13:11.439 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:13:11.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:11 smithi017 ceph-mon[215646]: pgmap v486: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s; 28708/282261 objects degraded (10.171%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:13:11.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:11 smithi017 ceph-mon[222878]: pgmap v486: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s; 28708/282261 objects degraded (10.171%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:13:11.669 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:11.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:11.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:11 smithi027 ceph-mon[195233]: pgmap v486: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 752 op/s; 28708/282261 objects degraded (10.171%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:13:11.871 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:12.133 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 95833 95817 72.9568 36.25 0.0205575 0.0137013 2023-09-23T15:13:12.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28708/282261 objects degraded (10.171%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1422285044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28708/282261 objects degraded (10.171%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1422285044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.560 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.586 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:12.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:12.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:12.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:12.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28708/282261 objects degraded (10.171%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:12.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1422285044' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.724 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:12.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:12.795 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:12.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:12.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:12.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:12.938 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:13:12.990 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:13.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:13:13.133 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 96424 96408 72.5232 36.9375 0.00704261 0.0137839 2023-09-23T15:13:13.188 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:13.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:13.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:13.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:13.382 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: pgmap v487: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 28708/282261 objects degraded (10.171%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: failed to encode map e194 with expected crc 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:13.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: pgmap v487: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 28708/282261 objects degraded (10.171%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: failed to encode map e194 with expected crc 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:13.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:13.825 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:13.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:13.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:13.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: pgmap v487: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 660 op/s; 28708/282261 objects degraded (10.171%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: failed to encode map e194 with expected crc 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: osdmap e194: 8 total, 8 up, 7 in 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:13.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:13.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:14.035 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:14.101 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:14.133 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 96926 96910 72.0337 31.375 0.0439459 0.0138758 2023-09-23T15:13:14.144 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:14.148 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:14.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:14.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:14.339 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:14.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:14 smithi017 ceph-mon[215646]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:14 smithi017 ceph-mon[215646]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:14 smithi017 ceph-mon[222878]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:14 smithi017 ceph-mon[222878]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.802 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:14.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:14.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:14.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:14 smithi027 ceph-mon[195233]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:14 smithi027 ceph-mon[195233]: failed to encode map e194 with expected crc 2023-09-23T15:13:14.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:14.949 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:15.009 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:15.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:15.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:15.114 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:15.137 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 97510 97494 71.613 36.5 0.0164604 0.0139576 2023-09-23T15:13:15.145 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:15.207 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:15.214 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:15.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:15.427 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:15.436 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:13:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:15 smithi027 ceph-mon[195233]: pgmap v489: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 28708/285042 objects degraded (10.071%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:13:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1940924067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[222878]: pgmap v489: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 28708/285042 objects degraded (10.071%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:13:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1940924067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[215646]: pgmap v489: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 30 MiB/s wr, 487 op/s; 28708/285042 objects degraded (10.071%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:13:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1940924067' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:13:15.853 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:15.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:15.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:15.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:13:15.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:16.077 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:16.137 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 98061 98045 71.1811 34.4375 0.0372918 0.0140432 2023-09-23T15:13:16.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[215646]: failed to encode map e195 with expected crc 2023-09-23T15:13:16.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:13:16.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[215646]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T15:13:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[222878]: failed to encode map e195 with expected crc 2023-09-23T15:13:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:13:16.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:16 smithi017 ceph-mon[222878]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T15:13:16.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:16.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:16.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:16 smithi027 ceph-mon[195233]: failed to encode map e195 with expected crc 2023-09-23T15:13:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:13:16.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:16 smithi027 ceph-mon[195233]: osdmap e195: 8 total, 8 up, 7 in 2023-09-23T15:13:16.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:17.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:17.137 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 98644 98628 70.782 36.4375 0.0165205 0.0141219 2023-09-23T15:13:17.381 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:17.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:17.420 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:17.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:13:17.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:17.552 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:17.613 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:17.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:17.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:17.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:17 smithi017 ceph-mon[215646]: pgmap v491: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 28480/290976 objects degraded (9.788%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:13:17.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28480/290976 objects degraded (9.788%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:17.689 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:17 smithi017 ceph-mon[222878]: pgmap v491: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 28480/290976 objects degraded (9.788%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:13:17.689 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28480/290976 objects degraded (9.788%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:17.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:17.745 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:17 smithi027 ceph-mon[195233]: pgmap v491: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 739 op/s; 28480/290976 objects degraded (9.788%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:13:17.746 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28480/290976 objects degraded (9.788%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:17.750 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:17.806 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:17.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:17.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:17.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:17.944 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:17.946 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:18.003 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:18.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:18.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:13:18.093 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:13:18.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:18.127 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:13:18.128 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:13:18.128 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:13:18.128 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:13:18.128 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:13:18.140 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 99193 99177 70.3658 34.3125 0.0110929 0.0142036 2023-09-23T15:13:18.197 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:18.248 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:18.256 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:18.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:18.336 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:18.381 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:18.423 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:18] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.43.0" 2023-09-23T15:13:18.423 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:18.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.453 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.454 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.454 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.454 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:13:18.623 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:18.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:18.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3383463665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:18.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3383463665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:18.732 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:18.762 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3383463665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:18.765 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:18.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:18.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:18.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:18.976 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:19.042 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:19.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:19.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:19.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 99787 99771 69.991 37.125 0.0115106 0.0142816 2023-09-23T15:13:19.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:19.227 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:19.629 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:19.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:19.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:19.741 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:19.771 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:19 smithi027 ceph-mon[195233]: pgmap v492: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 28480/290976 objects degraded (9.788%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:19.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:19 smithi017 ceph-mon[215646]: pgmap v492: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 28480/290976 objects degraded (9.788%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:19.773 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:19 smithi017 ceph-mon[222878]: pgmap v492: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.9 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 23 MiB/s wr, 362 op/s; 28480/290976 objects degraded (9.788%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:19.774 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:19.842 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:19.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:19.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:19.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:13:19.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:20.061 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:20.062 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:20.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0151 2023-09-23T15:13:20.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 100388 2023-09-23T15:13:20.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 69.7022 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 44.2299 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 247.75 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1115 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 707.685 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3964 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0143421 2023-09-23T15:13:20.143 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0272156 2023-09-23T15:13:20.144 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 1.78741 2023-09-23T15:13:20.144 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00114865 2023-09-23T15:13:20.154 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:13:20.284 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:20.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:20.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:20.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:20.430 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:20.477 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:20.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:20.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:20.632 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:21.180 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:21.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:21.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:21.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:21.397 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:21.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:21 smithi017 ceph-mon[222878]: pgmap v493: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 28244/299484 objects degraded (9.431%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:13:21.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:21 smithi017 ceph-mon[215646]: pgmap v493: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 28244/299484 objects degraded (9.431%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:13:21.733 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:21.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:21.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:21.838 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:21.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:21 smithi027 ceph-mon[195233]: pgmap v493: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 717 op/s; 28244/299484 objects degraded (9.431%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:13:21.873 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:21.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:13:21.945 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:21.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:21.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:22.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:22.079 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:22.181 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:22.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:22.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:22.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:22.378 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:22.494 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:22.545 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:22.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:22.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:22.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:22.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28244/299484 objects degraded (9.431%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/260103662' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28244/299484 objects degraded (9.431%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:22.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/260103662' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.781 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:22.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28244/299484 objects degraded (9.431%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/260103662' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:13:22.929 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:13:23.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[215646]: pgmap v494: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 28244/299484 objects degraded (9.431%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T15:13:23.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:23.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[215646]: failed to encode map e196 with expected crc 2023-09-23T15:13:23.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:13:23.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[215646]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T15:13:23.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[222878]: pgmap v494: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 28244/299484 objects degraded (9.431%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T15:13:23.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:23.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[222878]: failed to encode map e196 with expected crc 2023-09-23T15:13:23.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:13:23.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:23 smithi017 ceph-mon[222878]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T15:13:23.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:23 smithi027 ceph-mon[195233]: pgmap v494: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 38 MiB/s wr, 608 op/s; 28244/299484 objects degraded (9.431%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T15:13:23.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:23 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:23.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:23 smithi027 ceph-mon[195233]: failed to encode map e196 with expected crc 2023-09-23T15:13:23.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:23 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:13:23.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:23 smithi027 ceph-mon[195233]: osdmap e196: 8 total, 8 up, 7 in 2023-09-23T15:13:24.164 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:24.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:24.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:24.275 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:24.314 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:24.372 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:24.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:24.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:24.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:24.587 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:24.613 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:24.841 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:24.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:24.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:24.959 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:25.046 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:25.201 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:25.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:25.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:25.308 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:25.344 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:25.400 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:25.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:25.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:25.506 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:25.557 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:25.557 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:13:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:25 smithi027 ceph-mon[195233]: pgmap v496: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 420 op/s; 28244/298485 objects degraded (9.462%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T15:13:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1808975482' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.620 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:25.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:25.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:25.733 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:13:25.733 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:13:25.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:13:25.733 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:13:25.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.734 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78609 6.1 6.9e872b01 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:13:25.734 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:13:25.737 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.129697585000002, 2023-09-23T15:13:25.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25765822199999999, 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:25.738 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78609 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:13:25.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587121+0000", 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587196+0000", 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4975999999999995e-05 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587203+0000", 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8000000000000001e-06 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587210+0000", 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7819999999999996e-06 2023-09-23T15:13:25.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587261+0000", 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0929e-05 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587270+0000", 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844769+0000", 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.257499058 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844781+0000", 2023-09-23T15:13:25.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1984000000000001e-05 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.747 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78617 6.1 6.bd740781 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.129597209000003, 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25759161800000002, 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:25.748 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78617 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:13:25.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587222+0000", 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587267+0000", 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4282000000000001e-05 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587275+0000", 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9270000000000005e-06 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587283+0000", 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7940000000000008e-06 2023-09-23T15:13:25.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587707+0000", 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042393500000000001 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587715+0000", 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7619999999999995e-06 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844807+0000", 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25709155299999997 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844815+0000", 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9929999999999995e-06 2023-09-23T15:13:25.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78636 6.4 6.f7e668f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:13:25.757 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.104601432000003, 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.238510681, 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:25.758 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78636 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612217+0000", 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612283+0000", 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6523e-05 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612306+0000", 2023-09-23T15:13:25.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2892999999999999e-05 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612316+0000", 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6940000000000007e-06 2023-09-23T15:13:25.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612373+0000", 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7399999999999999e-05 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612397+0000", 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.37e-05 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850720+0000", 2023-09-23T15:13:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23832351900000001 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850730+0000", 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1220000000000001e-06 2023-09-23T15:13:25.766 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78637 6.1 6.c6e1dd41 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:13:25.767 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.104489088999998, 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23251106799999999, 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:25.768 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78637 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612374+0000", 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3646000000000002e-05 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612390+0000", 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5828000000000002e-05 2023-09-23T15:13:25.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612398+0000", 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5989999999999996e-06 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612435+0000", 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6967e-05 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:13:25.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612441+0000", 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9039999999999997e-06 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844835+0000", 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23239410299999999 2023-09-23T15:13:25.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844842+0000", 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4850000000000003e-06 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78638 6.4 6.d9b25834 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 44.103884844, 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.237823757, 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.777 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78638 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.778 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612976+0000", 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1334000000000001e-05 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612990+0000", 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4146e-05 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613002+0000", 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1308000000000001e-05 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613049+0000", 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7085000000000001e-05 2023-09-23T15:13:25.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613057+0000", 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7849999999999997e-06 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850751+0000", 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23769393899999999 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850759+0000", 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8979999999999996e-06 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.786 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:78875 6.1 e185/184 6:82239a6b:::benchmark_data_smithi017_350392_object78874:head v 185'4896, mlcod=0'0)", 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:43.573520+0000", 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.143300308000001, 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13971820600000001, 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573520+0000", 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573520+0000", 2023-09-23T15:13:25.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573519+0000", 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573551+0000", 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1900000000000003e-05 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573562+0000", 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1215e-05 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.573570+0000", 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6830000000000008e-06 2023-09-23T15:13:25.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.578869+0000", 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052993850000000002 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.578898+0000", 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8510999999999999e-05 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.713191+0000", 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13429286300000001 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.713238+0000", 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7715e-05 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:78881 6.4 e185/184 6:2d85b9db:::benchmark_data_smithi017_350392_object78880:head v 185'4965, mlcod=0'0)", 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:43.580240+0000", 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.136580373999998, 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13378405099999999, 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580240+0000", 2023-09-23T15:13:25.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580240+0000", 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580239+0000", 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580266+0000", 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7557000000000001e-05 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580277+0000", 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0550999999999999e-05 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580284+0000", 2023-09-23T15:13:25.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1940000000000001e-06 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580354+0000", 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0327999999999995e-05 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.580376+0000", 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1596000000000001e-05 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.713988+0000", 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.133611864 2023-09-23T15:13:25.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.714024+0000", 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6050999999999998e-05 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:78884 6.3 e185/184 6:cc0649b5:::benchmark_data_smithi017_350392_object78883:head v 185'4991, mlcod=0'0)", 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:43.584017+0000", 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.132803379000002, 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.134335489, 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.584017+0000", 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.584017+0000", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.584016+0000", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.584133+0000", 2023-09-23T15:13:25.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116595 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.710511+0000", 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12637849300000001 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.710524+0000", 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2670999999999999e-05 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711956+0000", 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014322810000000001 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711981+0000", 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4445000000000001e-05 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.718311+0000", 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063304559999999999 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.718352+0000", 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1294999999999997e-05 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(6.d 185/184 [PushOp(6:b4c006a7:::benchmark_data_smithi017_350392_object238:head, version: 174'12, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:b4c006a7:::benchmark_data_smithi017_350392_object238:head@174'12, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:43.595049+0000", 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.121771729999999, 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.118457985, 2023-09-23T15:13:25.805 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.595049+0000", 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.595049+0000", 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.595047+0000", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.595165+0000", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000118805 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711265+0000", 2023-09-23T15:13:25.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11609908400000001 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711302+0000", 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7638000000000001e-05 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711709+0000", 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00040702400000000003 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.711723+0000", 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3397999999999999e-05 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.713507+0000", 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001784009 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:79630 6.4 e186/184 6:239da09b:::benchmark_data_smithi017_350392_object79629:head v 186'5007, mlcod=0'0)", 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.830190084999998, 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18234605200000001, 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:13:25.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886629+0000", 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886655+0000", 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6469e-05 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048539+0000", 2023-09-23T15:13:25.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161884114 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048548+0000", 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3510000000000002e-06 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.051880+0000", 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033326800000000002 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.051911+0000", 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0719999999999997e-05 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.068933+0000", 2023-09-23T15:13:25.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017022036000000001 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.068976+0000", 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3101e-05 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:79638 6.3 e186/184 6:c2c44f00:::benchmark_data_smithi017_350392_object79637:head v 186'5035, mlcod=0'0)", 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:44.893516+0000", 2023-09-23T15:13:25.817 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.823304702999998, 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.171558237, 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.893516+0000", 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.893516+0000", 2023-09-23T15:13:25.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.893514+0000", 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.893615+0000", 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100813 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048433+0000", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.154817491 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048458+0000", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5490000000000002e-05 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.049192+0000", 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073424799999999995 2023-09-23T15:13:25.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.049254+0000", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1665e-05 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.065030+0000", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015775595 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.065074+0000", 2023-09-23T15:13:25.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4131000000000001e-05 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(6.d 186/184 [PushOp(6:b6a2cc72:::benchmark_data_smithi017_350392_object1013:head, version: 174'53, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:b6a2cc72:::benchmark_data_smithi017_350392_object1013:head@174'53, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:44.894302+0000", 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 40.822518371999998, 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15642345199999999, 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.894302+0000", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.894302+0000", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.894301+0000", 2023-09-23T15:13:25.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.894358+0000", 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7413000000000001e-05 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048644+0000", 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15428598099999999 2023-09-23T15:13:25.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048671+0000", 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6455e-05 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.049049+0000", 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037804899999999998 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.049065+0000", 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5838e-05 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.050725+0000", 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016607239999999999 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81740 6.e e188/184 6:732b0eb5:::benchmark_data_smithi017_350392_object81739:head v 188'5226, mlcod=0'0)", 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.719341+0000", 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 36.997479583999997, 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17366520399999999, 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719341+0000", 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719341+0000", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719340+0000", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719415+0000", 2023-09-23T15:13:25.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4899000000000002e-05 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719428+0000", 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2952000000000001e-05 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719438+0000", 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9429999999999995e-06 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.731358+0000", 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011920566 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.731388+0000", 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0049999999999999e-05 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.892953+0000", 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161565296 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.893006+0000", 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2502e-05 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.831 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81754 6.e e188/184 6:7fc47cda:::benchmark_data_smithi017_350392_object81753:head v 188'5227, mlcod=0'0)", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 36.997374084999997, 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17966953999999999, 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719487+0000", 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1573000000000003e-05 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719495+0000", 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4490000000000002e-06 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719500+0000", 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0799999999999996e-06 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.737767+0000", 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018267255 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.737792+0000", 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.493e-05 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.899075+0000", 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161283123 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.899116+0000", 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0494999999999998e-05 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81766 6.e e188/184 6:7bc810f6:::benchmark_data_smithi017_350392_object81765:head v 188'5228, mlcod=0'0)", 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 36.996791582999997, 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.185409134, 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.836 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720028+0000", 2023-09-23T15:13:25.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720256+0000", 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022787799999999999 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720267+0000", 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0808999999999999e-05 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720273+0000", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3899999999999998e-06 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.744243+0000", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023969824000000001 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.744258+0000", 2023-09-23T15:13:25.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5251e-05 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905371+0000", 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16111323899999999 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905438+0000", 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.840 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81770 6.e e188/184 6:75c4e65d:::benchmark_data_smithi017_350392_object81769:head v 188'5229, mlcod=0'0)", 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 36.995318527999999, 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.183962289, 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721501+0000", 2023-09-23T15:13:25.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721576+0000", 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5160000000000005e-05 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721592+0000", 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6030999999999999e-05 2023-09-23T15:13:25.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721600+0000", 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7919999999999993e-06 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.893038+0000", 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17143856499999999 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.893063+0000", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4309000000000001e-05 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905444+0000", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012381647000000001 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905464+0000", 2023-09-23T15:13:25.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9684e-05 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:101428 6.d e195/184 6:b6958f92:::benchmark_data_smithi017_350392_object1037:head v 195'6321, mlcod=175'1572)", 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:13:21.957602+0000", 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.7592179610000001, 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.162796266, 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.846 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957602+0000", 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957602+0000", 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957601+0000", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957610+0000", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3379999999999994e-06 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957623+0000", 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2822e-05 2023-09-23T15:13:25.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957631+0000", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4789999999999996e-06 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957677+0000", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5729000000000002e-05 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.957702+0000", 2023-09-23T15:13:25.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4465999999999999e-05 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.120339+0000", 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16263720600000001 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.120399+0000", 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9876000000000001e-05 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.850 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:101429 6.b e195/184 6:d1b4263b:::benchmark_data_smithi017_350392_object1038:head v 195'6394, mlcod=0'0)", 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:13:21.960350+0000", 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.7564707469999998, 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16419103199999999, 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960350+0000", 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960350+0000", 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960349+0000", 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960357+0000", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3979999999999999e-06 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960368+0000", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1272e-05 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960376+0000", 2023-09-23T15:13:25.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7060000000000004e-06 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.960464+0000", 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7724000000000002e-05 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117328+0000", 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.156864211 2023-09-23T15:13:25.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.124493+0000", 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071650409999999996 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.124541+0000", 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7648999999999997e-05 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPush(6.d 195/184 [PushOp(6:b425c51c:::benchmark_data_smithi017_350392_object24812:head, version: 175'1573, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(6:b425c51c:::benchmark_data_smithi017_350392_object24812:head@175'1573, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:13:21.964974+0000", 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.751845893, 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15531964100000001, 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.964974+0000", 2023-09-23T15:13:25.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.964974+0000", 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.964973+0000", 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.965026+0000", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2930999999999999e-05 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117297+0000", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.152270448 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117324+0000", 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7685999999999999e-05 2023-09-23T15:13:25.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117402+0000", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7547999999999997e-05 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117417+0000", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4565e-05 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.120294+0000", 2023-09-23T15:13:25.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028774830000000001 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:101436 6.4 e195/184 6:2a0d3c09:::benchmark_data_smithi017_350392_object1045:head v 195'6392, mlcod=0'0)", 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:13:21.967302+0000", 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 3.7495181909999999, 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15732950100000001, 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:25.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.967302+0000", 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.967302+0000", 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:25.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.967301+0000", 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:21.967311+0000", 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7589999999999992e-06 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117290+0000", 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14997933899999999 2023-09-23T15:13:25.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117311+0000", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0971e-05 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117366+0000", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4796999999999997e-05 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.117406+0000", 2023-09-23T15:13:25.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9996000000000001e-05 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.124602+0000", 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071956609999999999 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:13:22.124632+0000", 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9941e-05 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.864 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:25.865 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:25.865 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:13:25.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[222878]: pgmap v496: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 420 op/s; 28244/298485 objects degraded (9.462%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T15:13:25.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1808975482' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[215646]: pgmap v496: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.1 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 22 MiB/s wr, 420 op/s; 28244/298485 objects degraded (9.462%); 1.7 MiB/s, 26 objects/s recovering 2023-09-23T15:13:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1808975482' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.866 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:13:25.866 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:26.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:13:26.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[215646]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[215646]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:13:26.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[215646]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T15:13:26.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[222878]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[222878]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:13:26.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:26 smithi017 ceph-mon[222878]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T15:13:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:26 smithi027 ceph-mon[195233]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:26 smithi027 ceph-mon[195233]: failed to encode map e197 with expected crc 2023-09-23T15:13:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:13:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:26 smithi027 ceph-mon[195233]: osdmap e197: 8 total, 8 up, 7 in 2023-09-23T15:13:26.998 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:27.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:13:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:27.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:27.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:27.223 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:27.552 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:27.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:27.661 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:27.755 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:27.756 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:13:27.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:27 smithi017 ceph-mon[215646]: pgmap v498: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 27 MiB/s wr, 713 op/s; 28005/294381 objects degraded (9.513%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:27.832 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28005/294381 objects degraded (9.513%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:27.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:27 smithi017 ceph-mon[222878]: pgmap v498: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 27 MiB/s wr, 713 op/s; 28005/294381 objects degraded (9.513%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:27.832 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28005/294381 objects degraded (9.513%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:27 smithi027 ceph-mon[195233]: pgmap v498: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 27 MiB/s wr, 713 op/s; 28005/294381 objects degraded (9.513%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28005/294381 objects degraded (9.513%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:27.926 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:27.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:28.039 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:28.072 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:28.130 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:28.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:28.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:28.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:28.196 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:28] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.43.0" 2023-09-23T15:13:28.196 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:28.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:28.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:13:28.314 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:13:28.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:28.362 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:28.436 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:28.472 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:28.514 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:28.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:28.602 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:28.613 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.614 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:13:28.895 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:28.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:29.001 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:29.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2987924578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:29.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[222878]: pgmap v499: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 359 op/s; 28005/294381 objects degraded (9.513%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:13:29.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:29.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:29.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2987924578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:29.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:29 smithi017 ceph-mon[215646]: pgmap v499: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 359 op/s; 28005/294381 objects degraded (9.513%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:13:29.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2987924578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:29 smithi027 ceph-mon[195233]: pgmap v499: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 6.0 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 359 op/s; 28005/294381 objects degraded (9.513%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:13:31.002 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:31.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:31.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:31.094 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:31.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:31.147 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:31.200 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:31.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:31.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:31.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:31.339 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:31.404 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:31.410 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:31 smithi027 ceph-mon[195233]: pgmap v500: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 723 op/s; 27756/285645 objects degraded (9.717%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:31.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:31.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:31 smithi017 ceph-mon[215646]: pgmap v500: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 723 op/s; 27756/285645 objects degraded (9.717%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:31 smithi017 ceph-mon[222878]: pgmap v500: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 4.6 MiB/s wr, 723 op/s; 27756/285645 objects degraded (9.717%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:31.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:31.568 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:31.810 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:31.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:31.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:31.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:31.961 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:32.022 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:32.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:32.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:32.235 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:32.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27756/285645 objects degraded (9.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:32.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/400619502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.397 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:32.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:32.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:32.542 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:32.599 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:32.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:32.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:32.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27756/285645 objects degraded (9.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:32.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/400619502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.673 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:32 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:32.753 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:32.841 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:32.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:32.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27756/285645 objects degraded (9.717%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:32.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/400619502' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:32 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:32.895 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:13:32.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:13:32.926 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:13:32.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:33.069 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:33.347 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:33.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:33.351 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:33.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:33.524 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:33.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: pgmap v501: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 108 B/s rd, 3.9 MiB/s wr, 614 op/s; 27756/285645 objects degraded (9.717%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[215646]: failed to encode map e198 with expected crc 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: pgmap v501: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 108 B/s rd, 3.9 MiB/s wr, 614 op/s; 27756/285645 objects degraded (9.717%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:33.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:33.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:13:33.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T15:13:33.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:33 smithi017 ceph-mon[222878]: failed to encode map e198 with expected crc 2023-09-23T15:13:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: pgmap v501: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 108 B/s rd, 3.9 MiB/s wr, 614 op/s; 27756/285645 objects degraded (9.717%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/586060151' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: osdmap e198: 8 total, 8 up, 7 in 2023-09-23T15:13:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:33 smithi027 ceph-mon[195233]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.356 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:34.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:34.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:34.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:13:34.460 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:34.494 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:34.560 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:34.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:34.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:34.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[215646]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[215646]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[215646]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[215646]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[222878]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[222878]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[222878]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:34 smithi017 ceph-mon[222878]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:34.700 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:34 smithi027 ceph-mon[195233]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.700 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:34 smithi027 ceph-mon[195233]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.700 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:34 smithi027 ceph-mon[195233]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.700 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:34 smithi027 ceph-mon[195233]: failed to encode map e198 with expected crc 2023-09-23T15:13:34.704 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:34.779 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:34.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:34.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:34.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:34.980 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:13:34.980 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:13:35.004 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:35.014 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:35.222 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:35.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:35.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:35.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:35.373 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:35.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:35 smithi017 ceph-mon[215646]: pgmap v503: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 27756/282939 objects degraded (9.810%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:35.393 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:35 smithi017 ceph-mon[222878]: pgmap v503: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 27756/282939 objects degraded (9.810%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:35.443 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:35.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:35.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:35.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:35.618 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:35.729 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:35.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:35.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:35.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:35 smithi027 ceph-mon[195233]: pgmap v503: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 433 op/s; 27756/282939 objects degraded (9.810%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:13:35.882 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:35.957 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:35.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:36.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:36.094 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:36.156 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:36.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:36.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:36.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:13:36.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:36.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:13:36.396 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:36.427 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:37.194 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:37.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:37.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:37.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:37.346 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:37.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:37.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:37.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:37.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:37 smithi017 ceph-mon[215646]: pgmap v504: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:37.465 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27487/276858 objects degraded (9.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:37.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:37 smithi017 ceph-mon[222878]: pgmap v504: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:37.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27487/276858 objects degraded (9.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:37.516 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:37.553 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:37 smithi027 ceph-mon[195233]: pgmap v504: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27487/276858 objects degraded (9.928%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:37.626 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:37.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:37.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:37.838 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:37.870 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:38.027 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:38] "GET /metrics HTTP/1.1" 200 36323 "" "Prometheus/2.43.0" 2023-09-23T15:13:38.405 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:38.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:38.412 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:38.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:38.547 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:38.619 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:38.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:38.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:38.756 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:38.813 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:38.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:38.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:38.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:38.952 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:38.964 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:39.004 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:39.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:39.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:39.148 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:39.155 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:13:39.187 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:13:39.218 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:39.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:39.229 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:39.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:39.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:39.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:39.356 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:39.398 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:39.413 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:39.415 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:39 smithi027 ceph-mon[195233]: pgmap v505: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:39.415 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4034137802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:39.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:39.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:39.442 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:39.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:39.496 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:13:39.518 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:39.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:39 smithi017 ceph-mon[215646]: pgmap v505: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:39.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4034137802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:39.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:39 smithi017 ceph-mon[222878]: pgmap v505: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 583 op/s; 27487/276858 objects degraded (9.928%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:13:39.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4034137802' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:39.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:40.355 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:40.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:40.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:40.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:40.481 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:13:40.503 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:40.560 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:40.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:40.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/786889596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.631 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/786889596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.666 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:40.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/786889596' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.694 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:13:40.697 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:40.762 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:40.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:40.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:40.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:40.967 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[222878]: pgmap v506: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 27232/268221 objects degraded (10.153%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[222878]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:13:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[222878]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T15:13:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[222878]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[215646]: pgmap v506: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 27232/268221 objects degraded (10.153%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[215646]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:13:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[215646]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T15:13:41.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:41 smithi017 ceph-mon[215646]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:41 smithi027 ceph-mon[195233]: pgmap v506: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 27232/268221 objects degraded (10.153%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:41 smithi027 ceph-mon[195233]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:13:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:41 smithi027 ceph-mon[195233]: osdmap e199: 8 total, 8 up, 7 in 2023-09-23T15:13:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:41 smithi027 ceph-mon[195233]: failed to encode map e199 with expected crc 2023-09-23T15:13:41.940 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:41.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:41.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:42.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:42.076 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:42.123 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:42.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:42.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:42.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:42.302 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:42.365 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:13:42.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:13:42.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27232/268221 objects degraded (10.153%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:42.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27232/268221 objects degraded (10.153%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:42.821 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:42.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:42.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:42.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27232/268221 objects degraded (10.153%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:13:42.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:43.032 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:43.174 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:13:43.174 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:13:43.174 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:13:43.174 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:13:43.174 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:13:43.258 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:43.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:43.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:43.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:43.405 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:43.470 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:43.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:43.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:43.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:43.678 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:43.819 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:43.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:43.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:43.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:43.967 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:44.030 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:44.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:44.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:44.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:44.167 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:43 smithi027 ceph-mon[195233]: pgmap v508: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 27232/268221 objects degraded (10.153%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:13:44.167 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:44.172 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:44.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:43 smithi017 ceph-mon[215646]: pgmap v508: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 27232/268221 objects degraded (10.153%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:13:44.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:43 smithi017 ceph-mon[222878]: pgmap v508: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 27232/268221 objects degraded (10.153%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:13:44.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:44.222 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:44.232 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:44.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:44.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:44.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:13:44.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:44.433 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:44.623 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:44.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:44.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:44.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:44.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:13:44.808 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:44.934 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:13:44.990 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:45.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:45.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:45.135 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:45.184 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:45.185 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:45.190 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:45.192 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:13:45.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:45.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:45.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:45.329 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:45.387 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:45.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:45.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:45.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:45.530 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:45.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: pgmap v509: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27232/265581 objects degraded (10.254%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:45.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:45.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: failed to encode map e200 with expected crc 2023-09-23T15:13:45.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:45.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T15:13:45.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: failed to encode map e200 with expected crc 2023-09-23T15:13:45.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: failed to encode map e200 with expected crc 2023-09-23T15:13:45.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[222878]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.159 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:46.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:46.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:46.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: pgmap v509: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27232/265581 objects degraded (10.254%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:46.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:46.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.171 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T15:13:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.172 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:45 smithi017 ceph-mon[215646]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: pgmap v509: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 578 op/s; 27232/265581 objects degraded (10.254%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T15:13:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3337689412' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: osdmap e200: 8 total, 8 up, 7 in 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:45 smithi027 ceph-mon[195233]: failed to encode map e200 with expected crc 2023-09-23T15:13:46.400 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:46.450 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:46.532 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:46.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:46.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:46.550 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:46.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:46.687 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:46.757 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:46.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:46.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:46.904 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:46.983 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:46.989 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:46 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27232/265581 objects degraded (10.254%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:46.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:46.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:47.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:46 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27232/265581 objects degraded (10.254%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:47.089 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:46 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27232/265581 objects degraded (10.254%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:47.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:47.126 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:47.182 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:47.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:47.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:47.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:47.322 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:47.381 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:47.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:47.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:47.488 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:47.523 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:47.556 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:13:47.558 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:13:47.582 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:47.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:47.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:47.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:47.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:48.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:47 smithi017 ceph-mon[215646]: pgmap v511: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 26936/259887 objects degraded (10.365%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:13:48.197 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:48] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.43.0" 2023-09-23T15:13:48.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:47 smithi017 ceph-mon[222878]: pgmap v511: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 26936/259887 objects degraded (10.365%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:13:48.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:13:48.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:47 smithi027 ceph-mon[195233]: pgmap v511: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 706 op/s; 26936/259887 objects degraded (10.365%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:13:49.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:48 smithi017 ceph-mon[222878]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:49.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:48 smithi017 ceph-mon[215646]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:49.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:48 smithi027 ceph-mon[195233]: from='client.55111 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:49.467 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:49.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:49.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:49.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:13:49.622 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":511,"stamp":"2023-09-23T15:13:46.321021+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5680240860,"num_objects":86629,"num_object_clones":0,"num_object_copies":259887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26936,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86629,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":115204,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2826,"num_bytes_recovered":186685607,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108689,"ondisk_log_size":108689,"up":327,"acting":319,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17320820,"kb_used_data":15930492,"kb_used_omap":30,"kb_used_meta":1390241,"kb_avail":638895244,"statfs":{"total":671965249536,"available":654228729856,"internally_reserved":0,"allocated":16312823808,"data_stored":16307398550,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1423607384},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,7,4,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-370737152,"num_objects":-5657,"num_object_clones":0,"num_object_copies":-16971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5657,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":551,"num_bytes_recovered":36110336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003399"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":193,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885390+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:13:32.885390+0000","last_peered":"2023-09-23T15:13:32.885390+0000","last_clean":"2023-09-23T15:13:32.885390+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:13:32.885390+0000","last_undegraded":"2023-09-23T15:13:32.885390+0000","last_fullsized":"2023-09-23T15:13:32.885390+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":127,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455380+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:13:40.455380+0000","last_peered":"2023-09-23T15:13:40.455380+0000","last_clean":"2023-09-23T15:13:40.455380+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:13:40.455380+0000","last_undegraded":"2023-09-23T15:13:40.455380+0000","last_fullsized":"2023-09-23T15:13:40.455380+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456881+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:13:40.456881+0000","last_peered":"2023-09-23T15:13:40.456881+0000","last_clean":"2023-09-23T15:13:40.456881+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:13:40.456881+0000","last_undegraded":"2023-09-23T15:13:40.456881+0000","last_fullsized":"2023-09-23T15:13:40.456881+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":163,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908602+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:13:32.908602+0000","last_peered":"2023-09-23T15:13:32.908602+0000","last_clean":"2023-09-23T15:13:32.908602+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:13:32.908602+0000","last_undegraded":"2023-09-23T15:13:32.908602+0000","last_fullsized":"2023-09-23T15:13:32.908602+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":196,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.916078+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:13:32.916078+0000","last_peered":"2023-09-23T15:13:32.916078+0000","last_clean":"2023-09-23T15:13:32.916078+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:13:32.916078+0000","last_undegraded":"2023-09-23T15:13:32.916078+0000","last_fullsized":"2023-09-23T15:13:32.916078+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":173,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.904349+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:13:32.904349+0000","last_peered":"2023-09-23T15:13:32.904349+0000","last_clean":"2023-09-23T15:13:32.904349+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:13:32.904349+0000","last_undegraded":"2023-09-23T15:13:32.904349+0000","last_fullsized":"2023-09-23T15:13:32.904349+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":142,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.909191+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:13:32.909191+0000","last_peered":"2023-09-23T15:13:32.909191+0000","last_clean":"2023-09-23T15:13:32.909191+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:13:32.909191+0000","last_undegraded":"2023-09-23T15:13:32.909191+0000","last_fullsized":"2023-09-23T15:13:32.909191+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":178,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.916146+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:13:32.916146+0000","last_peered":"2023-09-23T15:13:32.916146+0000","last_clean":"2023-09-23T15:13:32.916146+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:13:32.916146+0000","last_undegraded":"2023-09-23T15:13:32.916146+0000","last_fullsized":"2023-09-23T15:13:32.916146+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908106+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:13:32.908106+0000","last_peered":"2023-09-23T15:13:32.908106+0000","last_clean":"2023-09-23T15:13:32.908106+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:13:32.908106+0000","last_undegraded":"2023-09-23T15:13:32.908106+0000","last_fullsized":"2023-09-23T15:13:32.908106+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":189,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895355+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:13:32.895355+0000","last_peered":"2023-09-23T15:13:32.895355+0000","last_clean":"2023-09-23T15:13:32.895355+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:13:32.895355+0000","last_undegraded":"2023-09-23T15:13:32.895355+0000","last_fullsized":"2023-09-23T15:13:32.895355+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":121,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.917138+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:13:32.917138+0000","last_peered":"2023-09-23T15:13:32.917138+0000","last_clean":"2023-09-23T15:13:32.917138+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:13:32.917138+0000","last_undegraded":"2023-09-23T15:13:32.917138+0000","last_fullsized":"2023-09-23T15:13:32.917138+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":60,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.915190+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:13:32.915190+0000","last_peered":"2023-09-23T15:13:32.915190+0000","last_clean":"2023-09-23T15:13:32.915190+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:13:32.915190+0000","last_undegraded":"2023-09-23T15:13:32.915190+0000","last_fullsized":"2023-09-23T15:13:32.915190+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456824+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:13:40.456824+0000","last_peered":"2023-09-23T15:13:40.456824+0000","last_clean":"2023-09-23T15:13:40.456824+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:13:40.456824+0000","last_undegraded":"2023-09-23T15:13:40.456824+0000","last_fullsized":"2023-09-23T15:13:40.456824+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":183,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885450+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:13:32.885450+0000","last_peered":"2023-09-23T15:13:32.885450+0000","last_clean":"2023-09-23T15:13:32.885450+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:13:32.885450+0000","last_undegraded":"2023-09-23T15:13:32.885450+0000","last_fullsized":"2023-09-23T15:13:32.885450+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885297+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:13:32.885297+0000","last_peered":"2023-09-23T15:13:32.885297+0000","last_clean":"2023-09-23T15:13:32.885297+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:13:32.885297+0000","last_undegraded":"2023-09-23T15:13:32.885297+0000","last_fullsized":"2023-09-23T15:13:32.885297+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":486,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.906967+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:13:32.906967+0000","last_peered":"2023-09-23T15:13:32.906967+0000","last_clean":"2023-09-23T15:13:32.906967+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:13:32.906967+0000","last_undegraded":"2023-09-23T15:13:32.906967+0000","last_fullsized":"2023-09-23T15:13:32.906967+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":61,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885388+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:13:32.885388+0000","last_peered":"2023-09-23T15:13:32.885388+0000","last_clean":"2023-09-23T15:13:32.885388+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:13:32.885388+0000","last_undegraded":"2023-09-23T15:13:32.885388+0000","last_fullsized":"2023-09-23T15:13:32.885388+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":189,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.919850+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:13:32.919850+0000","last_peered":"2023-09-23T15:13:32.919850+0000","last_clean":"2023-09-23T15:13:32.919850+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:13:32.919850+0000","last_undegraded":"2023-09-23T15:13:32.919850+0000","last_fullsized":"2023-09-23T15:13:32.919850+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":486,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.906008+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:13:32.906008+0000","last_peered":"2023-09-23T15:13:32.906008+0000","last_clean":"2023-09-23T15:13:32.906008+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:13:32.906008+0000","last_undegraded":"2023-09-23T15:13:32.906008+0000","last_fullsized":"2023-09-23T15:13:32.906008+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":162,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.909544+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:13:32.909544+0000","last_peered":"2023-09-23T15:13:32.909544+0000","last_clean":"2023-09-23T15:13:32.909544+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:13:32.909544+0000","last_undegraded":"2023-09-23T15:13:32.909544+0000","last_fullsized":"2023-09-23T15:13:32.909544+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":167,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.896385+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:13:32.896385+0000","last_peered":"2023-09-23T15:13:32.896385+0000","last_clean":"2023-09-23T15:13:32.896385+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:13:32.896385+0000","last_undegraded":"2023-09-23T15:13:32.896385+0000","last_fullsized":"2023-09-23T15:13:32.896385+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.906920+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:13:32.906920+0000","last_peered":"2023-09-23T15:13:32.906920+0000","last_clean":"2023-09-23T15:13:32.906920+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:13:32.906920+0000","last_undegraded":"2023-09-23T15:13:32.906920+0000","last_fullsized":"2023-09-23T15:13:32.906920+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885275+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:13:32.885275+0000","last_peered":"2023-09-23T15:13:32.885275+0000","last_clean":"2023-09-23T15:13:32.885275+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:13:32.885275+0000","last_undegraded":"2023-09-23T15:13:32.885275+0000","last_fullsized":"2023-09-23T15:13:32.885275+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":151,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908679+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:13:32.908679+0000","last_peered":"2023-09-23T15:13:32.908679+0000","last_clean":"2023-09-23T15:13:32.908679+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:13:32.908679+0000","last_undegraded":"2023-09-23T15:13:32.908679+0000","last_fullsized":"2023-09-23T15:13:32.908679+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":486,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.909832+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:13:32.909832+0000","last_peered":"2023-09-23T15:13:32.909832+0000","last_clean":"2023-09-23T15:13:32.909832+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:13:32.909832+0000","last_undegraded":"2023-09-23T15:13:32.909832+0000","last_fullsized":"2023-09-23T15:13:32.909832+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":165,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.903163+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:13:32.903163+0000","last_peered":"2023-09-23T15:13:32.903163+0000","last_clean":"2023-09-23T15:13:32.903163+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:13:32.903163+0000","last_undegraded":"2023-09-23T15:13:32.903163+0000","last_fullsized":"2023-09-23T15:13:32.903163+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":189,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.917923+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:13:32.917923+0000","last_peered":"2023-09-23T15:13:32.917923+0000","last_clean":"2023-09-23T15:13:32.917923+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:13:32.917923+0000","last_undegraded":"2023-09-23T15:13:32.917923+0000","last_fullsized":"2023-09-23T15:13:32.917923+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"199'7063","reported_seq":7099,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.634315+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:13:44.634315+0000","last_peered":"2023-09-23T15:13:44.634315+0000","last_clean":"2023-09-23T15:13:44.634315+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:13:44.634315+0000","last_undegraded":"2023-09-23T15:13:44.634315+0000","last_fullsized":"2023-09-23T15:13:44.634315+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7063,"log_dups_size":0,"ondisk_log_size":7063,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349372416,"num_objects":5331,"num_object_clones":0,"num_object_copies":15993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7063,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.910950+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:13:32.910950+0000","last_peered":"2023-09-23T15:13:32.910950+0000","last_clean":"2023-09-23T15:13:32.910950+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:13:32.910950+0000","last_undegraded":"2023-09-23T15:13:32.910950+0000","last_fullsized":"2023-09-23T15:13:32.910950+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":73,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885868+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:13:32.885868+0000","last_peered":"2023-09-23T15:13:32.885868+0000","last_clean":"2023-09-23T15:13:32.885868+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:13:32.885868+0000","last_undegraded":"2023-09-23T15:13:32.885868+0000","last_fullsized":"2023-09-23T15:13:32.885868+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":150,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895038+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:13:32.895038+0000","last_peered":"2023-09-23T15:13:32.895038+0000","last_clean":"2023-09-23T15:13:32.895038+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:13:32.895038+0000","last_undegraded":"2023-09-23T15:13:32.895038+0000","last_fullsized":"2023-09-23T15:13:32.895038+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"199'7208","reported_seq":7253,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.303568+0000","last_change":"2023-09-23T15:12:43.308627+0000","last_active":"2023-09-23T15:13:44.303568+0000","last_peered":"2023-09-23T15:13:44.303568+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:13:44.303568+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"175'800","ondisk_log_start":"175'800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6408,"log_dups_size":0,"ondisk_log_size":6408,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357695488,"num_objects":5458,"num_object_clones":0,"num_object_copies":16374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7208,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":461}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":465,"reported_epoch":198,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:32.907827+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:13:32.907827+0000","last_peered":"2023-09-23T15:13:32.907827+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:13:32.907827+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":73,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.887812+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:13:32.887812+0000","last_peered":"2023-09-23T15:13:32.887812+0000","last_clean":"2023-09-23T15:13:32.887812+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:13:32.887812+0000","last_undegraded":"2023-09-23T15:13:32.887812+0000","last_fullsized":"2023-09-23T15:13:32.887812+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":191,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885120+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:13:32.885120+0000","last_peered":"2023-09-23T15:13:32.885120+0000","last_clean":"2023-09-23T15:13:32.885120+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:13:32.885120+0000","last_undegraded":"2023-09-23T15:13:32.885120+0000","last_fullsized":"2023-09-23T15:13:32.885120+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"199'7024","reported_seq":7061,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:43.465988+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:13:43.465988+0000","last_peered":"2023-09-23T15:13:43.465988+0000","last_clean":"2023-09-23T15:13:43.465988+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:13:43.465988+0000","last_undegraded":"2023-09-23T15:13:43.465988+0000","last_fullsized":"2023-09-23T15:13:43.465988+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7024,"log_dups_size":0,"ondisk_log_size":7024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355074048,"num_objects":5418,"num_object_clones":0,"num_object_copies":16254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7024,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908887+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:13:32.908887+0000","last_peered":"2023-09-23T15:13:32.908887+0000","last_clean":"2023-09-23T15:13:32.908887+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:13:32.908887+0000","last_undegraded":"2023-09-23T15:13:32.908887+0000","last_fullsized":"2023-09-23T15:13:32.908887+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":59,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.896343+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:13:32.896343+0000","last_peered":"2023-09-23T15:13:32.896343+0000","last_clean":"2023-09-23T15:13:32.896343+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:13:32.896343+0000","last_undegraded":"2023-09-23T15:13:32.896343+0000","last_fullsized":"2023-09-23T15:13:32.896343+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.903594+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:13:32.903594+0000","last_peered":"2023-09-23T15:13:32.903594+0000","last_clean":"2023-09-23T15:13:32.903594+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:13:32.903594+0000","last_undegraded":"2023-09-23T15:13:32.903594+0000","last_fullsized":"2023-09-23T15:13:32.903594+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"199'7181","reported_seq":7218,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.333480+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:13:44.333480+0000","last_peered":"2023-09-23T15:13:44.333480+0000","last_clean":"2023-09-23T15:13:44.333480+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:13:44.333480+0000","last_undegraded":"2023-09-23T15:13:44.333480+0000","last_fullsized":"2023-09-23T15:13:44.333480+0000","mapping_epoch":171,"log_start":"175'1600","ondisk_log_start":"175'1600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5581,"log_dups_size":0,"ondisk_log_size":5581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359333888,"num_objects":5483,"num_object_clones":0,"num_object_copies":16449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7181,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908327+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:13:32.908327+0000","last_peered":"2023-09-23T15:13:32.908327+0000","last_clean":"2023-09-23T15:13:32.908327+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:13:32.908327+0000","last_undegraded":"2023-09-23T15:13:32.908327+0000","last_fullsized":"2023-09-23T15:13:32.908327+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":158,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895289+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:32.895289+0000","last_peered":"2023-09-23T15:13:32.895289+0000","last_clean":"2023-09-23T15:13:32.895289+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:13:32.895289+0000","last_undegraded":"2023-09-23T15:13:32.895289+0000","last_fullsized":"2023-09-23T15:13:32.895289+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":158,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379165+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:13:39.379165+0000","last_peered":"2023-09-23T15:13:39.379165+0000","last_clean":"2023-09-23T15:13:39.379165+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:13:39.379165+0000","last_undegraded":"2023-09-23T15:13:39.379165+0000","last_fullsized":"2023-09-23T15:13:39.379165+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"199'7341","reported_seq":7383,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.243389+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:13:44.243389+0000","last_peered":"2023-09-23T15:13:44.243389+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:13:44.243389+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7341,"log_dups_size":0,"ondisk_log_size":7341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":365101056,"num_objects":5571,"num_object_clones":0,"num_object_copies":16713,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5571,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7341,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":561}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":487,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442201+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:13:40.442201+0000","last_peered":"2023-09-23T15:13:40.442201+0000","last_clean":"2023-09-23T15:13:40.442201+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:13:40.442201+0000","last_undegraded":"2023-09-23T15:13:40.442201+0000","last_fullsized":"2023-09-23T15:13:40.442201+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":161,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.919670+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:13:32.919670+0000","last_peered":"2023-09-23T15:13:32.919670+0000","last_clean":"2023-09-23T15:13:32.919670+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:13:32.919670+0000","last_undegraded":"2023-09-23T15:13:32.919670+0000","last_fullsized":"2023-09-23T15:13:32.919670+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":157,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885533+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:13:32.885533+0000","last_peered":"2023-09-23T15:13:32.885533+0000","last_clean":"2023-09-23T15:13:32.885533+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:13:32.885533+0000","last_undegraded":"2023-09-23T15:13:32.885533+0000","last_fullsized":"2023-09-23T15:13:32.885533+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"199'7150","reported_seq":7187,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:43.457940+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:13:43.457940+0000","last_peered":"2023-09-23T15:13:43.457940+0000","last_clean":"2023-09-23T15:13:43.457940+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:13:43.457940+0000","last_undegraded":"2023-09-23T15:13:43.457940+0000","last_fullsized":"2023-09-23T15:13:43.457940+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7150,"log_dups_size":0,"ondisk_log_size":7150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355729408,"num_objects":5428,"num_object_clones":0,"num_object_copies":16284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7150,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":492,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.441832+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:13:40.441832+0000","last_peered":"2023-09-23T15:13:40.441832+0000","last_clean":"2023-09-23T15:13:40.441832+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:13:40.441832+0000","last_undegraded":"2023-09-23T15:13:40.441832+0000","last_fullsized":"2023-09-23T15:13:40.441832+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.903461+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:13:32.903461+0000","last_peered":"2023-09-23T15:13:32.903461+0000","last_clean":"2023-09-23T15:13:32.903461+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:13:32.903461+0000","last_undegraded":"2023-09-23T15:13:32.903461+0000","last_fullsized":"2023-09-23T15:13:32.903461+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":191,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885577+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:13:32.885577+0000","last_peered":"2023-09-23T15:13:32.885577+0000","last_clean":"2023-09-23T15:13:32.885577+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:13:32.885577+0000","last_undegraded":"2023-09-23T15:13:32.885577+0000","last_fullsized":"2023-09-23T15:13:32.885577+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"199'7134","reported_seq":7171,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.333424+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:13:44.333424+0000","last_peered":"2023-09-23T15:13:44.333424+0000","last_clean":"2023-09-23T15:13:44.333424+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:13:44.333424+0000","last_undegraded":"2023-09-23T15:13:44.333424+0000","last_fullsized":"2023-09-23T15:13:44.333424+0000","mapping_epoch":171,"log_start":"175'700","ondisk_log_start":"175'700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6434,"log_dups_size":0,"ondisk_log_size":6434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356777984,"num_objects":5444,"num_object_clones":0,"num_object_copies":16332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7134,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":474,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.907012+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:13:32.907012+0000","last_peered":"2023-09-23T15:13:32.907012+0000","last_clean":"2023-09-23T15:13:32.907012+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:13:32.907012+0000","last_undegraded":"2023-09-23T15:13:32.907012+0000","last_fullsized":"2023-09-23T15:13:32.907012+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":182,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.884884+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:13:32.884884+0000","last_peered":"2023-09-23T15:13:32.884884+0000","last_clean":"2023-09-23T15:13:32.884884+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:13:32.884884+0000","last_undegraded":"2023-09-23T15:13:32.884884+0000","last_fullsized":"2023-09-23T15:13:32.884884+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.887859+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:13:32.887859+0000","last_peered":"2023-09-23T15:13:32.887859+0000","last_clean":"2023-09-23T15:13:32.887859+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:13:32.887859+0000","last_undegraded":"2023-09-23T15:13:32.887859+0000","last_fullsized":"2023-09-23T15:13:32.887859+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"199'7129","reported_seq":12707,"reported_epoch":199,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.693646+0000","last_change":"2023-09-23T15:12:43.243089+0000","last_active":"2023-09-23T15:13:44.693646+0000","last_peered":"2023-09-23T15:13:44.693646+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.190605+0000","last_became_peered":"2023-09-23T15:12:43.190605+0000","last_unstale":"2023-09-23T15:13:44.693646+0000","last_undegraded":"2023-09-23T15:12:42.959077+0000","last_fullsized":"2023-09-23T15:12:42.845806+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7129,"log_dups_size":0,"ondisk_log_size":7129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:41:00.861011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354025496,"num_objects":5403,"num_object_clones":0,"num_object_copies":16209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2134,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5403,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7129,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2766,"num_bytes_recovered":181272576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":3269}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":487,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.443301+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:13:40.443301+0000","last_peered":"2023-09-23T15:13:40.443301+0000","last_clean":"2023-09-23T15:13:40.443301+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:13:40.443301+0000","last_undegraded":"2023-09-23T15:13:40.443301+0000","last_fullsized":"2023-09-23T15:13:40.443301+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":190,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.916005+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:13:32.916005+0000","last_peered":"2023-09-23T15:13:32.916005+0000","last_clean":"2023-09-23T15:13:32.916005+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:13:32.916005+0000","last_undegraded":"2023-09-23T15:13:32.916005+0000","last_fullsized":"2023-09-23T15:13:32.916005+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.462640+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:13:40.462640+0000","last_peered":"2023-09-23T15:13:40.462640+0000","last_clean":"2023-09-23T15:13:40.462640+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:13:40.462640+0000","last_undegraded":"2023-09-23T15:13:40.462640+0000","last_fullsized":"2023-09-23T15:13:40.462640+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"199'7162","reported_seq":7199,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.311904+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:13:44.311904+0000","last_peered":"2023-09-23T15:13:44.311904+0000","last_clean":"2023-09-23T15:13:44.311904+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:13:44.311904+0000","last_undegraded":"2023-09-23T15:13:44.311904+0000","last_fullsized":"2023-09-23T15:13:44.311904+0000","mapping_epoch":171,"log_start":"175'1600","ondisk_log_start":"175'1600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354287616,"num_objects":5406,"num_object_clones":0,"num_object_copies":16218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7162,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":475,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442869+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:13:40.442869+0000","last_peered":"2023-09-23T15:13:40.442869+0000","last_clean":"2023-09-23T15:13:40.442869+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:13:40.442869+0000","last_undegraded":"2023-09-23T15:13:40.442869+0000","last_fullsized":"2023-09-23T15:13:40.442869+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":375,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.459249+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:13:40.459249+0000","last_peered":"2023-09-23T15:13:40.459249+0000","last_clean":"2023-09-23T15:13:40.459249+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:13:40.459249+0000","last_undegraded":"2023-09-23T15:13:40.459249+0000","last_fullsized":"2023-09-23T15:13:40.459249+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":175,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.896294+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:32.896294+0000","last_peered":"2023-09-23T15:13:32.896294+0000","last_clean":"2023-09-23T15:13:32.896294+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:13:32.896294+0000","last_undegraded":"2023-09-23T15:13:32.896294+0000","last_fullsized":"2023-09-23T15:13:32.896294+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":119,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.893054+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:13:32.893054+0000","last_peered":"2023-09-23T15:13:32.893054+0000","last_clean":"2023-09-23T15:13:32.893054+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:13:32.893054+0000","last_undegraded":"2023-09-23T15:13:32.893054+0000","last_fullsized":"2023-09-23T15:13:32.893054+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"199'7049","reported_seq":7086,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.306095+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:13:44.306095+0000","last_peered":"2023-09-23T15:13:44.306095+0000","last_clean":"2023-09-23T15:13:44.306095+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:13:44.306095+0000","last_undegraded":"2023-09-23T15:13:44.306095+0000","last_fullsized":"2023-09-23T15:13:44.306095+0000","mapping_epoch":171,"log_start":"175'600","ondisk_log_start":"175'600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6449,"log_dups_size":0,"ondisk_log_size":6449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347668480,"num_objects":5305,"num_object_clones":0,"num_object_copies":15915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7049,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":475,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379033+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:13:39.379033+0000","last_peered":"2023-09-23T15:13:39.379033+0000","last_clean":"2023-09-23T15:13:39.379033+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:13:39.379033+0000","last_undegraded":"2023-09-23T15:13:39.379033+0000","last_fullsized":"2023-09-23T15:13:39.379033+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":717,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895335+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:13:32.895335+0000","last_peered":"2023-09-23T15:13:32.895335+0000","last_clean":"2023-09-23T15:13:32.895335+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:13:32.895335+0000","last_undegraded":"2023-09-23T15:13:32.895335+0000","last_fullsized":"2023-09-23T15:13:32.895335+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":158,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885218+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:13:32.885218+0000","last_peered":"2023-09-23T15:13:32.885218+0000","last_clean":"2023-09-23T15:13:32.885218+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:13:32.885218+0000","last_undegraded":"2023-09-23T15:13:32.885218+0000","last_fullsized":"2023-09-23T15:13:32.885218+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.458893+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:13:40.458893+0000","last_peered":"2023-09-23T15:13:40.458893+0000","last_clean":"2023-09-23T15:13:40.458893+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:13:40.458893+0000","last_undegraded":"2023-09-23T15:13:40.458893+0000","last_fullsized":"2023-09-23T15:13:40.458893+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"199'7196","reported_seq":7232,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.114100+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:13:44.114100+0000","last_peered":"2023-09-23T15:13:44.114100+0000","last_clean":"2023-09-23T15:13:44.114100+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:13:44.114100+0000","last_undegraded":"2023-09-23T15:13:44.114100+0000","last_fullsized":"2023-09-23T15:13:44.114100+0000","mapping_epoch":171,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5296,"log_dups_size":0,"ondisk_log_size":5296,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355860480,"num_objects":5430,"num_object_clones":0,"num_object_copies":16290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7196,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":487,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379073+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:13:39.379073+0000","last_peered":"2023-09-23T15:13:39.379073+0000","last_clean":"2023-09-23T15:13:39.379073+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:13:39.379073+0000","last_undegraded":"2023-09-23T15:13:39.379073+0000","last_fullsized":"2023-09-23T15:13:39.379073+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":810,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908020+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:13:32.908020+0000","last_peered":"2023-09-23T15:13:32.908020+0000","last_clean":"2023-09-23T15:13:32.908020+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:13:32.908020+0000","last_undegraded":"2023-09-23T15:13:32.908020+0000","last_fullsized":"2023-09-23T15:13:32.908020+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456488+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:13:40.456488+0000","last_peered":"2023-09-23T15:13:40.456488+0000","last_clean":"2023-09-23T15:13:40.456488+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:13:40.456488+0000","last_undegraded":"2023-09-23T15:13:40.456488+0000","last_fullsized":"2023-09-23T15:13:40.456488+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.462560+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:13:40.462560+0000","last_peered":"2023-09-23T15:13:40.462560+0000","last_clean":"2023-09-23T15:13:40.462560+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:13:40.462560+0000","last_undegraded":"2023-09-23T15:13:40.462560+0000","last_fullsized":"2023-09-23T15:13:40.462560+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"199'7068","reported_seq":7104,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.692058+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:13:44.692058+0000","last_peered":"2023-09-23T15:13:44.692058+0000","last_clean":"2023-09-23T15:13:44.692058+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:13:44.692058+0000","last_undegraded":"2023-09-23T15:13:44.692058+0000","last_fullsized":"2023-09-23T15:13:44.692058+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348127232,"num_objects":5312,"num_object_clones":0,"num_object_copies":15936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7068,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":486,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895260+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:13:32.895260+0000","last_peered":"2023-09-23T15:13:32.895260+0000","last_clean":"2023-09-23T15:13:32.895260+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:13:32.895260+0000","last_undegraded":"2023-09-23T15:13:32.895260+0000","last_fullsized":"2023-09-23T15:13:32.895260+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":487,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:34.376221+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:13:34.376221+0000","last_peered":"2023-09-23T15:13:34.376221+0000","last_clean":"2023-09-23T15:13:34.376221+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:13:34.376221+0000","last_undegraded":"2023-09-23T15:13:34.376221+0000","last_fullsized":"2023-09-23T15:13:34.376221+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":112,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.915524+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:13:32.915524+0000","last_peered":"2023-09-23T15:13:32.915524+0000","last_clean":"2023-09-23T15:13:32.915524+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:13:32.915524+0000","last_undegraded":"2023-09-23T15:13:32.915524+0000","last_fullsized":"2023-09-23T15:13:32.915524+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":149,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.904300+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:13:32.904300+0000","last_peered":"2023-09-23T15:13:32.904300+0000","last_clean":"2023-09-23T15:13:32.904300+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:13:32.904300+0000","last_undegraded":"2023-09-23T15:13:32.904300+0000","last_fullsized":"2023-09-23T15:13:32.904300+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"199'7194","reported_seq":7231,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.308858+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:13:44.308858+0000","last_peered":"2023-09-23T15:13:44.308858+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:13:44.308858+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"175'800","ondisk_log_start":"175'800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357171200,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7194,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":498}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":475,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379141+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:13:39.379141+0000","last_peered":"2023-09-23T15:13:39.379141+0000","last_clean":"2023-09-23T15:13:39.379141+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:13:39.379141+0000","last_undegraded":"2023-09-23T15:13:39.379141+0000","last_fullsized":"2023-09-23T15:13:39.379141+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":282,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456149+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:13:40.456149+0000","last_peered":"2023-09-23T15:13:40.456149+0000","last_clean":"2023-09-23T15:13:40.456149+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:13:40.456149+0000","last_undegraded":"2023-09-23T15:13:40.456149+0000","last_fullsized":"2023-09-23T15:13:40.456149+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":143,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.906891+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:13:32.906891+0000","last_peered":"2023-09-23T15:13:32.906891+0000","last_clean":"2023-09-23T15:13:32.906891+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:13:32.906891+0000","last_undegraded":"2023-09-23T15:13:32.906891+0000","last_fullsized":"2023-09-23T15:13:32.906891+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":173,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908424+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:13:32.908424+0000","last_peered":"2023-09-23T15:13:32.908424+0000","last_clean":"2023-09-23T15:13:32.908424+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:13:32.908424+0000","last_undegraded":"2023-09-23T15:13:32.908424+0000","last_fullsized":"2023-09-23T15:13:32.908424+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"199'7080","reported_seq":7116,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:43.480264+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:13:43.480264+0000","last_peered":"2023-09-23T15:13:43.480264+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:13:43.480264+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7080,"log_dups_size":0,"ondisk_log_size":7080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355729408,"num_objects":5428,"num_object_clones":0,"num_object_copies":16284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7080,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":554}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":475,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442908+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:13:40.442908+0000","last_peered":"2023-09-23T15:13:40.442908+0000","last_clean":"2023-09-23T15:13:40.442908+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:13:40.442908+0000","last_undegraded":"2023-09-23T15:13:40.442908+0000","last_fullsized":"2023-09-23T15:13:40.442908+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":804,"reported_epoch":198,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:39.379266+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:13:39.379266+0000","last_peered":"2023-09-23T15:13:39.379266+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:13:39.379266+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.459125+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:13:40.459125+0000","last_peered":"2023-09-23T15:13:40.459125+0000","last_clean":"2023-09-23T15:13:40.459125+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:13:40.459125+0000","last_undegraded":"2023-09-23T15:13:40.459125+0000","last_fullsized":"2023-09-23T15:13:40.459125+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":148,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.907420+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:13:32.907420+0000","last_peered":"2023-09-23T15:13:32.907420+0000","last_clean":"2023-09-23T15:13:32.907420+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:13:32.907420+0000","last_undegraded":"2023-09-23T15:13:32.907420+0000","last_fullsized":"2023-09-23T15:13:32.907420+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"199'6967","reported_seq":7003,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.283130+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:13:44.283130+0000","last_peered":"2023-09-23T15:13:44.283130+0000","last_clean":"2023-09-23T15:13:44.283130+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:13:44.283130+0000","last_undegraded":"2023-09-23T15:13:44.283130+0000","last_fullsized":"2023-09-23T15:13:44.283130+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6967,"log_dups_size":0,"ondisk_log_size":6967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349634560,"num_objects":5335,"num_object_clones":0,"num_object_copies":16005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6967,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":475,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379367+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:13:39.379367+0000","last_peered":"2023-09-23T15:13:39.379367+0000","last_clean":"2023-09-23T15:13:39.379367+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:13:39.379367+0000","last_undegraded":"2023-09-23T15:13:39.379367+0000","last_fullsized":"2023-09-23T15:13:39.379367+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.917849+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:13:32.917849+0000","last_peered":"2023-09-23T15:13:32.917849+0000","last_clean":"2023-09-23T15:13:32.917849+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:13:32.917849+0000","last_undegraded":"2023-09-23T15:13:32.917849+0000","last_fullsized":"2023-09-23T15:13:32.917849+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":659,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.469415+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:13:40.469415+0000","last_peered":"2023-09-23T15:13:40.469415+0000","last_clean":"2023-09-23T15:13:40.469415+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:13:40.469415+0000","last_undegraded":"2023-09-23T15:13:40.469415+0000","last_fullsized":"2023-09-23T15:13:40.469415+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":178,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.916217+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:13:32.916217+0000","last_peered":"2023-09-23T15:13:32.916217+0000","last_clean":"2023-09-23T15:13:32.916217+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:13:32.916217+0000","last_undegraded":"2023-09-23T15:13:32.916217+0000","last_fullsized":"2023-09-23T15:13:32.916217+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"199'7230","reported_seq":7272,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.144554+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:13:44.144554+0000","last_peered":"2023-09-23T15:13:44.144554+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:13:44.144554+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353894400,"num_objects":5400,"num_object_clones":0,"num_object_copies":16200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7230,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":434}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":475,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:39.379228+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:13:39.379228+0000","last_peered":"2023-09-23T15:13:39.379228+0000","last_clean":"2023-09-23T15:13:39.379228+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:13:39.379228+0000","last_undegraded":"2023-09-23T15:13:39.379228+0000","last_fullsized":"2023-09-23T15:13:39.379228+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":675,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.904254+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:13:32.904254+0000","last_peered":"2023-09-23T15:13:32.904254+0000","last_clean":"2023-09-23T15:13:32.904254+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:13:32.904254+0000","last_undegraded":"2023-09-23T15:13:32.904254+0000","last_fullsized":"2023-09-23T15:13:32.904254+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":161,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.908518+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:13:32.908518+0000","last_peered":"2023-09-23T15:13:32.908518+0000","last_clean":"2023-09-23T15:13:32.908518+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:13:32.908518+0000","last_undegraded":"2023-09-23T15:13:32.908518+0000","last_fullsized":"2023-09-23T15:13:32.908518+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":140,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885040+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:13:32.885040+0000","last_peered":"2023-09-23T15:13:32.885040+0000","last_clean":"2023-09-23T15:13:32.885040+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:13:32.885040+0000","last_undegraded":"2023-09-23T15:13:32.885040+0000","last_fullsized":"2023-09-23T15:13:32.885040+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":150,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.907332+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:13:32.907332+0000","last_peered":"2023-09-23T15:13:32.907332+0000","last_clean":"2023-09-23T15:13:32.907332+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:13:32.907332+0000","last_undegraded":"2023-09-23T15:13:32.907332+0000","last_fullsized":"2023-09-23T15:13:32.907332+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":198,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895503+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:13:32.895503+0000","last_peered":"2023-09-23T15:13:32.895503+0000","last_clean":"2023-09-23T15:13:32.895503+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:13:32.895503+0000","last_undegraded":"2023-09-23T15:13:32.895503+0000","last_fullsized":"2023-09-23T15:13:32.895503+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":177,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.895445+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:13:32.895445+0000","last_peered":"2023-09-23T15:13:32.895445+0000","last_clean":"2023-09-23T15:13:32.895445+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:13:32.895445+0000","last_undegraded":"2023-09-23T15:13:32.895445+0000","last_fullsized":"2023-09-23T15:13:32.895445+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":191,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885450+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:13:32.885450+0000","last_peered":"2023-09-23T15:13:32.885450+0000","last_clean":"2023-09-23T15:13:32.885450+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:13:32.885450+0000","last_undegraded":"2023-09-23T15:13:32.885450+0000","last_fullsized":"2023-09-23T15:13:32.885450+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":143,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.896243+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:13:32.896243+0000","last_peered":"2023-09-23T15:13:32.896243+0000","last_clean":"2023-09-23T15:13:32.896243+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:13:32.896243+0000","last_undegraded":"2023-09-23T15:13:32.896243+0000","last_fullsized":"2023-09-23T15:13:32.896243+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":174,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.885215+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:13:32.885215+0000","last_peered":"2023-09-23T15:13:32.885215+0000","last_clean":"2023-09-23T15:13:32.885215+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:13:32.885215+0000","last_undegraded":"2023-09-23T15:13:32.885215+0000","last_fullsized":"2023-09-23T15:13:32.885215+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":169,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.917816+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:13:32.917816+0000","last_peered":"2023-09-23T15:13:32.917816+0000","last_clean":"2023-09-23T15:13:32.917816+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:13:32.917816+0000","last_undegraded":"2023-09-23T15:13:32.917816+0000","last_fullsized":"2023-09-23T15:13:32.917816+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":184,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.917108+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:13:32.917108+0000","last_peered":"2023-09-23T15:13:32.917108+0000","last_clean":"2023-09-23T15:13:32.917108+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:13:32.917108+0000","last_undegraded":"2023-09-23T15:13:32.917108+0000","last_fullsized":"2023-09-23T15:13:32.917108+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":146,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.909282+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:13:32.909282+0000","last_peered":"2023-09-23T15:13:32.909282+0000","last_clean":"2023-09-23T15:13:32.909282+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:13:32.909282+0000","last_undegraded":"2023-09-23T15:13:32.909282+0000","last_fullsized":"2023-09-23T15:13:32.909282+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":167,"reported_epoch":198,"state":"active+clean","last_fresh":"2023-09-23T15:13:32.884919+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:13:32.884919+0000","last_peered":"2023-09-23T15:13:32.884919+0000","last_clean":"2023-09-23T15:13:32.884919+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:13:32.884919+0000","last_undegraded":"2023-09-23T15:13:32.884919+0000","last_fullsized":"2023-09-23T15:13:32.884919+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5675483160,"num_objects":86602,"num_object_clones":0,"num_object_copies":259806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26933,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86602,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114176,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2767,"num_bytes_recovered":181338112,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15552950272,"data_stored":15552938056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104276,"ondisk_log_size":104276,"up":48,"acting":42,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201508,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463144,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2915704,"kb_used_data":2773616,"kb_used_omap":0,"kb_used_meta":142080,"kb_avail":90829448,"statfs":{"total":95995035648,"available":93009354752,"internally_reserved":0,"allocated":2840182784,"data_stored":2839526501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":145489920},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7080000000000002,"5min":6.3949999999999996,"15min":2.5},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":41.491,"5min":108.39400000000001,"15min":108.39400000000001},"last":3.363},{"interface":"front","average":{"1min":3.6779999999999999,"5min":6.5780000000000003,"15min":2.5870000000000002},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":43.151000000000003,"5min":108.452,"15min":108.452},"last":0.74399999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":3.371,"15min":3.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":3.6070000000000002,"5min":3.6070000000000002,"15min":3.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":20.984000000000002,"5min":20.984000000000002,"15min":20.984000000000002},"last":3.4900000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":11.242000000000001,"15min":4.6829999999999998},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.8369999999999997,"5min":139.17500000000001,"15min":180.374},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.1179999999999999,"5min":11.199,"15min":4.6870000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.2080000000000002,"5min":146.999,"15min":181.685},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.2149999999999999,"15min":1.169},"min":{"1min":0.34499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9329999999999998,"5min":17.084,"15min":17.084},"last":0.57099999999999995},{"interface":"front","average":{"1min":3.4940000000000002,"5min":2.2240000000000002,"15min":1.1679999999999999},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.6769999999999996,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":0.40699999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5990000000000002,"5min":1.9590000000000001,"15min":1.0069999999999999},"min":{"1min":0.26800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":36.290999999999997,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.39400000000000002},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.1080000000000001,"15min":1.0940000000000001},"min":{"1min":0.32400000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":36.758000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.28699999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:13:06 2023","interfaces":[{"interface":"back","average":{"1min":6.9089999999999998,"5min":2.9380000000000002,"15min":1.359},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":131.90600000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":6.96,"5min":3.0390000000000001,"15min":1.427},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":131.61000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.671,"15min":0.98199999999999998},"min":{"1min":0.36099999999999999,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":3.7490000000000001,"5min":17,"15min":17},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.4750000000000001,"15min":0.95599999999999996},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.496,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.433}]}]},{"osd":1,"up_from":183,"seq":785979015182,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1436104,"kb_used_data":1327680,"kb_used_omap":6,"kb_used_meta":108409,"kb_avail":92309048,"statfs":{"total":95995035648,"available":94524465152,"internally_reserved":0,"allocated":1359544320,"data_stored":1358594663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":111010891},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.1449999999999996}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":7.915}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":5.1859999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":7.8330000000000002}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.379}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42999999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":7.8890000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893231,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1630932,"kb_used_data":1399692,"kb_used_omap":8,"kb_used_meta":231223,"kb_avail":92114220,"statfs":{"total":95995035648,"available":94324961280,"internally_reserved":0,"allocated":1433284608,"data_stored":1432432439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":236772948},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":31.510999999999999,"5min":31.510999999999999,"15min":31.510999999999999},"last":1.097},{"interface":"front","average":{"1min":4.71,"5min":4.71,"15min":4.71},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.378,"5min":31.378,"15min":31.378},"last":3.0489999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":4.923,"15min":1.929},"min":{"1min":0.189,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":13.157,"5min":122.20099999999999,"15min":122.20099999999999},"last":3.2200000000000002},{"interface":"front","average":{"1min":2.3029999999999999,"5min":4.2199999999999998,"15min":1.6850000000000001},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":14.662000000000001,"5min":122.071,"15min":122.071},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":12.659000000000001,"5min":5.0529999999999999,"15min":2.0529999999999999},"min":{"1min":0.28299999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":84.935000000000002,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.48399999999999999},{"interface":"front","average":{"1min":12.394,"5min":4.1619999999999999,"15min":2.085},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":78.902000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":0.60699999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":2.137,"15min":1.1919999999999999},"min":{"1min":0.32000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.8079999999999998,"5min":11.989000000000001,"15min":11.989000000000001},"last":2.9670000000000001},{"interface":"front","average":{"1min":6.1920000000000002,"5min":2.5299999999999998,"15min":1.276},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":99.168999999999997,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":5.9299999999999997,"5min":2.774,"15min":1.3899999999999999},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":109.369,"5min":109.369,"15min":109.369},"last":2.802},{"interface":"front","average":{"1min":7.1959999999999997,"5min":2.9649999999999999,"15min":1.5149999999999999},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":109.387,"5min":109.387,"15min":109.387},"last":3.0139999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":2.867,"15min":1.423},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":99.114999999999995,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.54800000000000004},{"interface":"front","average":{"1min":5.4470000000000001,"5min":2.5539999999999998,"15min":1.3029999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":98.921999999999997,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.63400000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":5.6859999999999999,"5min":2.7429999999999999,"15min":1.363},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":99.549999999999997,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.9550000000000001,"5min":1.9350000000000001,"15min":1.161},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.468,"5min":9.109,"15min":9.109},"last":3.2349999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566178,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3451484,"kb_used_data":3150740,"kb_used_omap":11,"kb_used_meta":300724,"kb_avail":90293668,"statfs":{"total":95995035648,"available":92460716032,"internally_reserved":0,"allocated":3226357760,"data_stored":3225501787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":307942160},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.8620000000000001,"15min":0.88700000000000001},"min":{"1min":0.186,"5min":0.184,"15min":0.184},"max":{"1min":9.5299999999999994,"5min":30.402000000000001,"15min":30.402000000000001},"last":0.67400000000000004},{"interface":"front","average":{"1min":1.891,"5min":1.716,"15min":0.82899999999999996},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.3309999999999995,"5min":30.550000000000001,"15min":30.550000000000001},"last":0.46100000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:08:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3879999999999999,"5min":0.98299999999999998,"15min":0.58499999999999996},"min":{"1min":0.19600000000000001,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":33.406999999999996,"5min":33.406999999999996,"15min":33.406999999999996},"last":6.8019999999999996},{"interface":"front","average":{"1min":4.351,"5min":1.1779999999999999,"15min":0.65100000000000002},"min":{"1min":0.28399999999999997,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":33.012,"5min":33.012,"15min":33.012},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 15:13:22 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":1.8420000000000001,"15min":1.1060000000000001},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":22.233000000000001,"5min":22.233000000000001,"15min":57.695},"last":3.7280000000000002},{"interface":"front","average":{"1min":3.968,"5min":1.895,"15min":1.1519999999999999},"min":{"1min":0.24399999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.254999999999999,"5min":22.254999999999999,"15min":58.387999999999998},"last":6.8410000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.4660000000000002,"15min":1.25},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1370000000000005,"5min":52.795999999999999,"15min":52.795999999999999},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.954,"5min":3.4020000000000001,"15min":1.571},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5749999999999993,"5min":68.894999999999996,"15min":68.894999999999996},"last":3.6480000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.7160000000000002,"15min":1.3240000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4500000000000002,"5min":41.723999999999997,"15min":41.723999999999997},"last":3.4409999999999998},{"interface":"front","average":{"1min":1.782,"5min":2.5739999999999998,"15min":1.3049999999999999},"min":{"1min":0.35099999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.1859999999999999,"5min":39.822000000000003,"15min":39.822000000000003},"last":3.4670000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.653,"5min":2.774,"15min":1.3999999999999999},"min":{"1min":0.41799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":14.917999999999999,"5min":37.834000000000003,"15min":37.834000000000003},"last":3.5179999999999998},{"interface":"front","average":{"1min":1.9370000000000001,"5min":2.4079999999999999,"15min":1.2689999999999999},"min":{"1min":0.434,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":9.7289999999999992,"5min":37.749000000000002,"15min":37.749000000000002},"last":0.70599999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":2.1280000000000001,"15min":1.1719999999999999},"min":{"1min":0.36099999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4939999999999998,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":3.5419999999999998},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.1619999999999999,"15min":1.1779999999999999},"min":{"1min":0.436,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.2560000000000002,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":3.5699999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402652,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4051428,"kb_used_data":3813148,"kb_used_omap":3,"kb_used_meta":238268,"kb_avail":89693724,"statfs":{"total":95995035648,"available":91846373376,"internally_reserved":0,"allocated":3904663552,"data_stored":3903853813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":243986601},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":3.694,"15min":1.518},"min":{"1min":0.26000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.276,"5min":79.611000000000004,"15min":79.611000000000004},"last":0.54500000000000004},{"interface":"front","average":{"1min":3.0499999999999998,"5min":3.831,"15min":1.5940000000000001},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.0809999999999995,"5min":79.656000000000006,"15min":79.656000000000006},"last":0.29599999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:08:19 2023","interfaces":[{"interface":"back","average":{"1min":8.0150000000000006,"5min":1.883,"15min":0.86099999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.253},"max":{"1min":126.89700000000001,"5min":126.89700000000001,"15min":126.89700000000001},"last":28.315000000000001},{"interface":"front","average":{"1min":7.2789999999999999,"5min":1.7709999999999999,"15min":0.85299999999999998},"min":{"1min":0.32300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":126.355,"5min":126.355,"15min":126.355},"last":10.862}]},{"osd":2,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":7.5570000000000004,"15min":2.952},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":10.467000000000001,"5min":109.077,"15min":109.077},"last":0.40200000000000002},{"interface":"front","average":{"1min":4.0830000000000002,"5min":8.4160000000000004,"15min":3.1589999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.265000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":2.4220000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8009999999999999,"15min":1.026},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0399999999999991,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":0.34599999999999997},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.9670000000000001,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6580000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":10.632999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.6669999999999998,"15min":1.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.9450000000000003,"5min":56.960999999999999,"15min":56.960999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.9209999999999998,"15min":1.5389999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.593,"5min":56.981999999999999,"15min":56.981999999999999},"last":10.738}]},{"osd":6,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.3069999999999999,"15min":1.2290000000000001},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.4749999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.5990000000000002,"15min":1.448},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6890000000000001,"5min":19.355,"15min":19.355},"last":10.561999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.6240000000000001,"15min":1.448},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7489999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":10.603},{"interface":"front","average":{"1min":2.8100000000000001,"5min":1.827,"15min":1.0680000000000001},"min":{"1min":0.47899999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.406000000000001,"5min":13.403,"15min":13.403},"last":0.69599999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986417,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1913204,"kb_used_data":1737132,"kb_used_omap":0,"kb_used_meta":176064,"kb_avail":91831948,"statfs":{"total":95995035648,"available":94035914752,"internally_reserved":0,"allocated":1778823168,"data_stored":1778170460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":180289536},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":3.806,"5min":4.202,"15min":2.2370000000000001},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":63.488,"5min":108.148,"15min":122.69199999999999},"last":1.901},{"interface":"front","average":{"1min":3.7050000000000001,"5min":4.2679999999999998,"15min":2.2639999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":55.951999999999998,"5min":108.124,"15min":122.745},"last":1.6439999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":4.5490000000000004,"15min":4.5490000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":53.832000000000001,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.997},{"interface":"front","average":{"1min":5.077,"5min":5.077,"15min":5.077},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.078000000000003,"5min":53.078000000000003,"15min":53.078000000000003},"last":7.8540000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":10.869999999999999,"5min":6.7279999999999998,"15min":2.633},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":148.08000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":2.1600000000000001},{"interface":"front","average":{"1min":10.5,"5min":6.9420000000000002,"15min":2.8010000000000002},"min":{"1min":0.33000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":147.88900000000001,"5min":147.88900000000001,"15min":147.88900000000001},"last":5.3730000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":20.986999999999998,"5min":7.718,"15min":3.0219999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":135.607,"5min":163.97499999999999,"15min":163.97499999999999},"last":7.4790000000000001},{"interface":"front","average":{"1min":20.789000000000001,"5min":7.1319999999999997,"15min":2.8220000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":135.33600000000001,"5min":164.447,"15min":164.447},"last":5.6020000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:13:38 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.3660000000000001,"15min":1.198},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.998,"5min":28.891999999999999,"15min":28.891999999999999},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.8340000000000001,"15min":1.0449999999999999},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9489999999999998,"5min":32.100999999999999,"15min":32.100999999999999},"last":2.2080000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:13:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.9449999999999998,"15min":1.73},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.927,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.5070000000000001,"15min":1.5980000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5659999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":2.2360000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.1059999999999999,"15min":1.1379999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":4.5869999999999997,"5min":18.23,"15min":18.23},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.78800000000000003,"5min":2.0270000000000001,"15min":1.149},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.879,"5min":14.653,"15min":14.653},"last":0.52700000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724780,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1921964,"kb_used_data":1728484,"kb_used_omap":0,"kb_used_meta":193472,"kb_avail":91823188,"statfs":{"total":95995035648,"available":94026944512,"internally_reserved":0,"allocated":1769967616,"data_stored":1769318887,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":198115328},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.3540000000000001,"15min":1.155},"min":{"1min":0.222,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":16.981999999999999,"5min":24.962,"15min":24.962},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.1240000000000001,"15min":1.073},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":18.814,"5min":25.317,"15min":25.317},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.673,"15min":1.673},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.717000000000001,"5min":11.717000000000001,"15min":11.717000000000001},"last":3.0489999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:12:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":4.5380000000000003,"15min":1.895},"min":{"1min":0.27600000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.3630000000000004,"5min":97.087999999999994,"15min":97.087999999999994},"last":8.7769999999999992},{"interface":"front","average":{"1min":1.363,"5min":4.1970000000000001,"15min":1.8},"min":{"1min":0.32700000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.556,"5min":97.305999999999997,"15min":97.305999999999997},"last":8.6720000000000006}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.2109999999999999,"15min":1.5129999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.7770000000000001,"5min":53.088999999999999,"15min":53.088999999999999},"last":1.0329999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.8260000000000001,"15min":1.4350000000000001},"min":{"1min":0.47299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.121,"5min":53.764000000000003,"15min":53.764000000000003},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.298,"15min":1.163},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7669999999999999,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.622,"5min":2.5449999999999999,"15min":1.242},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.1850000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.372}]},{"osd":6,"last update":"Sat Sep 23 15:13:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0870000000000002,"15min":1.0780000000000001},"min":{"1min":0.30199999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8959999999999999,"5min":15.196,"15min":15.196},"last":0.39400000000000002},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.7549999999999999,"15min":1.006},"min":{"1min":0.34999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.8820000000000001,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.41599999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.3519999999999999,"15min":1.2270000000000001},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.4260000000000002,"5min":15.301,"15min":15.301},"last":0.442},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.2410000000000001,"15min":1.2070000000000001},"min":{"1min":0.40999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":3.0249999999999999,"5min":15.762,"15min":15.762},"last":0.43099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1417809920,"data_stored":1417805848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":668143616,"data_stored":668139544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3216506880,"data_stored":3216506880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3892379648,"data_stored":3892379648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1764032512,"data_stored":1764032512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1764167680,"data_stored":1764163608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2829910016,"data_stored":2829910016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:13:49.630 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:13:49.758 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:49.948 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:13:49.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:49.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:13:49.984 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:49 smithi017 ceph-mon[215646]: pgmap v512: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 26936/259887 objects degraded (10.365%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T15:13:49.984 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:13:49.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:49 smithi017 ceph-mon[222878]: pgmap v512: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 26936/259887 objects degraded (10.365%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T15:13:49.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:13:50.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:50.210 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:50.277 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:13:50.277 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:13:50.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41909 5.3 5:c0f47ff1:::benchmark_data_smithi017_288048_object41908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.58258092300002, 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24766596900000001, 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.278 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41909 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690625+0000", 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690624+0000", 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690647+0000", 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3186000000000001e-05 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690657+0000", 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8619999999999998e-06 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690664+0000", 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1189999999999999e-06 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690697+0000", 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2558000000000002e-05 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690819+0000", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122149 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690983+0000", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016400900000000001 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694628+0000", 2023-09-23T15:13:50.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036453990000000001 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694757+0000", 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012914099999999999 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938211+0000", 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243453118 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938252+0000", 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1928000000000002e-05 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938291+0000", 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8724999999999999e-05 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41914 5.5 5:ab998acd:::benchmark_data_smithi017_288048_object41913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.58222082700001, 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25079286099999998, 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41914 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.285 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690985+0000", 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.690984+0000", 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691006+0000", 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1824e-05 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691017+0000", 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0903e-05 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691025+0000", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6860000000000006e-06 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691073+0000", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8544000000000001e-05 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691190+0000", 2023-09-23T15:13:50.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116915 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.691316+0000", 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012614200000000001 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694247+0000", 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029306789999999998 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938074+0000", 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24382672699999999 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941696+0000", 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003622218 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941738+0000", 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2103e-05 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941778+0000", 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9956999999999997e-05 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41916 5.3 5:c430999f:::benchmark_data_smithi017_288048_object41915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.58058982699998, 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24578065099999999, 2023-09-23T15:13:50.291 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41916 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692616+0000", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692615+0000", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692639+0000", 2023-09-23T15:13:50.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4159999999999999e-05 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692653+0000", 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3944000000000001e-05 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692662+0000", 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000001e-06 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692724+0000", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1750999999999999e-05 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692861+0000", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013759800000000001 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693020+0000", 2023-09-23T15:13:50.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158908 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694908+0000", 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001887583 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938087+0000", 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24317931600000001 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938324+0000", 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023659700000000001 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938357+0000", 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3395000000000001e-05 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938397+0000", 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9937000000000001e-05 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.297 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41912 5.c e122/114 5:3e59ae70:::benchmark_data_smithi017_288048_object41911:head v 122'2612, mlcod=122'2610)", 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.58041396900001, 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24531519199999999, 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692792+0000", 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692791+0000", 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692826+0000", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4484999999999999e-05 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692838+0000", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1950000000000001e-05 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692845+0000", 2023-09-23T15:13:50.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9709999999999998e-06 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692890+0000", 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5012999999999997e-05 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.692913+0000", 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3685000000000001e-05 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.696130+0000", 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003216394 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938107+0000", 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24197753799999999 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.302 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41910 5.7 e122/114 5:e1070690:::benchmark_data_smithi017_288048_object41909:head v 122'2667, mlcod=122'2664)", 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.58012709500002, 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24497349800000001, 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693079+0000", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693078+0000", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693114+0000", 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5808e-05 2023-09-23T15:13:50.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693124+0000", 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9410000000000002e-06 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693130+0000", 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1149999999999999e-06 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693231+0000", 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010016799999999999 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.693254+0000", 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3626e-05 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938007+0000", 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24475242799999999 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938053+0000", 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5976000000000002e-05 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41919 5.8 e122/114 5:1dfe6797:::benchmark_data_smithi017_288048_object41918:head v 122'2577, mlcod=122'2575)", 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.57834805800002, 2023-09-23T15:13:50.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243213651, 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694858+0000", 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694857+0000", 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694884+0000", 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6109999999999999e-05 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694894+0000", 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0916e-05 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694901+0000", 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8789999999999997e-06 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694939+0000", 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7837999999999999e-05 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.694959+0000", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9411000000000001e-05 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938037+0000", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243078029 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938072+0000", 2023-09-23T15:13:50.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5083999999999998e-05 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:41914 5.5 e122/114 ondisk, result = 0)", 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.57779014200003, 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.242661036, 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695416+0000", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695415+0000", 2023-09-23T15:13:50.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695421+0000", 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2460000000000003e-06 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695430+0000", 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.696e-06 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695437+0000", 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2990000000000002e-06 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938055+0000", 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.242618211 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938067+0000", 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2003e-05 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938077+0000", 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0231000000000001e-05 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:41921 5.5 5:ab56ab5a:::benchmark_data_smithi017_288048_object41920:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.57728036700001, 2023-09-23T15:13:50.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24943295900000001, 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 41921 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695926+0000", 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695924+0000", 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695946+0000", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2014e-05 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938001+0000", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24205537099999999 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938016+0000", 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4109e-05 2023-09-23T15:13:50.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938132+0000", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116421 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938297+0000", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000164709 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938483+0000", 2023-09-23T15:13:50.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018597500000000001 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945169+0000", 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066866069999999998 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945215+0000", 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5939999999999997e-05 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945262+0000", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7240000000000002e-05 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945319+0000", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6205000000000001e-05 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.945359+0000", 2023-09-23T15:13:50.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0071000000000003e-05 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.34764.0:41916 5.3 e122/114 ondisk, result = 0)", 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.57724432600003, 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24212840399999999, 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:13:50.323 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695962+0000", 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695961+0000", 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.695966+0000", 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0300000000000001e-06 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938006+0000", 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24203931400000001 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938015+0000", 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0350000000000007e-06 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938073+0000", 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8332999999999999e-05 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938082+0000", 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5310000000000001e-06 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938090+0000", 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6940000000000001e-06 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:41926 5.b e122/114 5:dd0eef87:::benchmark_data_smithi017_288048_object41925:head v 122'2626, mlcod=122'2624)", 2023-09-23T15:13:50.327 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.57400341499999, 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24265115500000001, 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699203+0000", 2023-09-23T15:13:50.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699202+0000", 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.699223+0000", 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1364000000000001e-05 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938011+0000", 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23878813800000001 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938020+0000", 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6649999999999992e-06 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938082+0000", 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2292999999999995e-05 2023-09-23T15:13:50.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.938104+0000", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1537999999999999e-05 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941816+0000", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037116190000000002 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:07:58.941854+0000", 2023-09-23T15:13:50.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8337000000000002e-05 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:105152 5.5 5:a2c1f34a:::benchmark_data_smithi017_288048_object105151:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 291.08597443500003, 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21794708300000001, 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 105152 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187232+0000", 2023-09-23T15:13:50.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187230+0000", 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187254+0000", 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4156e-05 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187266+0000", 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2211e-05 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.187275+0000", 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9600000000000006e-06 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188123+0000", 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0008476 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188222+0000", 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9710999999999995e-05 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.188306+0000", 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3798e-05 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.190666+0000", 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023600489999999999 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405032+0000", 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.214365852 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405088+0000", 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5915999999999998e-05 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.405179+0000", 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0875e-05 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.338 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:105154 5.7 e134/132 5:e4b6367c:::benchmark_data_smithi017_288048_object105153:head v 134'6643, mlcod=134'6641)", 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 291.07664864499998, 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21553064, 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196558+0000", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196557+0000", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196614+0000", 2023-09-23T15:13:50.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6879999999999998e-05 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196625+0000", 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1421e-05 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196633+0000", 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7600000000000002e-06 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196677+0000", 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3894999999999999e-05 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196694+0000", 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6900000000000001e-05 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412009+0000", 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21531566999999999 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412088+0000", 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8899999999999993e-05 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.34764.0:105155 5.7 e134/132 5:e6d00935:::benchmark_data_smithi017_288048_object105154:head v 134'6644, mlcod=134'6641)", 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 291.07655756499997, 2023-09-23T15:13:50.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21546711099999999, 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196649+0000", 2023-09-23T15:13:50.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196648+0000", 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196680+0000", 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1418000000000003e-05 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196687+0000", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3780000000000003e-06 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.196692+0000", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1080000000000001e-06 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404924+0000", 2023-09-23T15:13:50.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20823191099999999 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.404977+0000", 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2981e-05 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412091+0000", 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071144800000000003 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:08:59.412116+0000", 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4219999999999999e-05 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106188 5.3 5:cc2b8235:::benchmark_data_smithi017_288048_object106187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.54424388299998, 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25023266199999999, 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.348 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106188 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728961+0000", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728985+0000", 2023-09-23T15:13:50.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4165999999999999e-05 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729002+0000", 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6997e-05 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729010+0000", 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2709999999999992e-06 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729326+0000", 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031631699999999999 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729469+0000", 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014238 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:13:50.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729612+0000", 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014267699999999999 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732809+0000", 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031979479999999999 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732951+0000", 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014197299999999999 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979128+0000", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.246176479 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979168+0000", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0108000000000003e-05 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979195+0000", 2023-09-23T15:13:50.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6879000000000001e-05 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106190 5.5 5:a890e663:::benchmark_data_smithi017_288048_object106189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 289.54023296600002, 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24679059, 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106190 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:13:50.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732972+0000", 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732993+0000", 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1047e-05 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733006+0000", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2432999999999999e-05 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733011+0000", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733051+0000", 2023-09-23T15:13:50.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9368e-05 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733147+0000", 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6447999999999995e-05 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733242+0000", 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5328000000000006e-05 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.735970+0000", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027275289999999998 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979622+0000", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243651849 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979677+0000", 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5585999999999999e-05 2023-09-23T15:13:50.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979764+0000", 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6321000000000002e-05 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166954 5.3 5:c22244f8:::benchmark_data_smithi017_288048_object55185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:13:50.361 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 212.1811773, 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.279674321, 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166954 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092027+0000", 2023-09-23T15:13:50.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092034+0000", 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8050000000000001e-06 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092053+0000", 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9315000000000001e-05 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092061+0000", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9319999999999996e-06 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092120+0000", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8522e-05 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092275+0000", 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015473200000000001 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092403+0000", 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128739 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096986+0000", 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045829709999999999 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.099294+0000", 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023073640000000001 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371598+0000", 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27230407499999998 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371657+0000", 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9521000000000002e-05 2023-09-23T15:13:50.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371703+0000", 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5927000000000003e-05 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166955 5.5 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:13:50.368 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 212.18071601599999, 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41495601799999998, 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166955 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:13:50.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092489+0000", 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092494+0000", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2989999999999998e-06 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092512+0000", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7952999999999999e-05 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092521+0000", 2023-09-23T15:13:50.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1449999999999997e-06 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092566+0000", 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5461999999999999e-05 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092677+0000", 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011045200000000001 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092782+0000", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096249+0000", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00346767 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507308+0000", 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41105842100000001 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507348+0000", 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0608999999999997e-05 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507446+0000", 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7705000000000001e-05 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:187780 5.5 5:a2bdf4dc:::benchmark_data_smithi017_288048_object76011:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e152)", 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 183.922723168, 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21912024099999999, 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 187780 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350483+0000", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350482+0000", 2023-09-23T15:13:50.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350487+0000", 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9520000000000002e-06 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350497+0000", 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0518e-05 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350506+0000", 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5459999999999992e-06 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350617+0000", 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011165100000000001 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.350766+0000", 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014886700000000001 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.351318+0000", 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00055183399999999996 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.362549+0000", 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011230801 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569458+0000", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20690950299999999 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569518+0000", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9911999999999998e-05 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:46.569603+0000", 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5008999999999999e-05 2023-09-23T15:13:50.380 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:8700 6.0 6:0d712cec:::benchmark_data_smithi017_350392_object8699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:11:52.454006+0000", 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.81919982300001, 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.211242347, 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:50.381 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8700 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454006+0000", 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454006+0000", 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454005+0000", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454038+0000", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.29e-05 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454050+0000", 2023-09-23T15:13:50.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2058999999999999e-05 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454055+0000", 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7049999999999996e-06 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454099+0000", 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3396999999999998e-05 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454239+0000", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140168 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454346+0000", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106673 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.455310+0000", 2023-09-23T15:13:50.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00096440800000000004 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.456031+0000", 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072134599999999999 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665159+0000", 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20912810000000001 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665208+0000", 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8572999999999997e-05 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665249+0000", 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0672e-05 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:8701 6.9 6:979db853:::benchmark_data_smithi017_350392_object8700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:11:52.454070+0000", 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.819136638, 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21111776800000001, 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8701 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454070+0000", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454070+0000", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454069+0000", 2023-09-23T15:13:50.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454086+0000", 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7295e-05 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454091+0000", 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6800000000000001e-06 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454095+0000", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0579999999999999e-06 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454139+0000", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3634999999999998e-05 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454258+0000", 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011964900000000001 2023-09-23T15:13:50.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.454400+0000", 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141347 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.455768+0000", 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013679149999999999 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.455815+0000", 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7651999999999997e-05 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665083+0000", 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.20926815300000001 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665142+0000", 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8510000000000001e-05 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:13:50.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:11:52.665187+0000", 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5289e-05 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:13:50.394 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:13:50.395 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:49 smithi027 ceph-mon[195233]: pgmap v512: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 347 op/s; 26936/259887 objects degraded (10.365%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T15:13:50.395 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:13:50.395 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:50.396 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:50.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:50.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:13:50.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:50.532 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:13:50.544 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:50.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:13:50.612 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:13:50.657 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:50.660 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:13:50.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:50.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:50.690 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:13:50.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:13:50.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:50.784 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.801 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.801 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:13:50.805 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:50.883 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:50.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:50.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:50.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:51.026 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:51.096 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:51.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:51.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4286608488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:51.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4286608488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:51.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4286608488' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:13:51.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:51.276 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:51.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:13:51.831 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:51.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:51.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:51.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:51.974 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:52 smithi017 ceph-mon[215646]: pgmap v513: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 26634/251496 objects degraded (10.590%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:52.010 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26634/251496 objects degraded (10.590%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:52.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:52 smithi017 ceph-mon[222878]: pgmap v513: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 26634/251496 objects degraded (10.590%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:52.020 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26634/251496 objects degraded (10.590%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:52.037 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:52.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:52.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:52.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:52.241 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:52.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:52 smithi027 ceph-mon[195233]: pgmap v513: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 563 op/s; 26634/251496 objects degraded (10.590%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:13:52.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26634/251496 objects degraded (10.590%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:52.466 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:52.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:52.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:52.479 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:52.578 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:52.617 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:52.683 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:52.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:52.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:52.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:52.853 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:52.926 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:52.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:52.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:53.034 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:53.129 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:53.549 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:53.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:53.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:53.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:53.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[222878]: pgmap v514: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 26634/251496 objects degraded (10.590%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:53.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:53.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:53.776 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[215646]: pgmap v514: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 26634/251496 objects degraded (10.590%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:53.777 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:53.777 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:53.797 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:53.921 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:54.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:13:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:53 smithi027 ceph-mon[195233]: pgmap v514: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 556 op/s; 26634/251496 objects degraded (10.590%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:54.146 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:54.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:54.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:54.255 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:54.288 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:54.345 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:54.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:54.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:54.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:13:54.565 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:54.875 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:13:54.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:54.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:13:54.995 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:55.146 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:55.316 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:55.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:55.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:55.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:55.468 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:55.528 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:55.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:55.535 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:55.635 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:55.672 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:55.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:55 smithi027 ceph-mon[195233]: pgmap v515: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 26634/248916 objects degraded (10.700%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:55.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:55.721 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:55.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:55.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:55.777 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:13:55.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:13:55.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:55 smithi017 ceph-mon[215646]: pgmap v515: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 26634/248916 objects degraded (10.700%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:55.939 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:55 smithi017 ceph-mon[222878]: pgmap v515: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 5.1 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 26634/248916 objects degraded (10.700%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T15:13:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:13:55.943 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:56.158 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:13:56.160 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:13:56.498 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:13:56.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:56.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:13:56.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:56.737 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[215646]: failed to encode map e201 with expected crc 2023-09-23T15:13:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[215646]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T15:13:56.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[222878]: failed to encode map e201 with expected crc 2023-09-23T15:13:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:56.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:56 smithi017 ceph-mon[222878]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T15:13:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:56 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:13:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:56 smithi027 ceph-mon[195233]: failed to encode map e201 with expected crc 2023-09-23T15:13:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2570499852' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:13:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:56 smithi027 ceph-mon[195233]: osdmap e201: 8 total, 8 up, 7 in 2023-09-23T15:13:57.689 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:57.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:57.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:57.697 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:13:57.741 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":516,"stamp":"2023-09-23T15:13:55.737258+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5312977116,"num_objects":81025,"num_object_clones":0,"num_object_copies":243075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26372,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":81025,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":120808,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3390,"num_bytes_recovered":223647911,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":111393,"ondisk_log_size":111393,"up":327,"acting":319,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16300032,"kb_used_data":14912904,"kb_used_omap":30,"kb_used_meta":1386913,"kb_avail":639916032,"statfs":{"total":671965249536,"available":655274016768,"internally_reserved":0,"allocated":15270813696,"data_stored":15265360757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1420199512},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,4,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-367263744,"num_objects":-5604,"num_object_clones":0,"num_object_copies":-16812,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-564,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5604,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5604,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":564,"num_bytes_recovered":36962304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.829676"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":194,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452205+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:13:40.452205+0000","last_peered":"2023-09-23T15:13:40.452205+0000","last_clean":"2023-09-23T15:13:40.452205+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:13:40.452205+0000","last_undegraded":"2023-09-23T15:13:40.452205+0000","last_fullsized":"2023-09-23T15:13:40.452205+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":128,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.363668+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:13:44.363668+0000","last_peered":"2023-09-23T15:13:44.363668+0000","last_clean":"2023-09-23T15:13:44.363668+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:13:44.363668+0000","last_undegraded":"2023-09-23T15:13:44.363668+0000","last_fullsized":"2023-09-23T15:13:44.363668+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":76,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456881+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:13:40.456881+0000","last_peered":"2023-09-23T15:13:40.456881+0000","last_clean":"2023-09-23T15:13:40.456881+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:13:40.456881+0000","last_undegraded":"2023-09-23T15:13:40.456881+0000","last_fullsized":"2023-09-23T15:13:40.456881+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":164,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442821+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:13:40.442821+0000","last_peered":"2023-09-23T15:13:40.442821+0000","last_clean":"2023-09-23T15:13:40.442821+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:13:40.442821+0000","last_undegraded":"2023-09-23T15:13:40.442821+0000","last_fullsized":"2023-09-23T15:13:40.442821+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":197,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455568+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:13:40.455568+0000","last_peered":"2023-09-23T15:13:40.455568+0000","last_clean":"2023-09-23T15:13:40.455568+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:13:40.455568+0000","last_undegraded":"2023-09-23T15:13:40.455568+0000","last_fullsized":"2023-09-23T15:13:40.455568+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":174,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.433648+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:13:40.433648+0000","last_peered":"2023-09-23T15:13:40.433648+0000","last_clean":"2023-09-23T15:13:40.433648+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:13:40.433648+0000","last_undegraded":"2023-09-23T15:13:40.433648+0000","last_fullsized":"2023-09-23T15:13:40.433648+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":143,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442962+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:13:40.442962+0000","last_peered":"2023-09-23T15:13:40.442962+0000","last_clean":"2023-09-23T15:13:40.442962+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:13:40.442962+0000","last_undegraded":"2023-09-23T15:13:40.442962+0000","last_fullsized":"2023-09-23T15:13:40.442962+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455635+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:13:40.455635+0000","last_peered":"2023-09-23T15:13:40.455635+0000","last_clean":"2023-09-23T15:13:40.455635+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:13:40.455635+0000","last_undegraded":"2023-09-23T15:13:40.455635+0000","last_fullsized":"2023-09-23T15:13:40.455635+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.915319+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:13:44.915319+0000","last_peered":"2023-09-23T15:13:44.915319+0000","last_clean":"2023-09-23T15:13:44.915319+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:13:44.915319+0000","last_undegraded":"2023-09-23T15:13:44.915319+0000","last_fullsized":"2023-09-23T15:13:44.915319+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":190,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.446882+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:13:40.446882+0000","last_peered":"2023-09-23T15:13:40.446882+0000","last_clean":"2023-09-23T15:13:40.446882+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:13:40.446882+0000","last_undegraded":"2023-09-23T15:13:40.446882+0000","last_fullsized":"2023-09-23T15:13:40.446882+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455001+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:13:40.455001+0000","last_peered":"2023-09-23T15:13:40.455001+0000","last_clean":"2023-09-23T15:13:40.455001+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:13:40.455001+0000","last_undegraded":"2023-09-23T15:13:40.455001+0000","last_fullsized":"2023-09-23T15:13:40.455001+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":61,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455308+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:13:40.455308+0000","last_peered":"2023-09-23T15:13:40.455308+0000","last_clean":"2023-09-23T15:13:40.455308+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:13:40.455308+0000","last_undegraded":"2023-09-23T15:13:40.455308+0000","last_fullsized":"2023-09-23T15:13:40.455308+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456824+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:13:40.456824+0000","last_peered":"2023-09-23T15:13:40.456824+0000","last_clean":"2023-09-23T15:13:40.456824+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:13:40.456824+0000","last_undegraded":"2023-09-23T15:13:40.456824+0000","last_fullsized":"2023-09-23T15:13:40.456824+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":184,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452739+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:13:40.452739+0000","last_peered":"2023-09-23T15:13:40.452739+0000","last_clean":"2023-09-23T15:13:40.452739+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:13:40.452739+0000","last_undegraded":"2023-09-23T15:13:40.452739+0000","last_fullsized":"2023-09-23T15:13:40.452739+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452413+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:13:40.452413+0000","last_peered":"2023-09-23T15:13:40.452413+0000","last_clean":"2023-09-23T15:13:40.452413+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:13:40.452413+0000","last_undegraded":"2023-09-23T15:13:40.452413+0000","last_fullsized":"2023-09-23T15:13:40.452413+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":488,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.914202+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:13:44.914202+0000","last_peered":"2023-09-23T15:13:44.914202+0000","last_clean":"2023-09-23T15:13:44.914202+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:13:44.914202+0000","last_undegraded":"2023-09-23T15:13:44.914202+0000","last_fullsized":"2023-09-23T15:13:44.914202+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":62,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452464+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:13:40.452464+0000","last_peered":"2023-09-23T15:13:40.452464+0000","last_clean":"2023-09-23T15:13:40.452464+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:13:40.452464+0000","last_undegraded":"2023-09-23T15:13:40.452464+0000","last_fullsized":"2023-09-23T15:13:40.452464+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456586+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:13:40.456586+0000","last_peered":"2023-09-23T15:13:40.456586+0000","last_clean":"2023-09-23T15:13:40.456586+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:13:40.456586+0000","last_undegraded":"2023-09-23T15:13:40.456586+0000","last_fullsized":"2023-09-23T15:13:40.456586+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":488,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.916172+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:13:44.916172+0000","last_peered":"2023-09-23T15:13:44.916172+0000","last_clean":"2023-09-23T15:13:44.916172+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:13:44.916172+0000","last_undegraded":"2023-09-23T15:13:44.916172+0000","last_fullsized":"2023-09-23T15:13:44.916172+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":163,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.443093+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:13:40.443093+0000","last_peered":"2023-09-23T15:13:40.443093+0000","last_clean":"2023-09-23T15:13:40.443093+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:13:40.443093+0000","last_undegraded":"2023-09-23T15:13:40.443093+0000","last_fullsized":"2023-09-23T15:13:40.443093+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":168,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451882+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:13:40.451882+0000","last_peered":"2023-09-23T15:13:40.451882+0000","last_clean":"2023-09-23T15:13:40.451882+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:13:40.451882+0000","last_undegraded":"2023-09-23T15:13:40.451882+0000","last_fullsized":"2023-09-23T15:13:40.451882+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.914667+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:13:44.914667+0000","last_peered":"2023-09-23T15:13:44.914667+0000","last_clean":"2023-09-23T15:13:44.914667+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:13:44.914667+0000","last_undegraded":"2023-09-23T15:13:44.914667+0000","last_fullsized":"2023-09-23T15:13:44.914667+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":46,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452255+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:13:40.452255+0000","last_peered":"2023-09-23T15:13:40.452255+0000","last_clean":"2023-09-23T15:13:40.452255+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:13:40.452255+0000","last_undegraded":"2023-09-23T15:13:40.452255+0000","last_fullsized":"2023-09-23T15:13:40.452255+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":152,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442858+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:13:40.442858+0000","last_peered":"2023-09-23T15:13:40.442858+0000","last_clean":"2023-09-23T15:13:40.442858+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:13:40.442858+0000","last_undegraded":"2023-09-23T15:13:40.442858+0000","last_fullsized":"2023-09-23T15:13:40.442858+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":488,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.914542+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:13:44.914542+0000","last_peered":"2023-09-23T15:13:44.914542+0000","last_clean":"2023-09-23T15:13:44.914542+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:13:44.914542+0000","last_undegraded":"2023-09-23T15:13:44.914542+0000","last_fullsized":"2023-09-23T15:13:44.914542+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":166,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.432936+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:13:40.432936+0000","last_peered":"2023-09-23T15:13:40.432936+0000","last_clean":"2023-09-23T15:13:40.432936+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:13:40.432936+0000","last_undegraded":"2023-09-23T15:13:40.432936+0000","last_fullsized":"2023-09-23T15:13:40.432936+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455856+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:13:40.455856+0000","last_peered":"2023-09-23T15:13:40.455856+0000","last_clean":"2023-09-23T15:13:40.455856+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:13:40.455856+0000","last_undegraded":"2023-09-23T15:13:40.455856+0000","last_fullsized":"2023-09-23T15:13:40.455856+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"200'7394","reported_seq":7431,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.684346+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:13:54.684346+0000","last_peered":"2023-09-23T15:13:54.684346+0000","last_clean":"2023-09-23T15:13:54.684346+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:13:54.684346+0000","last_undegraded":"2023-09-23T15:13:54.684346+0000","last_fullsized":"2023-09-23T15:13:54.684346+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680000,"num_objects":5000,"num_object_clones":0,"num_object_copies":15000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7394,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.916225+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:13:44.916225+0000","last_peered":"2023-09-23T15:13:44.916225+0000","last_clean":"2023-09-23T15:13:44.916225+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:13:44.916225+0000","last_undegraded":"2023-09-23T15:13:44.916225+0000","last_fullsized":"2023-09-23T15:13:44.916225+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":74,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.454383+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:13:40.454383+0000","last_peered":"2023-09-23T15:13:40.454383+0000","last_clean":"2023-09-23T15:13:40.454383+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:13:40.454383+0000","last_undegraded":"2023-09-23T15:13:40.454383+0000","last_fullsized":"2023-09-23T15:13:40.454383+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":151,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.446771+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:13:40.446771+0000","last_peered":"2023-09-23T15:13:40.446771+0000","last_clean":"2023-09-23T15:13:40.446771+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:13:40.446771+0000","last_undegraded":"2023-09-23T15:13:40.446771+0000","last_fullsized":"2023-09-23T15:13:40.446771+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"200'7566","reported_seq":7612,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:54.342290+0000","last_change":"2023-09-23T15:12:43.308627+0000","last_active":"2023-09-23T15:13:54.342290+0000","last_peered":"2023-09-23T15:13:54.342290+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:13:54.342290+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"175'1100","ondisk_log_start":"175'1100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6466,"log_dups_size":0,"ondisk_log_size":6466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334233600,"num_objects":5100,"num_object_clones":0,"num_object_copies":15300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7566,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":103}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":467,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:44.915082+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:13:44.915082+0000","last_peered":"2023-09-23T15:13:44.915082+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:13:44.915082+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":74,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.458154+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:13:40.458154+0000","last_peered":"2023-09-23T15:13:40.458154+0000","last_clean":"2023-09-23T15:13:40.458154+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:13:40.458154+0000","last_undegraded":"2023-09-23T15:13:40.458154+0000","last_fullsized":"2023-09-23T15:13:40.458154+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":192,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452126+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:13:40.452126+0000","last_peered":"2023-09-23T15:13:40.452126+0000","last_clean":"2023-09-23T15:13:40.452126+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:13:40.452126+0000","last_undegraded":"2023-09-23T15:13:40.452126+0000","last_fullsized":"2023-09-23T15:13:40.452126+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"200'7383","reported_seq":7421,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:53.478960+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:13:53.478960+0000","last_peered":"2023-09-23T15:13:53.478960+0000","last_clean":"2023-09-23T15:13:53.478960+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:13:53.478960+0000","last_undegraded":"2023-09-23T15:13:53.478960+0000","last_fullsized":"2023-09-23T15:13:53.478960+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331546624,"num_objects":5059,"num_object_clones":0,"num_object_copies":15177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7383,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.914634+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:13:44.914634+0000","last_peered":"2023-09-23T15:13:44.914634+0000","last_clean":"2023-09-23T15:13:44.914634+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:13:44.914634+0000","last_undegraded":"2023-09-23T15:13:44.914634+0000","last_fullsized":"2023-09-23T15:13:44.914634+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":60,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451809+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:13:40.451809+0000","last_peered":"2023-09-23T15:13:40.451809+0000","last_clean":"2023-09-23T15:13:40.451809+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:13:40.451809+0000","last_undegraded":"2023-09-23T15:13:40.451809+0000","last_fullsized":"2023-09-23T15:13:40.451809+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":188,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.433282+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:13:40.433282+0000","last_peered":"2023-09-23T15:13:40.433282+0000","last_clean":"2023-09-23T15:13:40.433282+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:13:40.433282+0000","last_undegraded":"2023-09-23T15:13:40.433282+0000","last_fullsized":"2023-09-23T15:13:40.433282+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"200'7563","reported_seq":7602,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.357592+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:13:54.357592+0000","last_peered":"2023-09-23T15:13:54.357592+0000","last_clean":"2023-09-23T15:13:54.357592+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:13:54.357592+0000","last_undegraded":"2023-09-23T15:13:54.357592+0000","last_fullsized":"2023-09-23T15:13:54.357592+0000","mapping_epoch":171,"log_start":"175'2000","ondisk_log_start":"175'2000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299136,"num_objects":5101,"num_object_clones":0,"num_object_copies":15303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7563,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.915514+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:13:44.915514+0000","last_peered":"2023-09-23T15:13:44.915514+0000","last_clean":"2023-09-23T15:13:44.915514+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:13:44.915514+0000","last_undegraded":"2023-09-23T15:13:44.915514+0000","last_fullsized":"2023-09-23T15:13:44.915514+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.447225+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:40.447225+0000","last_peered":"2023-09-23T15:13:40.447225+0000","last_clean":"2023-09-23T15:13:40.447225+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:13:40.447225+0000","last_undegraded":"2023-09-23T15:13:40.447225+0000","last_fullsized":"2023-09-23T15:13:40.447225+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442332+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:13:40.442332+0000","last_peered":"2023-09-23T15:13:40.442332+0000","last_clean":"2023-09-23T15:13:40.442332+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:13:40.442332+0000","last_undegraded":"2023-09-23T15:13:40.442332+0000","last_fullsized":"2023-09-23T15:13:40.442332+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"200'7678","reported_seq":7717,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:54.298303+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:13:54.298303+0000","last_peered":"2023-09-23T15:13:54.298303+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:13:54.298303+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"174'100","ondisk_log_start":"174'100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7578,"log_dups_size":0,"ondisk_log_size":7578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343015424,"num_objects":5234,"num_object_clones":0,"num_object_copies":15702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7678,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":224}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":488,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.383062+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:13:44.383062+0000","last_peered":"2023-09-23T15:13:44.383062+0000","last_clean":"2023-09-23T15:13:44.383062+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:13:44.383062+0000","last_undegraded":"2023-09-23T15:13:44.383062+0000","last_fullsized":"2023-09-23T15:13:44.383062+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":162,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.461046+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:13:40.461046+0000","last_peered":"2023-09-23T15:13:40.461046+0000","last_clean":"2023-09-23T15:13:40.461046+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:13:40.461046+0000","last_undegraded":"2023-09-23T15:13:40.461046+0000","last_fullsized":"2023-09-23T15:13:40.461046+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452781+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:13:40.452781+0000","last_peered":"2023-09-23T15:13:40.452781+0000","last_clean":"2023-09-23T15:13:40.452781+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:13:40.452781+0000","last_undegraded":"2023-09-23T15:13:40.452781+0000","last_fullsized":"2023-09-23T15:13:40.452781+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"200'7496","reported_seq":7534,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:53.401505+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:13:53.401505+0000","last_peered":"2023-09-23T15:13:53.401505+0000","last_clean":"2023-09-23T15:13:53.401505+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:13:53.401505+0000","last_undegraded":"2023-09-23T15:13:53.401505+0000","last_fullsized":"2023-09-23T15:13:53.401505+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7496,"log_dups_size":0,"ondisk_log_size":7496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333053952,"num_objects":5082,"num_object_clones":0,"num_object_copies":15246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7496,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":493,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.383031+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:13:44.383031+0000","last_peered":"2023-09-23T15:13:44.383031+0000","last_clean":"2023-09-23T15:13:44.383031+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:13:44.383031+0000","last_undegraded":"2023-09-23T15:13:44.383031+0000","last_fullsized":"2023-09-23T15:13:44.383031+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.433203+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:13:40.433203+0000","last_peered":"2023-09-23T15:13:40.433203+0000","last_clean":"2023-09-23T15:13:40.433203+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:13:40.433203+0000","last_undegraded":"2023-09-23T15:13:40.433203+0000","last_fullsized":"2023-09-23T15:13:40.433203+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":192,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452900+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:13:40.452900+0000","last_peered":"2023-09-23T15:13:40.452900+0000","last_clean":"2023-09-23T15:13:40.452900+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:13:40.452900+0000","last_undegraded":"2023-09-23T15:13:40.452900+0000","last_fullsized":"2023-09-23T15:13:40.452900+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"200'7494","reported_seq":7532,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.300490+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:13:54.300490+0000","last_peered":"2023-09-23T15:13:54.300490+0000","last_clean":"2023-09-23T15:13:54.300490+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:13:54.300490+0000","last_undegraded":"2023-09-23T15:13:54.300490+0000","last_fullsized":"2023-09-23T15:13:54.300490+0000","mapping_epoch":171,"log_start":"175'1100","ondisk_log_start":"175'1100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6394,"log_dups_size":0,"ondisk_log_size":6394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333185024,"num_objects":5084,"num_object_clones":0,"num_object_copies":15252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7494,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":476,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.914754+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:13:44.914754+0000","last_peered":"2023-09-23T15:13:44.914754+0000","last_clean":"2023-09-23T15:13:44.914754+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:13:44.914754+0000","last_undegraded":"2023-09-23T15:13:44.914754+0000","last_fullsized":"2023-09-23T15:13:44.914754+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":183,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451891+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:13:40.451891+0000","last_peered":"2023-09-23T15:13:40.451891+0000","last_clean":"2023-09-23T15:13:40.451891+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:13:40.451891+0000","last_undegraded":"2023-09-23T15:13:40.451891+0000","last_fullsized":"2023-09-23T15:13:40.451891+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.458204+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:13:40.458204+0000","last_peered":"2023-09-23T15:13:40.458204+0000","last_clean":"2023-09-23T15:13:40.458204+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:13:40.458204+0000","last_undegraded":"2023-09-23T15:13:40.458204+0000","last_fullsized":"2023-09-23T15:13:40.458204+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"200'7500","reported_seq":14207,"reported_epoch":200,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:54.685655+0000","last_change":"2023-09-23T15:12:43.243089+0000","last_active":"2023-09-23T15:13:54.685655+0000","last_peered":"2023-09-23T15:13:54.685655+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.190605+0000","last_became_peered":"2023-09-23T15:12:43.190605+0000","last_unstale":"2023-09-23T15:13:54.685655+0000","last_undegraded":"2023-09-23T15:12:42.959077+0000","last_fullsized":"2023-09-23T15:12:42.845806+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7500,"log_dups_size":0,"ondisk_log_size":7500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:41:00.861011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329711640,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1570,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7500,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3330,"num_bytes_recovered":218234880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":3462}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":488,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.383124+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:13:44.383124+0000","last_peered":"2023-09-23T15:13:44.383124+0000","last_clean":"2023-09-23T15:13:44.383124+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:13:44.383124+0000","last_undegraded":"2023-09-23T15:13:44.383124+0000","last_fullsized":"2023-09-23T15:13:44.383124+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":191,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455493+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:13:40.455493+0000","last_peered":"2023-09-23T15:13:40.455493+0000","last_clean":"2023-09-23T15:13:40.455493+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:13:40.455493+0000","last_undegraded":"2023-09-23T15:13:40.455493+0000","last_fullsized":"2023-09-23T15:13:40.455493+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":76,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.462640+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:13:40.462640+0000","last_peered":"2023-09-23T15:13:40.462640+0000","last_clean":"2023-09-23T15:13:40.462640+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:13:40.462640+0000","last_undegraded":"2023-09-23T15:13:40.462640+0000","last_fullsized":"2023-09-23T15:13:40.462640+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"200'7491","reported_seq":7528,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.317021+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:13:54.317021+0000","last_peered":"2023-09-23T15:13:54.317021+0000","last_clean":"2023-09-23T15:13:54.317021+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:13:54.317021+0000","last_undegraded":"2023-09-23T15:13:54.317021+0000","last_fullsized":"2023-09-23T15:13:54.317021+0000","mapping_epoch":171,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5591,"log_dups_size":0,"ondisk_log_size":5591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332726272,"num_objects":5077,"num_object_clones":0,"num_object_copies":15231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5077,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7491,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.383407+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:13:44.383407+0000","last_peered":"2023-09-23T15:13:44.383407+0000","last_clean":"2023-09-23T15:13:44.383407+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:13:44.383407+0000","last_undegraded":"2023-09-23T15:13:44.383407+0000","last_fullsized":"2023-09-23T15:13:44.383407+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":375,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.459249+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:13:40.459249+0000","last_peered":"2023-09-23T15:13:40.459249+0000","last_clean":"2023-09-23T15:13:40.459249+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:13:40.459249+0000","last_undegraded":"2023-09-23T15:13:40.459249+0000","last_fullsized":"2023-09-23T15:13:40.459249+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":176,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451729+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:40.451729+0000","last_peered":"2023-09-23T15:13:40.451729+0000","last_clean":"2023-09-23T15:13:40.451729+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:13:40.451729+0000","last_undegraded":"2023-09-23T15:13:40.451729+0000","last_fullsized":"2023-09-23T15:13:40.451729+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.444441+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:13:40.444441+0000","last_peered":"2023-09-23T15:13:40.444441+0000","last_clean":"2023-09-23T15:13:40.444441+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:13:40.444441+0000","last_undegraded":"2023-09-23T15:13:40.444441+0000","last_fullsized":"2023-09-23T15:13:40.444441+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"200'7404","reported_seq":7443,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.386720+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:13:54.386720+0000","last_peered":"2023-09-23T15:13:54.386720+0000","last_clean":"2023-09-23T15:13:54.386720+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:13:54.386720+0000","last_undegraded":"2023-09-23T15:13:54.386720+0000","last_fullsized":"2023-09-23T15:13:54.386720+0000","mapping_epoch":171,"log_start":"175'1000","ondisk_log_start":"175'1000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324403200,"num_objects":4950,"num_object_clones":0,"num_object_copies":14850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7404,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442469+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:13:40.442469+0000","last_peered":"2023-09-23T15:13:40.442469+0000","last_clean":"2023-09-23T15:13:40.442469+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:13:40.442469+0000","last_undegraded":"2023-09-23T15:13:40.442469+0000","last_fullsized":"2023-09-23T15:13:40.442469+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":718,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.447548+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:13:40.447548+0000","last_peered":"2023-09-23T15:13:40.447548+0000","last_clean":"2023-09-23T15:13:40.447548+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:13:40.447548+0000","last_undegraded":"2023-09-23T15:13:40.447548+0000","last_fullsized":"2023-09-23T15:13:40.447548+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":159,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452374+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:13:40.452374+0000","last_peered":"2023-09-23T15:13:40.452374+0000","last_clean":"2023-09-23T15:13:40.452374+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:13:40.452374+0000","last_undegraded":"2023-09-23T15:13:40.452374+0000","last_fullsized":"2023-09-23T15:13:40.452374+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.458893+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:13:40.458893+0000","last_peered":"2023-09-23T15:13:40.458893+0000","last_clean":"2023-09-23T15:13:40.458893+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:13:40.458893+0000","last_undegraded":"2023-09-23T15:13:40.458893+0000","last_fullsized":"2023-09-23T15:13:40.458893+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"200'7535","reported_seq":7572,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.077892+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:13:54.077892+0000","last_peered":"2023-09-23T15:13:54.077892+0000","last_clean":"2023-09-23T15:13:54.077892+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:13:54.077892+0000","last_undegraded":"2023-09-23T15:13:54.077892+0000","last_fullsized":"2023-09-23T15:13:54.077892+0000","mapping_epoch":171,"log_start":"176'2200","ondisk_log_start":"176'2200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333643776,"num_objects":5091,"num_object_clones":0,"num_object_copies":15273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7535,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":488,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.443156+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:13:40.443156+0000","last_peered":"2023-09-23T15:13:40.443156+0000","last_clean":"2023-09-23T15:13:40.443156+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:13:40.443156+0000","last_undegraded":"2023-09-23T15:13:40.443156+0000","last_fullsized":"2023-09-23T15:13:40.443156+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":811,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.444832+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:13:40.444832+0000","last_peered":"2023-09-23T15:13:40.444832+0000","last_clean":"2023-09-23T15:13:40.444832+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:13:40.444832+0000","last_undegraded":"2023-09-23T15:13:40.444832+0000","last_fullsized":"2023-09-23T15:13:40.444832+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456488+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:13:40.456488+0000","last_peered":"2023-09-23T15:13:40.456488+0000","last_clean":"2023-09-23T15:13:40.456488+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:13:40.456488+0000","last_undegraded":"2023-09-23T15:13:40.456488+0000","last_fullsized":"2023-09-23T15:13:40.456488+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":95,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.462560+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:13:40.462560+0000","last_peered":"2023-09-23T15:13:40.462560+0000","last_clean":"2023-09-23T15:13:40.462560+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:13:40.462560+0000","last_undegraded":"2023-09-23T15:13:40.462560+0000","last_fullsized":"2023-09-23T15:13:40.462560+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"200'7403","reported_seq":7440,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.640014+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:13:54.640014+0000","last_peered":"2023-09-23T15:13:54.640014+0000","last_clean":"2023-09-23T15:13:54.640014+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:13:54.640014+0000","last_undegraded":"2023-09-23T15:13:54.640014+0000","last_fullsized":"2023-09-23T15:13:54.640014+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7403,"log_dups_size":0,"ondisk_log_size":7403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326172672,"num_objects":4977,"num_object_clones":0,"num_object_copies":14931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7403,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":487,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.447111+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:13:40.447111+0000","last_peered":"2023-09-23T15:13:40.447111+0000","last_clean":"2023-09-23T15:13:40.447111+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:13:40.447111+0000","last_undegraded":"2023-09-23T15:13:40.447111+0000","last_fullsized":"2023-09-23T15:13:40.447111+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":488,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.441953+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:13:40.441953+0000","last_peered":"2023-09-23T15:13:40.441953+0000","last_clean":"2023-09-23T15:13:40.441953+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:13:40.441953+0000","last_undegraded":"2023-09-23T15:13:40.441953+0000","last_fullsized":"2023-09-23T15:13:40.441953+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.454859+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:13:40.454859+0000","last_peered":"2023-09-23T15:13:40.454859+0000","last_clean":"2023-09-23T15:13:40.454859+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:13:40.454859+0000","last_undegraded":"2023-09-23T15:13:40.454859+0000","last_fullsized":"2023-09-23T15:13:40.454859+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":150,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.433540+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:13:40.433540+0000","last_peered":"2023-09-23T15:13:40.433540+0000","last_clean":"2023-09-23T15:13:40.433540+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:13:40.433540+0000","last_undegraded":"2023-09-23T15:13:40.433540+0000","last_fullsized":"2023-09-23T15:13:40.433540+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"200'7535","reported_seq":7574,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:54.327718+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:13:54.327718+0000","last_peered":"2023-09-23T15:13:54.327718+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:13:54.327718+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"175'1100","ondisk_log_start":"175'1100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6435,"log_dups_size":0,"ondisk_log_size":6435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334823424,"num_objects":5109,"num_object_clones":0,"num_object_copies":15327,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5109,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7535,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":157}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442250+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:13:40.442250+0000","last_peered":"2023-09-23T15:13:40.442250+0000","last_clean":"2023-09-23T15:13:40.442250+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:13:40.442250+0000","last_undegraded":"2023-09-23T15:13:40.442250+0000","last_fullsized":"2023-09-23T15:13:40.442250+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":282,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.456149+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:13:40.456149+0000","last_peered":"2023-09-23T15:13:40.456149+0000","last_clean":"2023-09-23T15:13:40.456149+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:13:40.456149+0000","last_undegraded":"2023-09-23T15:13:40.456149+0000","last_fullsized":"2023-09-23T15:13:40.456149+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":144,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442281+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:13:40.442281+0000","last_peered":"2023-09-23T15:13:40.442281+0000","last_clean":"2023-09-23T15:13:40.442281+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:13:40.442281+0000","last_undegraded":"2023-09-23T15:13:40.442281+0000","last_fullsized":"2023-09-23T15:13:40.442281+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":174,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442743+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:13:40.442743+0000","last_peered":"2023-09-23T15:13:40.442743+0000","last_clean":"2023-09-23T15:13:40.442743+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:13:40.442743+0000","last_undegraded":"2023-09-23T15:13:40.442743+0000","last_fullsized":"2023-09-23T15:13:40.442743+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"200'7440","reported_seq":7477,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:53.407327+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:13:53.407327+0000","last_peered":"2023-09-23T15:13:53.407327+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:13:53.407327+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7440,"log_dups_size":0,"ondisk_log_size":7440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332136448,"num_objects":5068,"num_object_clones":0,"num_object_copies":15204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7440,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":194}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.383358+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:13:44.383358+0000","last_peered":"2023-09-23T15:13:44.383358+0000","last_clean":"2023-09-23T15:13:44.383358+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:13:44.383358+0000","last_undegraded":"2023-09-23T15:13:44.383358+0000","last_fullsized":"2023-09-23T15:13:44.383358+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":805,"reported_epoch":199,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:40.444539+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:13:40.444539+0000","last_peered":"2023-09-23T15:13:40.444539+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:13:40.444539+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":71,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.459125+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:13:40.459125+0000","last_peered":"2023-09-23T15:13:40.459125+0000","last_clean":"2023-09-23T15:13:40.459125+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:13:40.459125+0000","last_undegraded":"2023-09-23T15:13:40.459125+0000","last_fullsized":"2023-09-23T15:13:40.459125+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":149,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442583+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:13:40.442583+0000","last_peered":"2023-09-23T15:13:40.442583+0000","last_clean":"2023-09-23T15:13:40.442583+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:13:40.442583+0000","last_undegraded":"2023-09-23T15:13:40.442583+0000","last_fullsized":"2023-09-23T15:13:40.442583+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"200'7303","reported_seq":7340,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:54.287662+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:13:54.287662+0000","last_peered":"2023-09-23T15:13:54.287662+0000","last_clean":"2023-09-23T15:13:54.287662+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:13:54.287662+0000","last_undegraded":"2023-09-23T15:13:54.287662+0000","last_fullsized":"2023-09-23T15:13:54.287662+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7303,"log_dups_size":0,"ondisk_log_size":7303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327614464,"num_objects":4999,"num_object_clones":0,"num_object_copies":14997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7303,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.444820+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:13:40.444820+0000","last_peered":"2023-09-23T15:13:40.444820+0000","last_clean":"2023-09-23T15:13:40.444820+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:13:40.444820+0000","last_undegraded":"2023-09-23T15:13:40.444820+0000","last_fullsized":"2023-09-23T15:13:40.444820+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455779+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:13:40.455779+0000","last_peered":"2023-09-23T15:13:40.455779+0000","last_clean":"2023-09-23T15:13:40.455779+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:13:40.455779+0000","last_undegraded":"2023-09-23T15:13:40.455779+0000","last_fullsized":"2023-09-23T15:13:40.455779+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":659,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.469415+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:13:40.469415+0000","last_peered":"2023-09-23T15:13:40.469415+0000","last_clean":"2023-09-23T15:13:40.469415+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:13:40.469415+0000","last_undegraded":"2023-09-23T15:13:40.469415+0000","last_fullsized":"2023-09-23T15:13:40.469415+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":179,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455705+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:13:40.455705+0000","last_peered":"2023-09-23T15:13:40.455705+0000","last_clean":"2023-09-23T15:13:40.455705+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:13:40.455705+0000","last_undegraded":"2023-09-23T15:13:40.455705+0000","last_fullsized":"2023-09-23T15:13:40.455705+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"200'7595","reported_seq":7638,"reported_epoch":200,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:54.156554+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:13:54.156554+0000","last_peered":"2023-09-23T15:13:54.156554+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:13:54.156554+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"176'2300","ondisk_log_start":"176'2300","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5295,"log_dups_size":0,"ondisk_log_size":5295,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329973760,"num_objects":5035,"num_object_clones":0,"num_object_copies":15105,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5035,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7595,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":69}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":476,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442684+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:13:40.442684+0000","last_peered":"2023-09-23T15:13:40.442684+0000","last_clean":"2023-09-23T15:13:40.442684+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:13:40.442684+0000","last_undegraded":"2023-09-23T15:13:40.442684+0000","last_fullsized":"2023-09-23T15:13:40.442684+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":676,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.433667+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:13:40.433667+0000","last_peered":"2023-09-23T15:13:40.433667+0000","last_clean":"2023-09-23T15:13:40.433667+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:13:40.433667+0000","last_undegraded":"2023-09-23T15:13:40.433667+0000","last_fullsized":"2023-09-23T15:13:40.433667+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":162,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442765+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:13:40.442765+0000","last_peered":"2023-09-23T15:13:40.442765+0000","last_clean":"2023-09-23T15:13:40.442765+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:13:40.442765+0000","last_undegraded":"2023-09-23T15:13:40.442765+0000","last_fullsized":"2023-09-23T15:13:40.442765+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":141,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452053+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:13:40.452053+0000","last_peered":"2023-09-23T15:13:40.452053+0000","last_clean":"2023-09-23T15:13:40.452053+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:13:40.452053+0000","last_undegraded":"2023-09-23T15:13:40.452053+0000","last_fullsized":"2023-09-23T15:13:40.452053+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":151,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.442503+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:13:40.442503+0000","last_peered":"2023-09-23T15:13:40.442503+0000","last_clean":"2023-09-23T15:13:40.442503+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:13:40.442503+0000","last_undegraded":"2023-09-23T15:13:40.442503+0000","last_fullsized":"2023-09-23T15:13:40.442503+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":199,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.444825+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:13:40.444825+0000","last_peered":"2023-09-23T15:13:40.444825+0000","last_clean":"2023-09-23T15:13:40.444825+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:13:40.444825+0000","last_undegraded":"2023-09-23T15:13:40.444825+0000","last_fullsized":"2023-09-23T15:13:40.444825+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":178,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.447381+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:13:40.447381+0000","last_peered":"2023-09-23T15:13:40.447381+0000","last_clean":"2023-09-23T15:13:40.447381+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:13:40.447381+0000","last_undegraded":"2023-09-23T15:13:40.447381+0000","last_fullsized":"2023-09-23T15:13:40.447381+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":192,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452518+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:13:40.452518+0000","last_peered":"2023-09-23T15:13:40.452518+0000","last_clean":"2023-09-23T15:13:40.452518+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:13:40.452518+0000","last_undegraded":"2023-09-23T15:13:40.452518+0000","last_fullsized":"2023-09-23T15:13:40.452518+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":144,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451649+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:13:40.451649+0000","last_peered":"2023-09-23T15:13:40.451649+0000","last_clean":"2023-09-23T15:13:40.451649+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:13:40.451649+0000","last_undegraded":"2023-09-23T15:13:40.451649+0000","last_fullsized":"2023-09-23T15:13:40.451649+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":175,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.452109+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:13:40.452109+0000","last_peered":"2023-09-23T15:13:40.452109+0000","last_clean":"2023-09-23T15:13:40.452109+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:13:40.452109+0000","last_undegraded":"2023-09-23T15:13:40.452109+0000","last_fullsized":"2023-09-23T15:13:40.452109+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":170,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.455698+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:13:40.455698+0000","last_peered":"2023-09-23T15:13:40.455698+0000","last_clean":"2023-09-23T15:13:40.455698+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:13:40.455698+0000","last_undegraded":"2023-09-23T15:13:40.455698+0000","last_fullsized":"2023-09-23T15:13:40.455698+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":185,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.454929+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:13:40.454929+0000","last_peered":"2023-09-23T15:13:40.454929+0000","last_clean":"2023-09-23T15:13:40.454929+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:13:40.454929+0000","last_undegraded":"2023-09-23T15:13:40.454929+0000","last_fullsized":"2023-09-23T15:13:40.454929+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":147,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.443014+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:13:40.443014+0000","last_peered":"2023-09-23T15:13:40.443014+0000","last_clean":"2023-09-23T15:13:40.443014+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:13:40.443014+0000","last_undegraded":"2023-09-23T15:13:40.443014+0000","last_fullsized":"2023-09-23T15:13:40.443014+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":168,"reported_epoch":199,"state":"active+clean","last_fresh":"2023-09-23T15:13:40.451646+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:13:40.451646+0000","last_peered":"2023-09-23T15:13:40.451646+0000","last_clean":"2023-09-23T15:13:40.451646+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:13:40.451646+0000","last_undegraded":"2023-09-23T15:13:40.451646+0000","last_fullsized":"2023-09-23T15:13:40.451646+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":5308219416,"num_objects":80998,"num_object_clones":0,"num_object_copies":242994,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":26369,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80998,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119780,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3331,"num_bytes_recovered":218300416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14600187904,"data_stored":14600175688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106980,"ondisk_log_size":106980,"up":48,"acting":42,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201510,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463146,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2733188,"kb_used_data":2592956,"kb_used_omap":0,"kb_used_meta":140224,"kb_avail":91011964,"statfs":{"total":95995035648,"available":93196251136,"internally_reserved":0,"allocated":2655186944,"data_stored":2654526675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":143589376},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7080000000000002,"5min":6.3949999999999996,"15min":2.5},"min":{"1min":0.23400000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":41.491,"5min":108.39400000000001,"15min":108.39400000000001},"last":0.46600000000000003},{"interface":"front","average":{"1min":3.6779999999999999,"5min":6.5780000000000003,"15min":2.5870000000000002},"min":{"1min":0.28199999999999997,"5min":0.223,"15min":0.223},"max":{"1min":43.151000000000003,"5min":108.452,"15min":108.452},"last":1.3}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":3.371,"15min":3.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":1.093},{"interface":"front","average":{"1min":3.6070000000000002,"5min":3.6070000000000002,"15min":3.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":20.984000000000002,"5min":20.984000000000002,"15min":20.984000000000002},"last":12.387}]},{"osd":2,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":11.242000000000001,"15min":4.6829999999999998},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.255},"max":{"1min":4.8369999999999997,"5min":139.17500000000001,"15min":180.374},"last":0.66800000000000004},{"interface":"front","average":{"1min":2.1179999999999999,"5min":11.199,"15min":4.6870000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":7.2080000000000002,"5min":146.999,"15min":181.685},"last":0.995}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.2149999999999999,"15min":1.169},"min":{"1min":0.34499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9329999999999998,"5min":17.084,"15min":17.084},"last":6.1849999999999996},{"interface":"front","average":{"1min":3.4940000000000002,"5min":2.2240000000000002,"15min":1.1679999999999999},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.6769999999999996,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":5.9690000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:13:06 2023","interfaces":[{"interface":"back","average":{"1min":2.5990000000000002,"5min":1.9590000000000001,"15min":1.0069999999999999},"min":{"1min":0.26800000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":36.290999999999997,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.6509999999999998,"5min":2.1080000000000001,"15min":1.0940000000000001},"min":{"1min":0.32400000000000001,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":36.758000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.48399999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:13:06 2023","interfaces":[{"interface":"back","average":{"1min":6.9089999999999998,"5min":2.9380000000000002,"15min":1.359},"min":{"1min":0.307,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":131.90600000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":6.96,"5min":3.0390000000000001,"15min":1.427},"min":{"1min":0.35099999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":131.61000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.63500000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.671,"15min":0.98199999999999998},"min":{"1min":0.36099999999999999,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":3.7490000000000001,"5min":17,"15min":17},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.4750000000000001,"15min":0.95599999999999996},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.496,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.505}]}]},{"osd":1,"up_from":183,"seq":785979015183,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1361224,"kb_used_data":1248704,"kb_used_omap":6,"kb_used_meta":112505,"kb_avail":92383928,"statfs":{"total":95995035648,"available":94601142272,"internally_reserved":0,"allocated":1278672896,"data_stored":1277723239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":115205195},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,4,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.8039999999999998,"15min":3.8039999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.626000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":1.855},{"interface":"front","average":{"1min":4.6100000000000003,"5min":4.6100000000000003,"15min":4.6100000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.131,"5min":16.131,"15min":16.131},"last":3.0190000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":13.843,"5min":13.843,"15min":13.843},"last":2.9729999999999999},{"interface":"front","average":{"1min":4.2229999999999999,"5min":4.2229999999999999,"15min":4.2229999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.658000000000001,"5min":16.658000000000001,"15min":16.658000000000001},"last":1.883}]},{"osd":3,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":123.426,"5min":123.426,"15min":123.426},"last":2.6299999999999999},{"interface":"front","average":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":123.32899999999999,"5min":123.32899999999999,"15min":123.32899999999999},"last":3.2690000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":1.794},{"interface":"front","average":{"1min":3.5579999999999998,"5min":3.5579999999999998,"15min":3.5579999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.544,"5min":16.544,"15min":16.544},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":16.166,"5min":16.166,"15min":16.166},"last":3.206},{"interface":"front","average":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":1.9199999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":16.074000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":3.2389999999999999},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":7.7910000000000004,"5min":7.7910000000000004,"15min":7.7910000000000004},"last":1.948}]},{"osd":7,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.157,"15min":3.157},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":1.976},{"interface":"front","average":{"1min":3.3540000000000001,"5min":3.3540000000000001,"15min":3.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":16.594000000000001,"5min":16.594000000000001,"15min":16.594000000000001},"last":0.39800000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893233,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1544800,"kb_used_data":1309336,"kb_used_omap":8,"kb_used_meta":235319,"kb_avail":92200352,"statfs":{"total":95995035648,"available":94413160448,"internally_reserved":0,"allocated":1340760064,"data_stored":1339903909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":240967252},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":31.510999999999999,"5min":31.510999999999999,"15min":31.510999999999999},"last":2.5630000000000002},{"interface":"front","average":{"1min":4.71,"5min":4.71,"15min":4.71},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.378,"5min":31.378,"15min":31.378},"last":1.6379999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":3.992,"5min":4.923,"15min":1.929},"min":{"1min":0.189,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":13.157,"5min":122.20099999999999,"15min":122.20099999999999},"last":7.5810000000000004},{"interface":"front","average":{"1min":2.3029999999999999,"5min":4.2199999999999998,"15min":1.6850000000000001},"min":{"1min":0.22500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":14.662000000000001,"5min":122.071,"15min":122.071},"last":2.4950000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":12.659000000000001,"5min":5.0529999999999999,"15min":2.0529999999999999},"min":{"1min":0.28299999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":84.935000000000002,"5min":99.204999999999998,"15min":99.204999999999998},"last":5.1879999999999997},{"interface":"front","average":{"1min":12.394,"5min":4.1619999999999999,"15min":2.085},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":78.902000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":2.5350000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":2.0150000000000001,"5min":2.137,"15min":1.1919999999999999},"min":{"1min":0.32000000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.8079999999999998,"5min":11.989000000000001,"15min":11.989000000000001},"last":1.675},{"interface":"front","average":{"1min":6.1920000000000002,"5min":2.5299999999999998,"15min":1.276},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":99.168999999999997,"5min":99.168999999999997,"15min":99.168999999999997},"last":5.2480000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":5.9299999999999997,"5min":2.774,"15min":1.3899999999999999},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":109.369,"5min":109.369,"15min":109.369},"last":2.5990000000000002},{"interface":"front","average":{"1min":7.1959999999999997,"5min":2.9649999999999999,"15min":1.5149999999999999},"min":{"1min":0.38700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":109.387,"5min":109.387,"15min":109.387},"last":1.698}]},{"osd":6,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":6.2619999999999996,"5min":2.867,"15min":1.423},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":99.114999999999995,"5min":99.114999999999995,"15min":99.114999999999995},"last":5.2770000000000001},{"interface":"front","average":{"1min":5.4470000000000001,"5min":2.5539999999999998,"15min":1.3029999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":98.921999999999997,"5min":98.921999999999997,"15min":98.921999999999997},"last":2.6539999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:12:58 2023","interfaces":[{"interface":"back","average":{"1min":5.6859999999999999,"5min":2.7429999999999999,"15min":1.363},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":99.549999999999997,"5min":99.549999999999997,"15min":99.549999999999997},"last":2.6850000000000001},{"interface":"front","average":{"1min":1.9550000000000001,"5min":1.9350000000000001,"15min":1.161},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.468,"5min":9.109,"15min":9.109},"last":1.72}]}]},{"osd":2,"up_from":105,"seq":450971566180,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3245352,"kb_used_data":2948704,"kb_used_omap":11,"kb_used_meta":296628,"kb_avail":90499800,"statfs":{"total":95995035648,"available":92671795200,"internally_reserved":0,"allocated":3019472896,"data_stored":3018612937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":303747856},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3540000000000001,"5min":1.8620000000000001,"15min":0.88700000000000001},"min":{"1min":0.186,"5min":0.184,"15min":0.184},"max":{"1min":9.5299999999999994,"5min":30.402000000000001,"15min":30.402000000000001},"last":6.8529999999999998},{"interface":"front","average":{"1min":1.891,"5min":1.716,"15min":0.82899999999999996},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":9.3309999999999995,"5min":30.550000000000001,"15min":30.550000000000001},"last":5.8140000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":5.04,"5min":5.04,"15min":5.04},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":2.7530000000000001},{"interface":"front","average":{"1min":2.5489999999999999,"5min":2.5489999999999999,"15min":2.5489999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":7.6669999999999998},"last":5.9329999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:13:22 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":1.8420000000000001,"15min":1.1060000000000001},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":22.233000000000001,"5min":22.233000000000001,"15min":57.695},"last":7.9409999999999998},{"interface":"front","average":{"1min":3.968,"5min":1.895,"15min":1.1519999999999999},"min":{"1min":0.24399999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.254999999999999,"5min":22.254999999999999,"15min":58.387999999999998},"last":5.9720000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.4660000000000002,"15min":1.25},"min":{"1min":0.33800000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1370000000000005,"5min":52.795999999999999,"15min":52.795999999999999},"last":5.8780000000000001},{"interface":"front","average":{"1min":1.954,"5min":3.4020000000000001,"15min":1.571},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":9.5749999999999993,"5min":68.894999999999996,"15min":68.894999999999996},"last":6.2779999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":1.9179999999999999,"5min":2.7160000000000002,"15min":1.3240000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.4500000000000002,"5min":41.723999999999997,"15min":41.723999999999997},"last":2.7949999999999999},{"interface":"front","average":{"1min":1.782,"5min":2.5739999999999998,"15min":1.3049999999999999},"min":{"1min":0.35099999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.1859999999999999,"5min":39.822000000000003,"15min":39.822000000000003},"last":5.8449999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.653,"5min":2.774,"15min":1.3999999999999999},"min":{"1min":0.41799999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":14.917999999999999,"5min":37.834000000000003,"15min":37.834000000000003},"last":5.96},{"interface":"front","average":{"1min":1.9370000000000001,"5min":2.4079999999999999,"15min":1.2689999999999999},"min":{"1min":0.434,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":9.7289999999999992,"5min":37.749000000000002,"15min":37.749000000000002},"last":6.2400000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:13:00 2023","interfaces":[{"interface":"back","average":{"1min":2.0019999999999998,"5min":2.1280000000000001,"15min":1.1719999999999999},"min":{"1min":0.36099999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.4939999999999998,"5min":9.2680000000000007,"15min":9.2680000000000007},"last":2.8250000000000002},{"interface":"front","average":{"1min":2.2440000000000002,"5min":2.1619999999999999,"15min":1.1779999999999999},"min":{"1min":0.436,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":9.2560000000000002,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":5.9000000000000004}]}]},{"osd":3,"up_from":110,"seq":472446402654,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3801012,"kb_used_data":3566828,"kb_used_omap":3,"kb_used_meta":234172,"kb_avail":89944140,"statfs":{"total":95995035648,"available":92102799360,"internally_reserved":0,"allocated":3652431872,"data_stored":3651614382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":239792297},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":3.694,"15min":1.518},"min":{"1min":0.26000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.276,"5min":79.611000000000004,"15min":79.611000000000004},"last":2.1280000000000001},{"interface":"front","average":{"1min":3.0499999999999998,"5min":3.831,"15min":1.5940000000000001},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.0809999999999995,"5min":79.656000000000006,"15min":79.656000000000006},"last":0.57099999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":4.5860000000000003,"15min":4.5860000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.315000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":2.8999999999999999},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.427,"5min":14.427,"15min":14.427},"last":0.89900000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":7.5570000000000004,"15min":2.952},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":10.467000000000001,"5min":109.077,"15min":109.077},"last":1.0349999999999999},{"interface":"front","average":{"1min":4.0830000000000002,"5min":8.4160000000000004,"15min":3.1589999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.265000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":1.52}]},{"osd":4,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8009999999999999,"15min":1.026},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0399999999999991,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.9670000000000001,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6580000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":3.1019999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.6669999999999998,"15min":1.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.9450000000000003,"5min":56.960999999999999,"15min":56.960999999999999},"last":6.2770000000000001},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.9209999999999998,"15min":1.5389999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.593,"5min":56.981999999999999,"15min":56.981999999999999},"last":3.0529999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.3069999999999999,"15min":1.2290000000000001},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.4749999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.5990000000000002,"15min":1.448},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6890000000000001,"5min":19.355,"15min":19.355},"last":2.855}]},{"osd":7,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.6240000000000001,"15min":1.448},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7489999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":2.9790000000000001},{"interface":"front","average":{"1min":2.8100000000000001,"5min":1.827,"15min":1.0680000000000001},"min":{"1min":0.47899999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.406000000000001,"5min":13.403,"15min":13.403},"last":0.871}]}]},{"osd":4,"up_from":31,"seq":133143986419,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1812672,"kb_used_data":1627192,"kb_used_omap":0,"kb_used_meta":185472,"kb_avail":91932480,"statfs":{"total":95995035648,"available":94138859520,"internally_reserved":0,"allocated":1666244608,"data_stored":1665587914,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":189923328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":3.806,"5min":4.202,"15min":2.2370000000000001},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":63.488,"5min":108.148,"15min":122.69199999999999},"last":0.33400000000000002},{"interface":"front","average":{"1min":3.7050000000000001,"5min":4.2679999999999998,"15min":2.2639999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":55.951999999999998,"5min":108.124,"15min":122.745},"last":1.329}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":4.5490000000000004,"15min":4.5490000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":53.832000000000001,"5min":53.832000000000001,"15min":53.832000000000001},"last":3.2850000000000001},{"interface":"front","average":{"1min":5.077,"5min":5.077,"15min":5.077},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.078000000000003,"5min":53.078000000000003,"15min":53.078000000000003},"last":3.1680000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:12:54 2023","interfaces":[{"interface":"back","average":{"1min":10.869999999999999,"5min":6.7279999999999998,"15min":2.633},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":148.08000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":4.9660000000000002},{"interface":"front","average":{"1min":10.5,"5min":6.9420000000000002,"15min":2.8010000000000002},"min":{"1min":0.33000000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":147.88900000000001,"5min":147.88900000000001,"15min":147.88900000000001},"last":1.268}]},{"osd":3,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":20.986999999999998,"5min":7.718,"15min":3.0219999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":135.607,"5min":163.97499999999999,"15min":163.97499999999999},"last":1.403},{"interface":"front","average":{"1min":20.789000000000001,"5min":7.1319999999999997,"15min":2.8220000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":135.33600000000001,"5min":164.447,"15min":164.447},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:38 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.3660000000000001,"15min":1.198},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.998,"5min":28.891999999999999,"15min":28.891999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.8340000000000001,"15min":1.0449999999999999},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9489999999999998,"5min":32.100999999999999,"15min":32.100999999999999},"last":0.32300000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:13:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.9449999999999998,"15min":1.73},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.927,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.433},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.5070000000000001,"15min":1.5980000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5659999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.36399999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.1059999999999999,"15min":1.1379999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":4.5869999999999997,"5min":18.23,"15min":18.23},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.78800000000000003,"5min":2.0270000000000001,"15min":1.149},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.879,"5min":14.653,"15min":14.653},"last":0.56899999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724782,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1801784,"kb_used_data":1619184,"kb_used_omap":0,"kb_used_meta":182592,"kb_avail":91943368,"statfs":{"total":95995035648,"available":94150008832,"internally_reserved":0,"allocated":1658044416,"data_stored":1657391701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":186974208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:12:57 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.3540000000000001,"15min":1.155},"min":{"1min":0.222,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":16.981999999999999,"5min":24.962,"15min":24.962},"last":0.248},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.1240000000000001,"15min":1.073},"min":{"1min":0.34399999999999997,"5min":0.222,"15min":0.222},"max":{"1min":18.814,"5min":25.317,"15min":25.317},"last":1.3839999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.673,"15min":1.673},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":4.2510000000000003},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.717000000000001,"5min":11.717000000000001,"15min":11.717000000000001},"last":2.2130000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:12:57 2023","interfaces":[{"interface":"back","average":{"1min":1.8680000000000001,"5min":4.5380000000000003,"15min":1.895},"min":{"1min":0.27600000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.3630000000000004,"5min":97.087999999999994,"15min":97.087999999999994},"last":3.7109999999999999},{"interface":"front","average":{"1min":1.363,"5min":4.1970000000000001,"15min":1.8},"min":{"1min":0.32700000000000001,"5min":0.254,"15min":0.254},"max":{"1min":7.556,"5min":97.305999999999997,"15min":97.305999999999997},"last":2.8109999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.2109999999999999,"15min":1.5129999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.7770000000000001,"5min":53.088999999999999,"15min":53.088999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":3.093,"5min":2.8260000000000001,"15min":1.4350000000000001},"min":{"1min":0.47299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.121,"5min":53.764000000000003,"15min":53.764000000000003},"last":10.401}]},{"osd":4,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.298,"15min":1.163},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7669999999999999,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.622,"5min":2.5449999999999999,"15min":1.242},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.1850000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.33100000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:13:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":2.0870000000000002,"15min":1.0780000000000001},"min":{"1min":0.30199999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.8959999999999999,"5min":15.196,"15min":15.196},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.7549999999999999,"15min":1.006},"min":{"1min":0.34999999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":3.8820000000000001,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.3519999999999999,"15min":1.2270000000000001},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.4260000000000002,"5min":15.301,"15min":15.301},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.2410000000000001,"15min":1.2070000000000001},"min":{"1min":0.40999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":3.0249999999999999,"5min":15.762,"15min":15.762},"last":0.40600000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1326125056,"data_stored":1326120984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":673779712,"data_stored":673775640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":3009609728,"data_stored":3009609728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3641180160,"data_stored":3641180160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1653276672,"data_stored":1653276672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1649479680,"data_stored":1649475608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2646736896,"data_stored":2646736896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:13:57.747 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:13:57.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:13:57.827 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:57.838 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[222878]: from='client.55120 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:57.838 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[222878]: pgmap v517: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:57.838 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26372/243075 objects degraded (10.849%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:57.838 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[215646]: from='client.55120 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:57.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[215646]: pgmap v517: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:57.839 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26372/243075 objects degraded (10.849%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:57.879 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:13:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:57.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:13:57.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:58.094 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:13:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:13:58] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.43.0" 2023-09-23T15:13:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:57 smithi027 ceph-mon[195233]: from='client.55120 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:13:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:57 smithi027 ceph-mon[195233]: pgmap v517: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26372/243075 objects degraded (10.849%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:13:58.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:58.223 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:13:58.223 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:13:58.388 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:58.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:58.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:13:58.525 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:58.584 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:13:58.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:58.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:13:58.692 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:13:58.728 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:58.789 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:13:58.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:58.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:13:58.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:59.003 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:59.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:59.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:59.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[215646]: pgmap v518: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:59.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:59.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:59.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:13:59 smithi017 ceph-mon[222878]: pgmap v518: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:59.459 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:13:59.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:59.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:13:59.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:13:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:13:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:13:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:13:59 smithi027 ceph-mon[195233]: pgmap v518: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.9 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 26372/243075 objects degraded (10.849%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T15:13:59.616 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:13:59.627 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:13:59.682 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:13:59.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:13:59.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:13:59.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:13:59.819 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:00.142 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:00.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:00.162 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:00.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:00.297 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:00.311 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:00.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:14:00.350 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:00.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:00.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:00.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:00.624 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:00.859 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:00.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:00.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:00.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:00.998 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:01.057 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:01.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:01.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:01.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:01.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:01.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:01.278 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:01.296 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:01.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:01.363 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:01.402 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:01.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:01.473 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:14:01.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:01 smithi017 ceph-mon[215646]: pgmap v519: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:01.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:01 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1946608064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:01.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:01 smithi017 ceph-mon[222878]: pgmap v519: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:01.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:01 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1946608064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:01.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:01 smithi027 ceph-mon[195233]: pgmap v519: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:01.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:01 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1946608064' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:02.242 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:02.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:02.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:02.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:02.430 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:02.702 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26088/234870 objects degraded (11.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:02.702 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26088/234870 objects degraded (11.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:02.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26088/234870 objects degraded (11.107%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:02.863 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:02.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:02.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:02.890 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:02.949 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:02.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:03.098 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:03.387 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:03.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:03.501 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:03.540 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:03.593 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:03.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:03.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:03.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:03.818 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:04.053 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:04.053 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26088/234870 objects degraded (11.107%), 8 pgs degraded, 8 pgs undersized","count":16},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":651,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":201,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":101},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":78290,"data_bytes":5133736156,"bytes_used":16114081792,"bytes_avail":655851167744,"bytes_total":671965249536,"degraded_objects":26088,"degraded_total":234870,"degraded_ratio":0.1110742112658066,"recovering_objects_per_sec":54,"recovering_bytes_per_sec":3573725,"recovering_keys_per_sec":0,"num_objects_recovered":546,"num_bytes_recovered":35782656,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":553},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (5m)\n [=========================...] (remaining: 23s)","progress":0.92660552263259888,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:14:04.142 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[222878]: pgmap v520: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:04.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[215646]: pgmap v520: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:04.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:03 smithi027 ceph-mon[195233]: pgmap v520: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 26088/234870 objects degraded (11.107%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:04.615 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:04.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:04.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:04.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:04.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:04.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:14:05.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3295871638' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:05.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:14:05.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:14:05.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:05.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3295871638' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:05.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:14:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:14:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:05.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3295871638' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:05.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:14:05.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:14:05.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:05.555 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:05.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:05.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:05.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:05.705 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:14:05.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:14:05.821 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:06.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:06.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:06.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:06.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:05 smithi017 ceph-mon[215646]: pgmap v521: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 26088/231333 objects degraded (11.277%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:06.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:05 smithi017 ceph-mon[222878]: pgmap v521: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 26088/231333 objects degraded (11.277%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:06.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:06.280 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:05 smithi027 ceph-mon[195233]: pgmap v521: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.7 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 585 op/s; 26088/231333 objects degraded (11.277%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:06.282 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:06.352 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:06.356 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:06.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:06.490 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:06.553 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:06.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:06.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:06.687 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:06.751 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:06.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:06.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:06.886 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:06.936 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:06.942 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:06.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:06.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:07.008 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:07.018 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:14:07.018 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:14:07.018 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:14:07.018 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:14:07.018 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:14:07.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:07.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:07.288 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:06 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26088/231333 objects degraded (11.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:07.289 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:06 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26088/231333 objects degraded (11.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:06 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26088/231333 objects degraded (11.277%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:07.441 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:07.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:07.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:07.579 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:07.633 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:07.639 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:07.740 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:07.808 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:08.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:07 smithi017 ceph-mon[222878]: pgmap v522: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 25791/226959 objects degraded (11.364%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:08.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:07 smithi017 ceph-mon[215646]: pgmap v522: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 25791/226959 objects degraded (11.364%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:08.198 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:08] "GET /metrics HTTP/1.1" 200 36327 "" "Prometheus/2.43.0" 2023-09-23T15:14:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:07 smithi027 ceph-mon[195233]: pgmap v522: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 25791/226959 objects degraded (11.364%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T15:14:08.377 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:08.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:08.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:08.518 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:08.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:14:08.579 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:08.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:08.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:08.718 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:08.776 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:08.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:08.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:08.884 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:08.915 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:08.967 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:08.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:08.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:09.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3778782135' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.024 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:14:09.073 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:09.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3778782135' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.106 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:09.151 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:09.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:09.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:09.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3778782135' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.345 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:09.352 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: pgmap v523: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 25791/226959 objects degraded (11.364%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T15:14:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[222878]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: pgmap v523: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 25791/226959 objects degraded (11.364%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T15:14:10.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:10 smithi017 ceph-mon[215646]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: pgmap v523: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.6 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 447 op/s; 25791/226959 objects degraded (11.364%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T15:14:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: osdmap e202: 8 total, 8 up, 7 in 2023-09-23T15:14:10.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:10 smithi027 ceph-mon[195233]: failed to encode map e202 with expected crc 2023-09-23T15:14:10.529 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:10.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:10.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:10.770 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:10.913 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:10.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:11.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:11.067 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:11.076 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:11 smithi017 ceph-mon[215646]: pgmap v525: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:11.084 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:11 smithi017 ceph-mon[222878]: pgmap v525: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:11.133 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:11.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:11.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:11.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:11 smithi027 ceph-mon[195233]: pgmap v525: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:11.367 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:11.525 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:11.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:11.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:11.594 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:11.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:11.642 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:11.716 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:11.740 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:11.755 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:11.811 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:11.847 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:11.889 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:11.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:11.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:11.899 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:11.933 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:11.965 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:11.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.000 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.001 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.001 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:14:12.031 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:12.087 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:12.094 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:12.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:12.100 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:12.105 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:14:12.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:12.292 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3118520356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:12.292 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25394/218694 objects degraded (11.612%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:12.293 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3118520356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:12.293 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25394/218694 objects degraded (11.612%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:12.302 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:12.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3118520356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:12.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25394/218694 objects degraded (11.612%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:12.818 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:12.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:12.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:12.865 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:14:12.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:12.961 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:13 smithi017 ceph-mon[215646]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:13 smithi017 ceph-mon[215646]: pgmap v526: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:13.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:13 smithi017 ceph-mon[222878]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:13.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:13 smithi017 ceph-mon[222878]: pgmap v526: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:13.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:13 smithi027 ceph-mon[195233]: from='client.35352 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:13.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:13 smithi027 ceph-mon[195233]: pgmap v526: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 25394/218694 objects degraded (11.612%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:13.748 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":525,"stamp":"2023-09-23T15:14:10.346569+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4780366044,"num_objects":72898,"num_object_clones":0,"num_object_copies":218694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25394,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72898,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":128935,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4368,"num_bytes_recovered":287742119,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114620,"ondisk_log_size":114620,"up":327,"acting":319,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":14758576,"kb_used_data":13322296,"kb_used_omap":30,"kb_used_meta":1436001,"kb_avail":641457488,"statfs":{"total":671965249536,"available":656852467712,"internally_reserved":0,"allocated":13642031104,"data_stored":13636538526,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1470465624},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,5,0,5,6,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-353370112,"num_objects":-5392,"num_object_clones":0,"num_object_copies":-16176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-694,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5392,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":694,"num_bytes_recovered":45481984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005970"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":196,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.747662+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:13:55.747662+0000","last_peered":"2023-09-23T15:13:55.747662+0000","last_clean":"2023-09-23T15:13:55.747662+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:13:55.747662+0000","last_undegraded":"2023-09-23T15:13:55.747662+0000","last_fullsized":"2023-09-23T15:13:55.747662+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":130,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.824294+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:13:55.824294+0000","last_peered":"2023-09-23T15:13:55.824294+0000","last_clean":"2023-09-23T15:13:55.824294+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:13:55.824294+0000","last_undegraded":"2023-09-23T15:13:55.824294+0000","last_fullsized":"2023-09-23T15:13:55.824294+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":77,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.948338+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:13:44.948338+0000","last_peered":"2023-09-23T15:13:44.948338+0000","last_clean":"2023-09-23T15:13:44.948338+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:13:44.948338+0000","last_undegraded":"2023-09-23T15:13:44.948338+0000","last_fullsized":"2023-09-23T15:13:44.948338+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795336+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:13:55.795336+0000","last_peered":"2023-09-23T15:13:55.795336+0000","last_clean":"2023-09-23T15:13:55.795336+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:13:55.795336+0000","last_undegraded":"2023-09-23T15:13:55.795336+0000","last_fullsized":"2023-09-23T15:13:55.795336+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":199,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759145+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:13:55.759145+0000","last_peered":"2023-09-23T15:13:55.759145+0000","last_clean":"2023-09-23T15:13:55.759145+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:13:55.759145+0000","last_undegraded":"2023-09-23T15:13:55.759145+0000","last_fullsized":"2023-09-23T15:13:55.759145+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":176,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.768531+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:13:55.768531+0000","last_peered":"2023-09-23T15:13:55.768531+0000","last_clean":"2023-09-23T15:13:55.768531+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:13:55.768531+0000","last_undegraded":"2023-09-23T15:13:55.768531+0000","last_fullsized":"2023-09-23T15:13:55.768531+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":145,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.796143+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:13:55.796143+0000","last_peered":"2023-09-23T15:13:55.796143+0000","last_clean":"2023-09-23T15:13:55.796143+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:13:55.796143+0000","last_undegraded":"2023-09-23T15:13:55.796143+0000","last_fullsized":"2023-09-23T15:13:55.796143+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759233+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:13:55.759233+0000","last_peered":"2023-09-23T15:13:55.759233+0000","last_clean":"2023-09-23T15:13:55.759233+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:13:55.759233+0000","last_undegraded":"2023-09-23T15:13:55.759233+0000","last_fullsized":"2023-09-23T15:13:55.759233+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794849+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:13:55.794849+0000","last_peered":"2023-09-23T15:13:55.794849+0000","last_clean":"2023-09-23T15:13:55.794849+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:13:55.794849+0000","last_undegraded":"2023-09-23T15:13:55.794849+0000","last_fullsized":"2023-09-23T15:13:55.794849+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":192,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.758956+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:13:55.758956+0000","last_peered":"2023-09-23T15:13:55.758956+0000","last_clean":"2023-09-23T15:13:55.758956+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:13:55.758956+0000","last_undegraded":"2023-09-23T15:13:55.758956+0000","last_fullsized":"2023-09-23T15:13:55.758956+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":124,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759900+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:13:55.759900+0000","last_peered":"2023-09-23T15:13:55.759900+0000","last_clean":"2023-09-23T15:13:55.759900+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:13:55.759900+0000","last_undegraded":"2023-09-23T15:13:55.759900+0000","last_fullsized":"2023-09-23T15:13:55.759900+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":64,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:59.405731+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:13:59.405731+0000","last_peered":"2023-09-23T15:13:59.405731+0000","last_clean":"2023-09-23T15:13:59.405731+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:13:59.405731+0000","last_undegraded":"2023-09-23T15:13:59.405731+0000","last_fullsized":"2023-09-23T15:13:59.405731+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.947931+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:13:44.947931+0000","last_peered":"2023-09-23T15:13:44.947931+0000","last_clean":"2023-09-23T15:13:44.947931+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:13:44.947931+0000","last_undegraded":"2023-09-23T15:13:44.947931+0000","last_fullsized":"2023-09-23T15:13:44.947931+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":186,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746669+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:13:55.746669+0000","last_peered":"2023-09-23T15:13:55.746669+0000","last_clean":"2023-09-23T15:13:55.746669+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:13:55.746669+0000","last_undegraded":"2023-09-23T15:13:55.746669+0000","last_fullsized":"2023-09-23T15:13:55.746669+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746782+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:13:55.746782+0000","last_peered":"2023-09-23T15:13:55.746782+0000","last_clean":"2023-09-23T15:13:55.746782+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:13:55.746782+0000","last_undegraded":"2023-09-23T15:13:55.746782+0000","last_fullsized":"2023-09-23T15:13:55.746782+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794044+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:13:55.794044+0000","last_peered":"2023-09-23T15:13:55.794044+0000","last_clean":"2023-09-23T15:13:55.794044+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:13:55.794044+0000","last_undegraded":"2023-09-23T15:13:55.794044+0000","last_fullsized":"2023-09-23T15:13:55.794044+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":64,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746870+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:13:55.746870+0000","last_peered":"2023-09-23T15:13:55.746870+0000","last_clean":"2023-09-23T15:13:55.746870+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:13:55.746870+0000","last_undegraded":"2023-09-23T15:13:55.746870+0000","last_fullsized":"2023-09-23T15:13:55.746870+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":192,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.825228+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:13:55.825228+0000","last_peered":"2023-09-23T15:13:55.825228+0000","last_clean":"2023-09-23T15:13:55.825228+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:13:55.825228+0000","last_undegraded":"2023-09-23T15:13:55.825228+0000","last_fullsized":"2023-09-23T15:13:55.825228+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794482+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:13:55.794482+0000","last_peered":"2023-09-23T15:13:55.794482+0000","last_clean":"2023-09-23T15:13:55.794482+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:13:55.794482+0000","last_undegraded":"2023-09-23T15:13:55.794482+0000","last_fullsized":"2023-09-23T15:13:55.794482+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":166,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:59.404565+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:13:59.404565+0000","last_peered":"2023-09-23T15:13:59.404565+0000","last_clean":"2023-09-23T15:13:59.404565+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:13:59.404565+0000","last_undegraded":"2023-09-23T15:13:59.404565+0000","last_fullsized":"2023-09-23T15:13:59.404565+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.761659+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:13:55.761659+0000","last_peered":"2023-09-23T15:13:55.761659+0000","last_clean":"2023-09-23T15:13:55.761659+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:13:55.761659+0000","last_undegraded":"2023-09-23T15:13:55.761659+0000","last_fullsized":"2023-09-23T15:13:55.761659+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.793901+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:13:55.793901+0000","last_peered":"2023-09-23T15:13:55.793901+0000","last_clean":"2023-09-23T15:13:55.793901+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:13:55.793901+0000","last_undegraded":"2023-09-23T15:13:55.793901+0000","last_fullsized":"2023-09-23T15:13:55.793901+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":48,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746863+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:13:55.746863+0000","last_peered":"2023-09-23T15:13:55.746863+0000","last_clean":"2023-09-23T15:13:55.746863+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:13:55.746863+0000","last_undegraded":"2023-09-23T15:13:55.746863+0000","last_fullsized":"2023-09-23T15:13:55.746863+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795417+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:13:55.795417+0000","last_peered":"2023-09-23T15:13:55.795417+0000","last_clean":"2023-09-23T15:13:55.795417+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:13:55.795417+0000","last_undegraded":"2023-09-23T15:13:55.795417+0000","last_fullsized":"2023-09-23T15:13:55.795417+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.797093+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:13:55.797093+0000","last_peered":"2023-09-23T15:13:55.797093+0000","last_clean":"2023-09-23T15:13:55.797093+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:13:55.797093+0000","last_undegraded":"2023-09-23T15:13:55.797093+0000","last_fullsized":"2023-09-23T15:13:55.797093+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":168,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.767736+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:13:55.767736+0000","last_peered":"2023-09-23T15:13:55.767736+0000","last_clean":"2023-09-23T15:13:55.767736+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:13:55.767736+0000","last_undegraded":"2023-09-23T15:13:55.767736+0000","last_fullsized":"2023-09-23T15:13:55.767736+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":192,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.828695+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:13:55.828695+0000","last_peered":"2023-09-23T15:13:55.828695+0000","last_clean":"2023-09-23T15:13:55.828695+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:13:55.828695+0000","last_undegraded":"2023-09-23T15:13:55.828695+0000","last_fullsized":"2023-09-23T15:13:55.828695+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"202'7937","reported_seq":7976,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.622687+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:14:09.622687+0000","last_peered":"2023-09-23T15:14:09.622687+0000","last_clean":"2023-09-23T15:14:09.622687+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:14:09.622687+0000","last_undegraded":"2023-09-23T15:14:09.622687+0000","last_fullsized":"2023-09-23T15:14:09.622687+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7937,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795632+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:13:55.795632+0000","last_peered":"2023-09-23T15:13:55.795632+0000","last_clean":"2023-09-23T15:13:55.795632+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:13:55.795632+0000","last_undegraded":"2023-09-23T15:13:55.795632+0000","last_fullsized":"2023-09-23T15:13:55.795632+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":76,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.749937+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:13:55.749937+0000","last_peered":"2023-09-23T15:13:55.749937+0000","last_clean":"2023-09-23T15:13:55.749937+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:13:55.749937+0000","last_undegraded":"2023-09-23T15:13:55.749937+0000","last_fullsized":"2023-09-23T15:13:55.749937+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":153,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.758687+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:13:55.758687+0000","last_peered":"2023-09-23T15:13:55.758687+0000","last_clean":"2023-09-23T15:13:55.758687+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:13:55.758687+0000","last_undegraded":"2023-09-23T15:13:55.758687+0000","last_fullsized":"2023-09-23T15:13:55.758687+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"202'8125","reported_seq":8174,"reported_epoch":202,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:09.426489+0000","last_change":"2023-09-23T15:12:43.308627+0000","last_active":"2023-09-23T15:14:09.426489+0000","last_peered":"2023-09-23T15:14:09.426489+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:14:09.426489+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"175'1700","ondisk_log_start":"175'1700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297598976,"num_objects":4541,"num_object_clones":0,"num_object_copies":13623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4997,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8125,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-456}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":469,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:55.794598+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:13:55.794598+0000","last_peered":"2023-09-23T15:13:55.794598+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:13:55.794598+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":76,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.747337+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:13:55.747337+0000","last_peered":"2023-09-23T15:13:55.747337+0000","last_clean":"2023-09-23T15:13:55.747337+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:13:55.747337+0000","last_undegraded":"2023-09-23T15:13:55.747337+0000","last_fullsized":"2023-09-23T15:13:55.747337+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746716+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:13:55.746716+0000","last_peered":"2023-09-23T15:13:55.746716+0000","last_clean":"2023-09-23T15:13:55.746716+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:13:55.746716+0000","last_undegraded":"2023-09-23T15:13:55.746716+0000","last_fullsized":"2023-09-23T15:13:55.746716+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"201'7892","reported_seq":7932,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:14:08.495317+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:14:08.495317+0000","last_peered":"2023-09-23T15:14:08.495317+0000","last_clean":"2023-09-23T15:14:08.495317+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:14:08.495317+0000","last_undegraded":"2023-09-23T15:14:08.495317+0000","last_fullsized":"2023-09-23T15:14:08.495317+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7892,"log_dups_size":0,"ondisk_log_size":7892,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298188800,"num_objects":4550,"num_object_clones":0,"num_object_copies":13650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7892,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795856+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:13:55.795856+0000","last_peered":"2023-09-23T15:13:55.795856+0000","last_clean":"2023-09-23T15:13:55.795856+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:13:55.795856+0000","last_undegraded":"2023-09-23T15:13:55.795856+0000","last_fullsized":"2023-09-23T15:13:55.795856+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":62,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.761596+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:13:55.761596+0000","last_peered":"2023-09-23T15:13:55.761596+0000","last_clean":"2023-09-23T15:13:55.761596+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:13:55.761596+0000","last_undegraded":"2023-09-23T15:13:55.761596+0000","last_fullsized":"2023-09-23T15:13:55.761596+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":190,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.768324+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:13:55.768324+0000","last_peered":"2023-09-23T15:13:55.768324+0000","last_clean":"2023-09-23T15:13:55.768324+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:13:55.768324+0000","last_undegraded":"2023-09-23T15:13:55.768324+0000","last_fullsized":"2023-09-23T15:13:55.768324+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"202'8063","reported_seq":8104,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.346735+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:14:09.346735+0000","last_peered":"2023-09-23T15:14:09.346735+0000","last_clean":"2023-09-23T15:14:09.346735+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:14:09.346735+0000","last_undegraded":"2023-09-23T15:14:09.346735+0000","last_fullsized":"2023-09-23T15:14:09.346735+0000","mapping_epoch":171,"log_start":"177'2500","ondisk_log_start":"177'2500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5563,"log_dups_size":0,"ondisk_log_size":5563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301531136,"num_objects":4601,"num_object_clones":0,"num_object_copies":13803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8063,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795057+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:13:55.795057+0000","last_peered":"2023-09-23T15:13:55.795057+0000","last_clean":"2023-09-23T15:13:55.795057+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:13:55.795057+0000","last_undegraded":"2023-09-23T15:13:55.795057+0000","last_fullsized":"2023-09-23T15:13:55.795057+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":161,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.762350+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:55.762350+0000","last_peered":"2023-09-23T15:13:55.762350+0000","last_clean":"2023-09-23T15:13:55.762350+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:13:55.762350+0000","last_undegraded":"2023-09-23T15:13:55.762350+0000","last_fullsized":"2023-09-23T15:13:55.762350+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":161,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794155+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:13:55.794155+0000","last_peered":"2023-09-23T15:13:55.794155+0000","last_clean":"2023-09-23T15:13:55.794155+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:13:55.794155+0000","last_undegraded":"2023-09-23T15:13:55.794155+0000","last_fullsized":"2023-09-23T15:13:55.794155+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"202'8214","reported_seq":8259,"reported_epoch":202,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:09.325889+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:14:09.325889+0000","last_peered":"2023-09-23T15:14:09.325889+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:14:09.325889+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"175'700","ondisk_log_start":"175'700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7514,"log_dups_size":0,"ondisk_log_size":7514,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307888128,"num_objects":4698,"num_object_clones":0,"num_object_copies":14094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8214,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-312}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.793891+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:13:55.793891+0000","last_peered":"2023-09-23T15:13:55.793891+0000","last_clean":"2023-09-23T15:13:55.793891+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:13:55.793891+0000","last_undegraded":"2023-09-23T15:13:55.793891+0000","last_fullsized":"2023-09-23T15:13:55.793891+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.828695+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:13:55.828695+0000","last_peered":"2023-09-23T15:13:55.828695+0000","last_clean":"2023-09-23T15:13:55.828695+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:13:55.828695+0000","last_undegraded":"2023-09-23T15:13:55.828695+0000","last_fullsized":"2023-09-23T15:13:55.828695+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746751+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:13:55.746751+0000","last_peered":"2023-09-23T15:13:55.746751+0000","last_clean":"2023-09-23T15:13:55.746751+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:13:55.746751+0000","last_undegraded":"2023-09-23T15:13:55.746751+0000","last_fullsized":"2023-09-23T15:13:55.746751+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"201'7958","reported_seq":7998,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:14:08.464129+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:14:08.464129+0000","last_peered":"2023-09-23T15:14:08.464129+0000","last_clean":"2023-09-23T15:14:08.464129+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:14:08.464129+0000","last_undegraded":"2023-09-23T15:14:08.464129+0000","last_fullsized":"2023-09-23T15:14:08.464129+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7958,"log_dups_size":0,"ondisk_log_size":7958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302776320,"num_objects":4620,"num_object_clones":0,"num_object_copies":13860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7958,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":495,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794396+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:13:55.794396+0000","last_peered":"2023-09-23T15:13:55.794396+0000","last_clean":"2023-09-23T15:13:55.794396+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:13:55.794396+0000","last_undegraded":"2023-09-23T15:13:55.794396+0000","last_fullsized":"2023-09-23T15:13:55.794396+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.768158+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:13:55.768158+0000","last_peered":"2023-09-23T15:13:55.768158+0000","last_clean":"2023-09-23T15:13:55.768158+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:13:55.768158+0000","last_undegraded":"2023-09-23T15:13:55.768158+0000","last_fullsized":"2023-09-23T15:13:55.768158+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":194,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746782+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:13:55.746782+0000","last_peered":"2023-09-23T15:13:55.746782+0000","last_clean":"2023-09-23T15:13:55.746782+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:13:55.746782+0000","last_undegraded":"2023-09-23T15:13:55.746782+0000","last_fullsized":"2023-09-23T15:13:55.746782+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"202'7979","reported_seq":8020,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.436641+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:14:09.436641+0000","last_peered":"2023-09-23T15:14:09.436641+0000","last_clean":"2023-09-23T15:14:09.436641+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:14:09.436641+0000","last_undegraded":"2023-09-23T15:14:09.436641+0000","last_fullsized":"2023-09-23T15:14:09.436641+0000","mapping_epoch":171,"log_start":"175'1500","ondisk_log_start":"175'1500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6479,"log_dups_size":0,"ondisk_log_size":6479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301400064,"num_objects":4599,"num_object_clones":0,"num_object_copies":13797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7979,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.793990+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:13:55.793990+0000","last_peered":"2023-09-23T15:13:55.793990+0000","last_clean":"2023-09-23T15:13:55.793990+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:13:55.793990+0000","last_undegraded":"2023-09-23T15:13:55.793990+0000","last_fullsized":"2023-09-23T15:13:55.793990+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":185,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746481+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:13:55.746481+0000","last_peered":"2023-09-23T15:13:55.746481+0000","last_clean":"2023-09-23T15:13:55.746481+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:13:55.746481+0000","last_undegraded":"2023-09-23T15:13:55.746481+0000","last_fullsized":"2023-09-23T15:13:55.746481+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.747408+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:13:55.747408+0000","last_peered":"2023-09-23T15:13:55.747408+0000","last_clean":"2023-09-23T15:13:55.747408+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:13:55.747408+0000","last_undegraded":"2023-09-23T15:13:55.747408+0000","last_fullsized":"2023-09-23T15:13:55.747408+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"202'7998","reported_seq":16663,"reported_epoch":202,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:09.698928+0000","last_change":"2023-09-23T15:12:43.243089+0000","last_active":"2023-09-23T15:14:09.698928+0000","last_peered":"2023-09-23T15:14:09.698928+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.190605+0000","last_became_peered":"2023-09-23T15:12:43.190605+0000","last_unstale":"2023-09-23T15:14:09.698928+0000","last_undegraded":"2023-09-23T15:12:42.959077+0000","last_fullsized":"2023-09-23T15:12:42.845806+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7998,"log_dups_size":0,"ondisk_log_size":7998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:41:00.861011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297074712,"num_objects":4534,"num_object_clones":0,"num_object_copies":13602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":592,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4534,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7998,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4308,"num_bytes_recovered":282329088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":3942}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.796110+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:13:55.796110+0000","last_peered":"2023-09-23T15:13:55.796110+0000","last_clean":"2023-09-23T15:13:55.796110+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:13:55.796110+0000","last_undegraded":"2023-09-23T15:13:55.796110+0000","last_fullsized":"2023-09-23T15:13:55.796110+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":193,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759135+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:13:55.759135+0000","last_peered":"2023-09-23T15:13:55.759135+0000","last_clean":"2023-09-23T15:13:55.759135+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:13:55.759135+0000","last_undegraded":"2023-09-23T15:13:55.759135+0000","last_fullsized":"2023-09-23T15:13:55.759135+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":77,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.948362+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:13:44.948362+0000","last_peered":"2023-09-23T15:13:44.948362+0000","last_clean":"2023-09-23T15:13:44.948362+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:13:44.948362+0000","last_undegraded":"2023-09-23T15:13:44.948362+0000","last_fullsized":"2023-09-23T15:13:44.948362+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"202'7995","reported_seq":8036,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.335129+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:14:09.335129+0000","last_peered":"2023-09-23T15:14:09.335129+0000","last_clean":"2023-09-23T15:14:09.335129+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:14:09.335129+0000","last_undegraded":"2023-09-23T15:14:09.335129+0000","last_fullsized":"2023-09-23T15:14:09.335129+0000","mapping_epoch":171,"log_start":"176'2400","ondisk_log_start":"176'2400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5595,"log_dups_size":0,"ondisk_log_size":5595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7995,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795945+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:13:55.795945+0000","last_peered":"2023-09-23T15:13:55.795945+0000","last_clean":"2023-09-23T15:13:55.795945+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:13:55.795945+0000","last_undegraded":"2023-09-23T15:13:55.795945+0000","last_fullsized":"2023-09-23T15:13:55.795945+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":376,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.949110+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:13:44.949110+0000","last_peered":"2023-09-23T15:13:44.949110+0000","last_clean":"2023-09-23T15:13:44.949110+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:13:44.949110+0000","last_undegraded":"2023-09-23T15:13:44.949110+0000","last_fullsized":"2023-09-23T15:13:44.949110+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":178,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.761510+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:13:55.761510+0000","last_peered":"2023-09-23T15:13:55.761510+0000","last_clean":"2023-09-23T15:13:55.761510+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:13:55.761510+0000","last_undegraded":"2023-09-23T15:13:55.761510+0000","last_fullsized":"2023-09-23T15:13:55.761510+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":122,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.767480+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:13:55.767480+0000","last_peered":"2023-09-23T15:13:55.767480+0000","last_clean":"2023-09-23T15:13:55.767480+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:13:55.767480+0000","last_undegraded":"2023-09-23T15:13:55.767480+0000","last_fullsized":"2023-09-23T15:13:55.767480+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"202'7886","reported_seq":7927,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.367578+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:14:09.367578+0000","last_peered":"2023-09-23T15:14:09.367578+0000","last_clean":"2023-09-23T15:14:09.367578+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:14:09.367578+0000","last_undegraded":"2023-09-23T15:14:09.367578+0000","last_fullsized":"2023-09-23T15:14:09.367578+0000","mapping_epoch":171,"log_start":"175'1500","ondisk_log_start":"175'1500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292814848,"num_objects":4468,"num_object_clones":0,"num_object_copies":13404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7886,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794122+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:13:55.794122+0000","last_peered":"2023-09-23T15:13:55.794122+0000","last_clean":"2023-09-23T15:13:55.794122+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:13:55.794122+0000","last_undegraded":"2023-09-23T15:13:55.794122+0000","last_fullsized":"2023-09-23T15:13:55.794122+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":720,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759120+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:13:55.759120+0000","last_peered":"2023-09-23T15:13:55.759120+0000","last_clean":"2023-09-23T15:13:55.759120+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:13:55.759120+0000","last_undegraded":"2023-09-23T15:13:55.759120+0000","last_fullsized":"2023-09-23T15:13:55.759120+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":161,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746710+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:13:55.746710+0000","last_peered":"2023-09-23T15:13:55.746710+0000","last_clean":"2023-09-23T15:13:55.746710+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:13:55.746710+0000","last_undegraded":"2023-09-23T15:13:55.746710+0000","last_fullsized":"2023-09-23T15:13:55.746710+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":92,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.948793+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:13:44.948793+0000","last_peered":"2023-09-23T15:13:44.948793+0000","last_clean":"2023-09-23T15:13:44.948793+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:13:44.948793+0000","last_undegraded":"2023-09-23T15:13:44.948793+0000","last_fullsized":"2023-09-23T15:13:44.948793+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"202'8038","reported_seq":8077,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.045397+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:14:09.045397+0000","last_peered":"2023-09-23T15:14:09.045397+0000","last_clean":"2023-09-23T15:14:09.045397+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:14:09.045397+0000","last_undegraded":"2023-09-23T15:14:09.045397+0000","last_fullsized":"2023-09-23T15:14:09.045397+0000","mapping_epoch":171,"log_start":"177'2700","ondisk_log_start":"177'2700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5338,"log_dups_size":0,"ondisk_log_size":5338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300679168,"num_objects":4588,"num_object_clones":0,"num_object_copies":13764,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4588,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8038,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795972+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:13:55.795972+0000","last_peered":"2023-09-23T15:13:55.795972+0000","last_clean":"2023-09-23T15:13:55.795972+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:13:55.795972+0000","last_undegraded":"2023-09-23T15:13:55.795972+0000","last_fullsized":"2023-09-23T15:13:55.795972+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":813,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.771141+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:13:55.771141+0000","last_peered":"2023-09-23T15:13:55.771141+0000","last_clean":"2023-09-23T15:13:55.771141+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:13:55.771141+0000","last_undegraded":"2023-09-23T15:13:55.771141+0000","last_fullsized":"2023-09-23T15:13:55.771141+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.947431+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:13:44.947431+0000","last_peered":"2023-09-23T15:13:44.947431+0000","last_clean":"2023-09-23T15:13:44.947431+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:13:44.947431+0000","last_undegraded":"2023-09-23T15:13:44.947431+0000","last_fullsized":"2023-09-23T15:13:44.947431+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":96,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.948289+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:13:44.948289+0000","last_peered":"2023-09-23T15:13:44.948289+0000","last_clean":"2023-09-23T15:13:44.948289+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:13:44.948289+0000","last_undegraded":"2023-09-23T15:13:44.948289+0000","last_fullsized":"2023-09-23T15:13:44.948289+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"202'7899","reported_seq":7938,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.661765+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:14:09.661765+0000","last_peered":"2023-09-23T15:14:09.661765+0000","last_clean":"2023-09-23T15:14:09.661765+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:14:09.661765+0000","last_undegraded":"2023-09-23T15:14:09.661765+0000","last_fullsized":"2023-09-23T15:14:09.661765+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7899,"log_dups_size":0,"ondisk_log_size":7899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293666816,"num_objects":4481,"num_object_clones":0,"num_object_copies":13443,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4481,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7899,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":489,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.758906+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:13:55.758906+0000","last_peered":"2023-09-23T15:13:55.758906+0000","last_clean":"2023-09-23T15:13:55.758906+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:13:55.758906+0000","last_undegraded":"2023-09-23T15:13:55.758906+0000","last_fullsized":"2023-09-23T15:13:55.758906+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":490,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794642+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:13:55.794642+0000","last_peered":"2023-09-23T15:13:55.794642+0000","last_clean":"2023-09-23T15:13:55.794642+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:13:55.794642+0000","last_undegraded":"2023-09-23T15:13:55.794642+0000","last_fullsized":"2023-09-23T15:13:55.794642+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":115,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.761061+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:13:55.761061+0000","last_peered":"2023-09-23T15:13:55.761061+0000","last_clean":"2023-09-23T15:13:55.761061+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:13:55.761061+0000","last_undegraded":"2023-09-23T15:13:55.761061+0000","last_fullsized":"2023-09-23T15:13:55.761061+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.768044+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:13:55.768044+0000","last_peered":"2023-09-23T15:13:55.768044+0000","last_clean":"2023-09-23T15:13:55.768044+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:13:55.768044+0000","last_undegraded":"2023-09-23T15:13:55.768044+0000","last_fullsized":"2023-09-23T15:13:55.768044+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"202'8063","reported_seq":8104,"reported_epoch":202,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:09.350286+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:14:09.350286+0000","last_peered":"2023-09-23T15:14:09.350286+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:14:09.350286+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"175'1600","ondisk_log_start":"175'1600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6463,"log_dups_size":0,"ondisk_log_size":6463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300220416,"num_objects":4581,"num_object_clones":0,"num_object_copies":13743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8063,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-371}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794078+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:13:55.794078+0000","last_peered":"2023-09-23T15:13:55.794078+0000","last_clean":"2023-09-23T15:13:55.794078+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:13:55.794078+0000","last_undegraded":"2023-09-23T15:13:55.794078+0000","last_fullsized":"2023-09-23T15:13:55.794078+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":283,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.947255+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:13:44.947255+0000","last_peered":"2023-09-23T15:13:44.947255+0000","last_clean":"2023-09-23T15:13:44.947255+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:13:44.947255+0000","last_undegraded":"2023-09-23T15:13:44.947255+0000","last_fullsized":"2023-09-23T15:13:44.947255+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":146,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.793968+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:13:55.793968+0000","last_peered":"2023-09-23T15:13:55.793968+0000","last_clean":"2023-09-23T15:13:55.793968+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:13:55.793968+0000","last_undegraded":"2023-09-23T15:13:55.793968+0000","last_fullsized":"2023-09-23T15:13:55.793968+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":176,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795168+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:13:55.795168+0000","last_peered":"2023-09-23T15:13:55.795168+0000","last_clean":"2023-09-23T15:13:55.795168+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:13:55.795168+0000","last_undegraded":"2023-09-23T15:13:55.795168+0000","last_fullsized":"2023-09-23T15:13:55.795168+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"201'7914","reported_seq":7953,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:08.400483+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:14:08.400483+0000","last_peered":"2023-09-23T15:14:08.400483+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:14:08.400483+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301072384,"num_objects":4594,"num_object_clones":0,"num_object_copies":13782,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4594,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7914,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-280}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795500+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:13:55.795500+0000","last_peered":"2023-09-23T15:13:55.795500+0000","last_clean":"2023-09-23T15:13:55.795500+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:13:55.795500+0000","last_undegraded":"2023-09-23T15:13:55.795500+0000","last_fullsized":"2023-09-23T15:13:55.795500+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":807,"reported_epoch":201,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:13:55.795379+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:13:55.795379+0000","last_peered":"2023-09-23T15:13:55.795379+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:13:55.795379+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":72,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.948982+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:13:44.948982+0000","last_peered":"2023-09-23T15:13:44.948982+0000","last_clean":"2023-09-23T15:13:44.948982+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:13:44.948982+0000","last_undegraded":"2023-09-23T15:13:44.948982+0000","last_fullsized":"2023-09-23T15:13:44.948982+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794381+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:13:55.794381+0000","last_peered":"2023-09-23T15:13:55.794381+0000","last_clean":"2023-09-23T15:13:55.794381+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:13:55.794381+0000","last_undegraded":"2023-09-23T15:13:55.794381+0000","last_fullsized":"2023-09-23T15:13:55.794381+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"202'7812","reported_seq":7851,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.292056+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:14:09.292056+0000","last_peered":"2023-09-23T15:14:09.292056+0000","last_clean":"2023-09-23T15:14:09.292056+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:14:09.292056+0000","last_undegraded":"2023-09-23T15:14:09.292056+0000","last_fullsized":"2023-09-23T15:14:09.292056+0000","mapping_epoch":171,"log_start":"175'300","ondisk_log_start":"175'300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7512,"log_dups_size":0,"ondisk_log_size":7512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294256640,"num_objects":4490,"num_object_clones":0,"num_object_copies":13470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7812,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795552+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:13:55.795552+0000","last_peered":"2023-09-23T15:13:55.795552+0000","last_clean":"2023-09-23T15:13:55.795552+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:13:55.795552+0000","last_undegraded":"2023-09-23T15:13:55.795552+0000","last_fullsized":"2023-09-23T15:13:55.795552+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.824611+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:13:55.824611+0000","last_peered":"2023-09-23T15:13:55.824611+0000","last_clean":"2023-09-23T15:13:55.824611+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:13:55.824611+0000","last_undegraded":"2023-09-23T15:13:55.824611+0000","last_fullsized":"2023-09-23T15:13:55.824611+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":660,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.957205+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:13:44.957205+0000","last_peered":"2023-09-23T15:13:44.957205+0000","last_clean":"2023-09-23T15:13:44.957205+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:13:44.957205+0000","last_undegraded":"2023-09-23T15:13:44.957205+0000","last_fullsized":"2023-09-23T15:13:44.957205+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":181,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.824294+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:13:55.824294+0000","last_peered":"2023-09-23T15:13:55.824294+0000","last_clean":"2023-09-23T15:13:55.824294+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:13:55.824294+0000","last_undegraded":"2023-09-23T15:13:55.824294+0000","last_fullsized":"2023-09-23T15:13:55.824294+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"202'8134","reported_seq":8180,"reported_epoch":202,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:09.086043+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:14:09.086043+0000","last_peered":"2023-09-23T15:14:09.086043+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:14:09.086043+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"177'2800","ondisk_log_start":"177'2800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294649856,"num_objects":4496,"num_object_clones":0,"num_object_copies":13488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8134,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-470}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":478,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794765+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:13:55.794765+0000","last_peered":"2023-09-23T15:13:55.794765+0000","last_clean":"2023-09-23T15:13:55.794765+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:13:55.794765+0000","last_undegraded":"2023-09-23T15:13:55.794765+0000","last_fullsized":"2023-09-23T15:13:55.794765+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":678,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.768384+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:13:55.768384+0000","last_peered":"2023-09-23T15:13:55.768384+0000","last_clean":"2023-09-23T15:13:55.768384+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:13:55.768384+0000","last_undegraded":"2023-09-23T15:13:55.768384+0000","last_fullsized":"2023-09-23T15:13:55.768384+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":164,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.795257+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:13:55.795257+0000","last_peered":"2023-09-23T15:13:55.795257+0000","last_clean":"2023-09-23T15:13:55.795257+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:13:55.795257+0000","last_undegraded":"2023-09-23T15:13:55.795257+0000","last_fullsized":"2023-09-23T15:13:55.795257+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":143,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746636+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:13:55.746636+0000","last_peered":"2023-09-23T15:13:55.746636+0000","last_clean":"2023-09-23T15:13:55.746636+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:13:55.746636+0000","last_undegraded":"2023-09-23T15:13:55.746636+0000","last_fullsized":"2023-09-23T15:13:55.746636+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":153,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.794302+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:13:55.794302+0000","last_peered":"2023-09-23T15:13:55.794302+0000","last_clean":"2023-09-23T15:13:55.794302+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:13:55.794302+0000","last_undegraded":"2023-09-23T15:13:55.794302+0000","last_fullsized":"2023-09-23T15:13:55.794302+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":200,"reported_epoch":200,"state":"active+clean","last_fresh":"2023-09-23T15:13:44.951937+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:13:44.951937+0000","last_peered":"2023-09-23T15:13:44.951937+0000","last_clean":"2023-09-23T15:13:44.951937+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:13:44.951937+0000","last_undegraded":"2023-09-23T15:13:44.951937+0000","last_fullsized":"2023-09-23T15:13:44.951937+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":180,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.762481+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:13:55.762481+0000","last_peered":"2023-09-23T15:13:55.762481+0000","last_clean":"2023-09-23T15:13:55.762481+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:13:55.762481+0000","last_undegraded":"2023-09-23T15:13:55.762481+0000","last_fullsized":"2023-09-23T15:13:55.762481+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":194,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746943+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:13:55.746943+0000","last_peered":"2023-09-23T15:13:55.746943+0000","last_clean":"2023-09-23T15:13:55.746943+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:13:55.746943+0000","last_undegraded":"2023-09-23T15:13:55.746943+0000","last_fullsized":"2023-09-23T15:13:55.746943+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":146,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.761422+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:13:55.761422+0000","last_peered":"2023-09-23T15:13:55.761422+0000","last_clean":"2023-09-23T15:13:55.761422+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:13:55.761422+0000","last_undegraded":"2023-09-23T15:13:55.761422+0000","last_fullsized":"2023-09-23T15:13:55.761422+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":177,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.747483+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:13:55.747483+0000","last_peered":"2023-09-23T15:13:55.747483+0000","last_clean":"2023-09-23T15:13:55.747483+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:13:55.747483+0000","last_undegraded":"2023-09-23T15:13:55.747483+0000","last_fullsized":"2023-09-23T15:13:55.747483+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":172,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.762101+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:13:55.762101+0000","last_peered":"2023-09-23T15:13:55.762101+0000","last_clean":"2023-09-23T15:13:55.762101+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:13:55.762101+0000","last_undegraded":"2023-09-23T15:13:55.762101+0000","last_fullsized":"2023-09-23T15:13:55.762101+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":187,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.759813+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:13:55.759813+0000","last_peered":"2023-09-23T15:13:55.759813+0000","last_clean":"2023-09-23T15:13:55.759813+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:13:55.759813+0000","last_undegraded":"2023-09-23T15:13:55.759813+0000","last_fullsized":"2023-09-23T15:13:55.759813+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":149,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.796225+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:13:55.796225+0000","last_peered":"2023-09-23T15:13:55.796225+0000","last_clean":"2023-09-23T15:13:55.796225+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:13:55.796225+0000","last_undegraded":"2023-09-23T15:13:55.796225+0000","last_fullsized":"2023-09-23T15:13:55.796225+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":201,"state":"active+clean","last_fresh":"2023-09-23T15:13:55.746447+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:13:55.746447+0000","last_peered":"2023-09-23T15:13:55.746447+0000","last_clean":"2023-09-23T15:13:55.746447+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:13:55.746447+0000","last_undegraded":"2023-09-23T15:13:55.746447+0000","last_fullsized":"2023-09-23T15:13:55.746447+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4775608344,"num_objects":72871,"num_object_clones":0,"num_object_copies":218613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25391,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":72871,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":127907,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4309,"num_bytes_recovered":282394624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13230813184,"data_stored":13230800968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110207,"ondisk_log_size":110207,"up":48,"acting":42,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201513,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463149,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2473680,"kb_used_data":2330120,"kb_used_omap":0,"kb_used_meta":143552,"kb_avail":91271472,"statfs":{"total":95995035648,"available":93461987328,"internally_reserved":0,"allocated":2386042880,"data_stored":2385378625,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":146997248},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":4.2370000000000001,"15min":2.633},"min":{"1min":0.23699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.926,"5min":95.721000000000004,"15min":108.39400000000001},"last":0.78900000000000003},{"interface":"front","average":{"1min":2.863,"5min":4.4660000000000002,"15min":2.7400000000000002},"min":{"1min":0.23400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.614000000000001,"5min":96.019000000000005,"15min":108.452},"last":0.88300000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":3.371,"15min":3.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":3.448},{"interface":"front","average":{"1min":3.6070000000000002,"5min":3.6070000000000002,"15min":3.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":20.984000000000002,"5min":20.984000000000002,"15min":20.984000000000002},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":6.6059999999999999,"15min":4.8700000000000001},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.664000000000001,"5min":139.17500000000001,"15min":180.374},"last":3.3109999999999999},{"interface":"front","average":{"1min":3.3690000000000002,"5min":6.4640000000000004,"15min":4.8700000000000001},"min":{"1min":0.33300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":16.552,"5min":146.999,"15min":181.685},"last":3.427}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.2149999999999999,"15min":1.169},"min":{"1min":0.34499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9329999999999998,"5min":17.084,"15min":17.084},"last":4.4390000000000001},{"interface":"front","average":{"1min":3.4940000000000002,"5min":2.2240000000000002,"15min":1.1679999999999999},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.6769999999999996,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":1.002}]},{"osd":4,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.8999999999999999,"15min":1.0329999999999999},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5119999999999996,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.998,"15min":1.0980000000000001},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.5760000000000001,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.8399999999999999,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":5.5880000000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9470000000000001,"15min":1.431},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4550000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.671,"15min":0.98199999999999998},"min":{"1min":0.36099999999999999,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":3.7490000000000001,"5min":17,"15min":17},"last":0.504},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.4750000000000001,"15min":0.95599999999999996},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.496,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.434}]}]},{"osd":1,"up_from":183,"seq":785979015186,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152216,"kb_used_data":1025744,"kb_used_omap":6,"kb_used_meta":126457,"kb_avail":92592936,"statfs":{"total":95995035648,"available":94815166464,"internally_reserved":0,"allocated":1050361856,"data_stored":1049404448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":129492043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.8039999999999998,"15min":3.8039999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.626000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":4.4450000000000003},{"interface":"front","average":{"1min":4.6100000000000003,"5min":4.6100000000000003,"15min":4.6100000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.131,"5min":16.131,"15min":16.131},"last":4.9039999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":13.843,"5min":13.843,"15min":13.843},"last":8.3569999999999993},{"interface":"front","average":{"1min":4.2229999999999999,"5min":4.2229999999999999,"15min":4.2229999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.658000000000001,"5min":16.658000000000001,"15min":16.658000000000001},"last":4.4749999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":123.426,"5min":123.426,"15min":123.426},"last":4.9749999999999996},{"interface":"front","average":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":123.32899999999999,"5min":123.32899999999999,"15min":123.32899999999999},"last":4.992}]},{"osd":4,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":5.1970000000000001},{"interface":"front","average":{"1min":3.5579999999999998,"5min":3.5579999999999998,"15min":3.5579999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.544,"5min":16.544,"15min":16.544},"last":0.56399999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":16.166,"5min":16.166,"15min":16.166},"last":3.8290000000000002},{"interface":"front","average":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":5.1710000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":16.074000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":3.8730000000000002},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":7.7910000000000004,"5min":7.7910000000000004,"15min":7.7910000000000004},"last":5.1399999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.157,"15min":3.157},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":5.2229999999999999},{"interface":"front","average":{"1min":3.3540000000000001,"5min":3.3540000000000001,"15min":3.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":16.594000000000001,"5min":16.594000000000001,"15min":16.594000000000001},"last":0.59699999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893236,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1421548,"kb_used_data":1176356,"kb_used_omap":8,"kb_used_meta":245175,"kb_avail":92323604,"statfs":{"total":95995035648,"available":94539370496,"internally_reserved":0,"allocated":1204588544,"data_stored":1203728403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":251059796},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":31.510999999999999,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.47999999999999998},{"interface":"front","average":{"1min":4.71,"5min":4.71,"15min":4.71},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.378,"5min":31.378,"15min":31.378},"last":1.389}]},{"osd":2,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":5.359,"15min":2.375},"min":{"1min":0.46000000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":31.271999999999998,"5min":122.20099999999999,"15min":122.20099999999999},"last":6.5739999999999998},{"interface":"front","average":{"1min":4.9180000000000001,"5min":4.4139999999999997,"15min":1.9850000000000001},"min":{"1min":0.51500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":31.542999999999999,"5min":122.071,"15min":122.071},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":12.659000000000001,"5min":5.0529999999999999,"15min":2.0529999999999999},"min":{"1min":0.28299999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":84.935000000000002,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.35299999999999998},{"interface":"front","average":{"1min":12.394,"5min":4.1619999999999999,"15min":2.085},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":78.902000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":3.0129999999999999,"15min":1.5209999999999999},"min":{"1min":0.51200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":31.341999999999999,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":4.556,"5min":2.9670000000000001,"15min":1.538},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":31.059999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.39900000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":3.0569999999999999,"15min":1.6180000000000001},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":31.25,"5min":109.369,"15min":109.369},"last":0.55000000000000004},{"interface":"front","average":{"1min":4.032,"5min":3.532,"15min":1.7350000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":31.295000000000002,"5min":109.387,"15min":109.387},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":1.5780000000000001},"min":{"1min":0.41299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.893999999999998,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":4.0350000000000001,"5min":2.9220000000000002,"15min":1.526},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":31.431000000000001,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":3.2130000000000001,"15min":1.6259999999999999},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":31.472000000000001,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.60499999999999998},{"interface":"front","average":{"1min":4.5099999999999998,"5min":2.4510000000000001,"15min":1.4139999999999999},"min":{"1min":0.53700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":31.225999999999999,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.497}]}]},{"osd":2,"up_from":105,"seq":450971566183,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2947316,"kb_used_data":2654764,"kb_used_omap":11,"kb_used_meta":292532,"kb_avail":90797836,"statfs":{"total":95995035648,"available":92976984064,"internally_reserved":0,"allocated":2718478336,"data_stored":2717614391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":299553552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":5.8540000000000001,"5min":2.7320000000000002,"15min":1.25},"min":{"1min":0.23000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":38.274000000000001,"5min":38.274000000000001,"15min":38.274000000000001},"last":4.2999999999999998},{"interface":"front","average":{"1min":4.6130000000000004,"5min":2.3580000000000001,"15min":1.111},"min":{"1min":0.307,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":33.026000000000003,"5min":33.026000000000003,"15min":33.026000000000003},"last":3.0510000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":5.04,"5min":5.04,"15min":5.04},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":2.5489999999999999,"5min":2.5489999999999999,"15min":2.5489999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":7.6669999999999998},"last":3.093}]},{"osd":3,"last update":"Sat Sep 23 15:13:22 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":1.8420000000000001,"15min":1.1060000000000001},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":22.233000000000001,"5min":22.233000000000001,"15min":57.695},"last":4.3369999999999997},{"interface":"front","average":{"1min":3.968,"5min":1.895,"15min":1.1519999999999999},"min":{"1min":0.24399999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.254999999999999,"5min":22.254999999999999,"15min":58.387999999999998},"last":4.7430000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.6190000000000002,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.6589999999999998,"5min":52.795999999999999,"15min":52.795999999999999},"last":3.1339999999999999},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.6280000000000001,"15min":1.7090000000000001},"min":{"1min":0.434,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.478999999999999,"5min":68.894999999999996,"15min":68.894999999999996},"last":0.441}]},{"osd":5,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":2.7719999999999998,"15min":1.4790000000000001},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.161000000000001,"5min":41.723999999999997,"15min":41.723999999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.7200000000000002,"5min":2.7469999999999999,"15min":1.4410000000000001},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.694,"5min":39.822000000000003,"15min":39.822000000000003},"last":3.1640000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.9239999999999999,"15min":1.5329999999999999},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5800000000000001,"5min":37.834000000000003,"15min":37.834000000000003},"last":8.9809999999999999},{"interface":"front","average":{"1min":2.4910000000000001,"5min":2.3959999999999999,"15min":1.387},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.019,"5min":37.749000000000002,"15min":37.749000000000002},"last":4.2080000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.383,"15min":1.3260000000000001},"min":{"1min":0.42399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.207999999999998,"5min":18.207999999999998,"15min":18.207999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.2669999999999999,"15min":1.3149999999999999},"min":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6299999999999999,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":8.9220000000000006}]}]},{"osd":3,"up_from":110,"seq":472446402657,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3465484,"kb_used_data":3213060,"kb_used_omap":3,"kb_used_meta":252220,"kb_avail":90279668,"statfs":{"total":95995035648,"available":92446380032,"internally_reserved":0,"allocated":3290173440,"data_stored":3289347978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":258273449},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":3.694,"15min":1.518},"min":{"1min":0.26000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.276,"5min":79.611000000000004,"15min":79.611000000000004},"last":1.4610000000000001},{"interface":"front","average":{"1min":3.0499999999999998,"5min":3.831,"15min":1.5940000000000001},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.0809999999999995,"5min":79.656000000000006,"15min":79.656000000000006},"last":2.6499999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":4.5860000000000003,"15min":4.5860000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.315000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":1.1120000000000001},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.427,"5min":14.427,"15min":14.427},"last":0.96699999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":7.5570000000000004,"15min":2.952},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":10.467000000000001,"5min":109.077,"15min":109.077},"last":1.7829999999999999},{"interface":"front","average":{"1min":4.0830000000000002,"5min":8.4160000000000004,"15min":3.1589999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.265000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":2.6800000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8009999999999999,"15min":1.026},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0399999999999991,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":2.5859999999999999},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.9670000000000001,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6580000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":1.0640000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.6669999999999998,"15min":1.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.9450000000000003,"5min":56.960999999999999,"15min":56.960999999999999},"last":2.5489999999999999},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.9209999999999998,"15min":1.5389999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.593,"5min":56.981999999999999,"15min":56.981999999999999},"last":1.05}]},{"osd":6,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.3069999999999999,"15min":1.2290000000000001},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.4749999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":2.617},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.5990000000000002,"15min":1.448},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6890000000000001,"5min":19.355,"15min":19.355},"last":1.089}]},{"osd":7,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.6240000000000001,"15min":1.448},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7489999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.026},{"interface":"front","average":{"1min":2.8100000000000001,"5min":1.827,"15min":1.0680000000000001},"min":{"1min":0.47899999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.406000000000001,"5min":13.403,"15min":13.403},"last":0.89200000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986422,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1646988,"kb_used_data":1467908,"kb_used_omap":0,"kb_used_meta":179072,"kb_avail":92098164,"statfs":{"total":95995035648,"available":94308519936,"internally_reserved":0,"allocated":1503137792,"data_stored":1502477112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":183369728},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":4.0780000000000003,"15min":2.2869999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":6.6799999999999997,"5min":108.148,"15min":122.69199999999999},"last":0.27300000000000002},{"interface":"front","average":{"1min":1.1679999999999999,"5min":3.903,"15min":2.306},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6719999999999997,"5min":108.124,"15min":122.745},"last":7.0300000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":4.5490000000000004,"15min":4.5490000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":53.832000000000001,"5min":53.832000000000001,"15min":53.832000000000001},"last":2.431},{"interface":"front","average":{"1min":5.077,"5min":5.077,"15min":5.077},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.078000000000003,"5min":53.078000000000003,"15min":53.078000000000003},"last":2.5529999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":6.2439999999999998,"15min":2.7719999999999998},"min":{"1min":0.41099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":6.96,"5min":148.08000000000001,"15min":148.08000000000001},"last":7.3019999999999996},{"interface":"front","average":{"1min":4.1719999999999997,"5min":6.6150000000000002,"15min":3.0299999999999998},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":34.798999999999999,"5min":147.88900000000001,"15min":147.88900000000001},"last":4.9249999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":20.986999999999998,"5min":7.718,"15min":3.0219999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":135.607,"5min":163.97499999999999,"15min":163.97499999999999},"last":3.464},{"interface":"front","average":{"1min":20.789000000000001,"5min":7.1319999999999997,"15min":2.8220000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":135.33600000000001,"5min":164.447,"15min":164.447},"last":3.254}]},{"osd":5,"last update":"Sat Sep 23 15:13:38 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.3660000000000001,"15min":1.198},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.998,"5min":28.891999999999999,"15min":28.891999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.8340000000000001,"15min":1.0449999999999999},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9489999999999998,"5min":32.100999999999999,"15min":32.100999999999999},"last":2.577}]},{"osd":6,"last update":"Sat Sep 23 15:13:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.9449999999999998,"15min":1.73},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.927,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.5070000000000001,"15min":1.5980000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5659999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":2.5179999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.1059999999999999,"15min":1.1379999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":4.5869999999999997,"5min":18.23,"15min":18.23},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.78800000000000003,"5min":2.0270000000000001,"15min":1.149},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.879,"5min":14.653,"15min":14.653},"last":0.46600000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724785,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1651344,"kb_used_data":1454344,"kb_used_omap":0,"kb_used_meta":196992,"kb_avail":92093808,"statfs":{"total":95995035648,"available":94304059392,"internally_reserved":0,"allocated":1489248256,"data_stored":1488587569,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":201719808},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":2.0859999999999999,"15min":1.2190000000000001},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.968,"5min":17.044,"15min":24.962},"last":2.4849999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.823,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.95,"5min":18.814,"15min":25.317},"last":4.6139999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.673,"15min":1.673},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":2.71},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.717000000000001,"5min":11.717000000000001,"15min":11.717000000000001},"last":4.6479999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":4.7000000000000002,"15min":2.097},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.7859999999999996,"5min":97.087999999999994,"15min":97.087999999999994},"last":2.6219999999999999},{"interface":"front","average":{"1min":4.5129999999999999,"5min":4.4930000000000003,"15min":2.0619999999999998},"min":{"1min":0.33400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":13.332000000000001,"5min":97.305999999999997,"15min":97.305999999999997},"last":2.1379999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.2109999999999999,"15min":1.5129999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.7770000000000001,"5min":53.088999999999999,"15min":53.088999999999999},"last":2.4119999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.8260000000000001,"15min":1.4350000000000001},"min":{"1min":0.47299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.121,"5min":53.764000000000003,"15min":53.764000000000003},"last":3.286}]},{"osd":4,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.298,"15min":1.163},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7669999999999999,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.316},{"interface":"front","average":{"1min":0.622,"5min":2.5449999999999999,"15min":1.242},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.1850000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":2.0539999999999998,"15min":1.107},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8760000000000003,"5min":15.196,"15min":15.196},"last":1.349},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.752,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":1.6419999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":1.367}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.3519999999999999,"15min":1.2270000000000001},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.4260000000000002,"5min":15.301,"15min":15.301},"last":1.331},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.2410000000000001,"15min":1.2070000000000001},"min":{"1min":0.40999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":3.0249999999999999,"5min":15.762,"15min":15.762},"last":0.59199999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1191645184,"data_stored":1191641112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":698421248,"data_stored":698417176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2709258240,"data_stored":2709258240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3279093760,"data_stored":3279093760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1490747392,"data_stored":1490747392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1483018240,"data_stored":1483014168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2378629120,"data_stored":2378629120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:14:14.168 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:14.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:14.330 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:14.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:14.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:14.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:14.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:14.550 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:14.575 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:14.737 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:14.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:14.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:14.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:14.920 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:15.118 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:15.123 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:14:15.123 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:14:15.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:15.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:15.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:15.333 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:15.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:15 smithi017 ceph-mon[215646]: pgmap v527: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 25394/214977 objects degraded (11.812%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:15.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:15 smithi017 ceph-mon[222878]: pgmap v527: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 25394/214977 objects degraded (11.812%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:15 smithi027 ceph-mon[195233]: pgmap v527: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 25394/214977 objects degraded (11.812%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:14:15.863 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:15.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:15.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:15.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:16.018 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:16.077 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:16.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:16.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:16.186 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:16.208 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:14:16.216 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:16.280 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:16.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:16.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:16.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:16.420 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:16.476 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:16.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:16.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:16.582 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:16.611 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:16.665 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:16.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:16.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:14:16.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:16.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:14:16.803 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:16.857 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:16.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:16.864 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:16.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:16.997 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:17.048 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:17.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:17.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:17.186 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:17.239 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:17.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:17.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:17.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:17 smithi017 ceph-mon[222878]: pgmap v528: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 25012/210306 objects degraded (11.893%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T15:14:17.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25012/210306 objects degraded (11.893%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:17.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:17 smithi017 ceph-mon[215646]: pgmap v528: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 25012/210306 objects degraded (11.893%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T15:14:17.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25012/210306 objects degraded (11.893%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:17.474 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:17.475 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:17 smithi027 ceph-mon[195233]: pgmap v528: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 25012/210306 objects degraded (11.893%); 4.9 MiB/s, 77 objects/s recovering 2023-09-23T15:14:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25012/210306 objects degraded (11.893%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:17.606 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:17.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:17.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:17.666 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:17.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:17.752 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:17.809 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:17.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:17.816 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:17.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:17.945 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:17.999 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:18.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:18.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:18.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:18.174 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:18] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-09-23T15:14:18.182 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:18.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:18.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:18.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:18.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:18.810 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:18.911 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[215646]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[215646]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[215646]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[215646]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[222878]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T15:14:19.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[222878]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[222878]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[222878]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:18 smithi027 ceph-mon[195233]: osdmap e203: 8 total, 8 up, 7 in 2023-09-23T15:14:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:18 smithi027 ceph-mon[195233]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:18 smithi027 ceph-mon[195233]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:18 smithi027 ceph-mon[195233]: failed to encode map e203 with expected crc 2023-09-23T15:14:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: pgmap v530: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 25012/210306 objects degraded (11.893%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: failed to encode map e204 with expected crc 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: failed to encode map e204 with expected crc 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T15:14:19.908 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[222878]: failed to encode map e204 with expected crc 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: pgmap v530: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 25012/210306 objects degraded (11.893%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: failed to encode map e204 with expected crc 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: failed to encode map e204 with expected crc 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T15:14:19.909 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:19 smithi017 ceph-mon[215646]: failed to encode map e204 with expected crc 2023-09-23T15:14:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: pgmap v530: 109 pgs: 101 active+clean, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 25012/210306 objects degraded (11.893%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: failed to encode map e204 with expected crc 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: failed to encode map e204 with expected crc 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4046830014' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: osdmap e204: 8 total, 8 up, 7 in 2023-09-23T15:14:20.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:19 smithi027 ceph-mon[195233]: failed to encode map e204 with expected crc 2023-09-23T15:14:20.588 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:20.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:20.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:20.738 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:20.806 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:20.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:20.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:20.944 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.002 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:21.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:21.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:21.110 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:21.149 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.198 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:21.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:21.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:21.304 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:21.334 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.394 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:21.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:21.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:21.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:21.534 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.596 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:21.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:21.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:21.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:21.741 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.786 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:21.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:21.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:21.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:21.917 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:21 smithi027 ceph-mon[195233]: pgmap v532: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:21.917 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:21 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24712/203133 objects degraded (12.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:21.921 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:21.988 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:22.001 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:22.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:22.133 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:22.187 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:22.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:21 smithi017 ceph-mon[215646]: pgmap v532: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:21 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24712/203133 objects degraded (12.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:21 smithi017 ceph-mon[222878]: pgmap v532: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 648 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:21 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24712/203133 objects degraded (12.165%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:22.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:22.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:22.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:22.330 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:22.385 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:22.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:22.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:22.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:22.524 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:22.584 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:22.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:22.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:22.722 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:22.738 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:22.772 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:22.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:22.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:22.799 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:22.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:22.881 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:22.913 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:22.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:22.956 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:23.008 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:23.036 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:135213 6.7 e204/171 6:ecf4575c:::benchmark_data_smithi017_350392_object34822:head v 204'8364, mlcod=204'8361)", 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:23.014776+0000", 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.017754519999999999, 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017788157999999998, 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014776+0000", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014776+0000", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014775+0000", 2023-09-23T15:14:23.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014787+0000", 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2150999999999999e-05 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014798+0000", 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0555e-05 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.014805+0000", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6950000000000003e-06 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.015668+0000", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000863093 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.015686+0000", 2023-09-23T15:14:23.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8000999999999999e-05 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:135217 6.9 e204/171 6:994e4f11:::benchmark_data_smithi017_350392_object34826:head v 204'8510, mlcod=204'8508)", 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:23.021212+0000", 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.011318619, 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011425246999999999, 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021212+0000", 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021212+0000", 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021211+0000", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021220+0000", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2290000000000003e-06 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021230+0000", 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0522e-05 2023-09-23T15:14:23.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021237+0000", 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.64e-06 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021299+0000", 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2584999999999998e-05 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.021319+0000", 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9843999999999998e-05 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:135216 6.4 6:29c1852e:::benchmark_data_smithi017_350392_object34825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:23.029340+0000", 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0031910240000000002, 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003369698, 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.045 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 135216 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029340+0000", 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029340+0000", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029338+0000", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029344+0000", 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1650000000000003e-06 2023-09-23T15:14:23.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029358+0000", 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3875e-05 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029367+0000", 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4090000000000003e-06 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029434+0000", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7076000000000006e-05 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029623+0000", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000189323 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029783+0000", 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160249 2023-09-23T15:14:23.049 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:135224 6.6 6:6adabfeb:::benchmark_data_smithi017_350392_object34833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:23.029387+0000", 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0031432740000000002, 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034075590000000001, 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:14:23.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 135224 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029387+0000", 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029387+0000", 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029382+0000", 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999943 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029390+0000", 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5199999999999997e-06 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029398+0000", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6790000000000005e-06 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029403+0000", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5040000000000002e-06 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029545+0000", 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141889 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029663+0000", 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011756000000000001 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029783+0000", 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120313 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:135220 6.2 e204/171 6:407f52f8:::benchmark_data_smithi017_350392_object34829:head v 204'8238, mlcod=204'8235)", 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:23.029426+0000", 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0031045740000000001, 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034514509999999999, 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029426+0000", 2023-09-23T15:14:23.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029426+0000", 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029425+0000", 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029435+0000", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6970000000000005e-06 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029444+0000", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5059999999999995e-06 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029449+0000", 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4079999999999997e-06 2023-09-23T15:14:23.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029503+0000", 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4039999999999998e-05 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.029520+0000", 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7027000000000001e-05 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:135206 6.b 6:dc96c360:::benchmark_data_smithi017_350392_object34815:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e204)", 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:22.991658+0000", 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.040873103000000001, 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.041324315, 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 135206 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.061 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991658+0000", 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991658+0000", 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991656+0000", 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991664+0000", 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0679999999999997e-06 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:14:23.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991677+0000", 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.325e-05 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:22.991685+0000", 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9300000000000003e-06 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.004748+0000", 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013062616000000001 2023-09-23T15:14:23.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.004885+0000", 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013728199999999999 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.005038+0000", 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015248200000000001 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.028060+0000", 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023022007000000001 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:23.028075+0000", 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5591999999999999e-05 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.066 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:14:23.067 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:14:23.067 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 6 2023-09-23T15:14:23.067 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:14:23.067 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:23.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:14:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2091019132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:23.107 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:23.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.112 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.113 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.113 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.113 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:14:23.133 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:23.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:23.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:23.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:23.272 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:23.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2091019132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:23.283 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2091019132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:23.322 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:23.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:23.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:23.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:23.469 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:23.524 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:23.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:23.630 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:23.661 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:23.724 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:23.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:23.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:23.831 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:23.953 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:24.162 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:24.178 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:14:24.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:23 smithi017 ceph-mon[215646]: pgmap v533: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:24.198 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:23 smithi017 ceph-mon[222878]: pgmap v533: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:23 smithi027 ceph-mon[195233]: pgmap v533: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 24712/203133 objects degraded (12.165%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:14:25.475 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":533,"stamp":"2023-09-23T15:14:22.353501+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4440430812,"num_objects":67711,"num_object_clones":0,"num_object_copies":203133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24712,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67711,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":134122,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5050,"num_bytes_recovered":332437671,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116707,"ondisk_log_size":116707,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":336,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13678056,"kb_used_data":12202160,"kb_used_omap":30,"kb_used_meta":1475681,"kb_avail":642538008,"statfs":{"total":671965249536,"available":657958920192,"internally_reserved":0,"allocated":12495011840,"data_stored":12489455446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1511097944},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-258736128,"num_objects":-3948,"num_object_clones":0,"num_object_copies":-11844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-682,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3948,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":44695552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005449"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":198,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902382+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:14:17.902382+0000","last_peered":"2023-09-23T15:14:17.902382+0000","last_clean":"2023-09-23T15:14:17.902382+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:14:17.902382+0000","last_undegraded":"2023-09-23T15:14:17.902382+0000","last_fullsized":"2023-09-23T15:14:17.902382+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.917838+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:14:17.917838+0000","last_peered":"2023-09-23T15:14:17.917838+0000","last_clean":"2023-09-23T15:14:17.917838+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:14:17.917838+0000","last_undegraded":"2023-09-23T15:14:17.917838+0000","last_fullsized":"2023-09-23T15:14:17.917838+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":80,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.910408+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:14:17.910408+0000","last_peered":"2023-09-23T15:14:17.910408+0000","last_clean":"2023-09-23T15:14:17.910408+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:14:17.910408+0000","last_undegraded":"2023-09-23T15:14:17.910408+0000","last_fullsized":"2023-09-23T15:14:17.910408+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920617+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:14:17.920617+0000","last_peered":"2023-09-23T15:14:17.920617+0000","last_clean":"2023-09-23T15:14:17.920617+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:14:17.920617+0000","last_undegraded":"2023-09-23T15:14:17.920617+0000","last_fullsized":"2023-09-23T15:14:17.920617+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.939609+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:14:17.939609+0000","last_peered":"2023-09-23T15:14:17.939609+0000","last_clean":"2023-09-23T15:14:17.939609+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:14:17.939609+0000","last_undegraded":"2023-09-23T15:14:17.939609+0000","last_fullsized":"2023-09-23T15:14:17.939609+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.918833+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:14:17.918833+0000","last_peered":"2023-09-23T15:14:17.918833+0000","last_clean":"2023-09-23T15:14:17.918833+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:14:17.918833+0000","last_undegraded":"2023-09-23T15:14:17.918833+0000","last_fullsized":"2023-09-23T15:14:17.918833+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":148,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909323+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:14:18.909323+0000","last_peered":"2023-09-23T15:14:18.909323+0000","last_clean":"2023-09-23T15:14:18.909323+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:14:18.909323+0000","last_undegraded":"2023-09-23T15:14:18.909323+0000","last_fullsized":"2023-09-23T15:14:18.909323+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":183,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.939689+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:14:17.939689+0000","last_peered":"2023-09-23T15:14:17.939689+0000","last_clean":"2023-09-23T15:14:17.939689+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:14:17.939689+0000","last_undegraded":"2023-09-23T15:14:17.939689+0000","last_fullsized":"2023-09-23T15:14:17.939689+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.919298+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:14:17.919298+0000","last_peered":"2023-09-23T15:14:17.919298+0000","last_clean":"2023-09-23T15:14:17.919298+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:14:17.919298+0000","last_undegraded":"2023-09-23T15:14:17.919298+0000","last_fullsized":"2023-09-23T15:14:17.919298+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":194,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904751+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:14:17.904751+0000","last_peered":"2023-09-23T15:14:17.904751+0000","last_clean":"2023-09-23T15:14:17.904751+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:14:17.904751+0000","last_undegraded":"2023-09-23T15:14:17.904751+0000","last_fullsized":"2023-09-23T15:14:17.904751+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":126,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.917083+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:14:17.917083+0000","last_peered":"2023-09-23T15:14:17.917083+0000","last_clean":"2023-09-23T15:14:17.917083+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:14:17.917083+0000","last_undegraded":"2023-09-23T15:14:17.917083+0000","last_fullsized":"2023-09-23T15:14:17.917083+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.917772+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:14:17.917772+0000","last_peered":"2023-09-23T15:14:17.917772+0000","last_clean":"2023-09-23T15:14:17.917772+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:14:17.917772+0000","last_undegraded":"2023-09-23T15:14:17.917772+0000","last_fullsized":"2023-09-23T15:14:17.917772+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.909579+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:14:17.909579+0000","last_peered":"2023-09-23T15:14:17.909579+0000","last_clean":"2023-09-23T15:14:17.909579+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:14:17.909579+0000","last_undegraded":"2023-09-23T15:14:17.909579+0000","last_fullsized":"2023-09-23T15:14:17.909579+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":188,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902556+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:14:17.902556+0000","last_peered":"2023-09-23T15:14:17.902556+0000","last_clean":"2023-09-23T15:14:17.902556+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:14:17.902556+0000","last_undegraded":"2023-09-23T15:14:17.902556+0000","last_fullsized":"2023-09-23T15:14:17.902556+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":179,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902458+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:14:17.902458+0000","last_peered":"2023-09-23T15:14:17.902458+0000","last_clean":"2023-09-23T15:14:17.902458+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:14:17.902458+0000","last_undegraded":"2023-09-23T15:14:17.902458+0000","last_fullsized":"2023-09-23T15:14:17.902458+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920433+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:14:17.920433+0000","last_peered":"2023-09-23T15:14:17.920433+0000","last_clean":"2023-09-23T15:14:17.920433+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:14:17.920433+0000","last_undegraded":"2023-09-23T15:14:17.920433+0000","last_fullsized":"2023-09-23T15:14:17.920433+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":66,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902496+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:14:17.902496+0000","last_peered":"2023-09-23T15:14:17.902496+0000","last_clean":"2023-09-23T15:14:17.902496+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:14:17.902496+0000","last_undegraded":"2023-09-23T15:14:17.902496+0000","last_fullsized":"2023-09-23T15:14:17.902496+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922430+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:14:17.922430+0000","last_peered":"2023-09-23T15:14:17.922430+0000","last_clean":"2023-09-23T15:14:17.922430+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:14:17.922430+0000","last_undegraded":"2023-09-23T15:14:17.922430+0000","last_fullsized":"2023-09-23T15:14:17.922430+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921223+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:14:17.921223+0000","last_peered":"2023-09-23T15:14:17.921223+0000","last_clean":"2023-09-23T15:14:17.921223+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:14:17.921223+0000","last_undegraded":"2023-09-23T15:14:17.921223+0000","last_fullsized":"2023-09-23T15:14:17.921223+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":168,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921721+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:14:17.921721+0000","last_peered":"2023-09-23T15:14:17.921721+0000","last_clean":"2023-09-23T15:14:17.921721+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:14:17.921721+0000","last_undegraded":"2023-09-23T15:14:17.921721+0000","last_fullsized":"2023-09-23T15:14:17.921721+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":172,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904956+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:14:17.904956+0000","last_peered":"2023-09-23T15:14:17.904956+0000","last_clean":"2023-09-23T15:14:17.904956+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:14:17.904956+0000","last_undegraded":"2023-09-23T15:14:17.904956+0000","last_fullsized":"2023-09-23T15:14:17.904956+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922475+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:14:17.922475+0000","last_peered":"2023-09-23T15:14:17.922475+0000","last_clean":"2023-09-23T15:14:17.922475+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:14:17.922475+0000","last_undegraded":"2023-09-23T15:14:17.922475+0000","last_fullsized":"2023-09-23T15:14:17.922475+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902316+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:14:17.902316+0000","last_peered":"2023-09-23T15:14:17.902316+0000","last_clean":"2023-09-23T15:14:17.902316+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:14:17.902316+0000","last_undegraded":"2023-09-23T15:14:17.902316+0000","last_fullsized":"2023-09-23T15:14:17.902316+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":157,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909262+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:14:18.909262+0000","last_peered":"2023-09-23T15:14:18.909262+0000","last_clean":"2023-09-23T15:14:18.909262+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:14:18.909262+0000","last_undegraded":"2023-09-23T15:14:18.909262+0000","last_fullsized":"2023-09-23T15:14:18.909262+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921982+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:14:17.921982+0000","last_peered":"2023-09-23T15:14:17.921982+0000","last_clean":"2023-09-23T15:14:17.921982+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:14:17.921982+0000","last_undegraded":"2023-09-23T15:14:17.921982+0000","last_fullsized":"2023-09-23T15:14:17.921982+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":170,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.918164+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:14:17.918164+0000","last_peered":"2023-09-23T15:14:17.918164+0000","last_clean":"2023-09-23T15:14:17.918164+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:14:17.918164+0000","last_undegraded":"2023-09-23T15:14:17.918164+0000","last_fullsized":"2023-09-23T15:14:17.918164+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":194,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.943201+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:14:17.943201+0000","last_peered":"2023-09-23T15:14:17.943201+0000","last_clean":"2023-09-23T15:14:17.943201+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:14:17.943201+0000","last_undegraded":"2023-09-23T15:14:17.943201+0000","last_fullsized":"2023-09-23T15:14:17.943201+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"204'8256","reported_seq":8297,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.669414+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:14:19.669414+0000","last_peered":"2023-09-23T15:14:19.669414+0000","last_clean":"2023-09-23T15:14:19.669414+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:14:19.669414+0000","last_undegraded":"2023-09-23T15:14:19.669414+0000","last_fullsized":"2023-09-23T15:14:19.669414+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8256,"log_dups_size":0,"ondisk_log_size":8256,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271187968,"num_objects":4138,"num_object_clones":0,"num_object_copies":12414,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4138,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8256,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.923169+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:14:17.923169+0000","last_peered":"2023-09-23T15:14:17.923169+0000","last_clean":"2023-09-23T15:14:17.923169+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:14:17.923169+0000","last_undegraded":"2023-09-23T15:14:17.923169+0000","last_fullsized":"2023-09-23T15:14:17.923169+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":78,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902862+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:14:17.902862+0000","last_peered":"2023-09-23T15:14:17.902862+0000","last_clean":"2023-09-23T15:14:17.902862+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:14:17.902862+0000","last_undegraded":"2023-09-23T15:14:17.902862+0000","last_fullsized":"2023-09-23T15:14:17.902862+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":155,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904368+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:14:17.904368+0000","last_peered":"2023-09-23T15:14:17.904368+0000","last_clean":"2023-09-23T15:14:17.904368+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:14:17.904368+0000","last_undegraded":"2023-09-23T15:14:17.904368+0000","last_fullsized":"2023-09-23T15:14:17.904368+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"204'8424","reported_seq":8654,"reported_epoch":204,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:19.442986+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:14:19.442986+0000","last_peered":"2023-09-23T15:14:19.442986+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:14:19.442986+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"175'2000","ondisk_log_start":"175'2000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6424,"log_dups_size":0,"ondisk_log_size":6424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278003712,"num_objects":4242,"num_object_clones":0,"num_object_copies":12726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4907,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8424,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":91,"num_bytes_recovered":5963776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-665}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":471,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:17.919058+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:14:17.919058+0000","last_peered":"2023-09-23T15:14:17.919058+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:14:17.919058+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":78,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902862+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:14:17.902862+0000","last_peered":"2023-09-23T15:14:17.902862+0000","last_clean":"2023-09-23T15:14:17.902862+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:14:17.902862+0000","last_undegraded":"2023-09-23T15:14:17.902862+0000","last_fullsized":"2023-09-23T15:14:17.902862+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":196,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902221+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:14:17.902221+0000","last_peered":"2023-09-23T15:14:17.902221+0000","last_clean":"2023-09-23T15:14:17.902221+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:14:17.902221+0000","last_undegraded":"2023-09-23T15:14:17.902221+0000","last_fullsized":"2023-09-23T15:14:17.902221+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"203'8210","reported_seq":8252,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.377742+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:14:18.377742+0000","last_peered":"2023-09-23T15:14:18.377742+0000","last_clean":"2023-09-23T15:14:18.377742+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:14:18.377742+0000","last_undegraded":"2023-09-23T15:14:18.377742+0000","last_fullsized":"2023-09-23T15:14:18.377742+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8210,"log_dups_size":0,"ondisk_log_size":8210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277348352,"num_objects":4232,"num_object_clones":0,"num_object_copies":12696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8210,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920061+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:14:17.920061+0000","last_peered":"2023-09-23T15:14:17.920061+0000","last_clean":"2023-09-23T15:14:17.920061+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:14:17.920061+0000","last_undegraded":"2023-09-23T15:14:17.920061+0000","last_fullsized":"2023-09-23T15:14:17.920061+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":64,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904885+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:14:17.904885+0000","last_peered":"2023-09-23T15:14:17.904885+0000","last_clean":"2023-09-23T15:14:17.904885+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:14:17.904885+0000","last_undegraded":"2023-09-23T15:14:17.904885+0000","last_fullsized":"2023-09-23T15:14:17.904885+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":192,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920863+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:14:17.920863+0000","last_peered":"2023-09-23T15:14:17.920863+0000","last_clean":"2023-09-23T15:14:17.920863+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:14:17.920863+0000","last_undegraded":"2023-09-23T15:14:17.920863+0000","last_fullsized":"2023-09-23T15:14:17.920863+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"204'8387","reported_seq":8430,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.334525+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:14:19.334525+0000","last_peered":"2023-09-23T15:14:19.334525+0000","last_clean":"2023-09-23T15:14:19.334525+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:14:19.334525+0000","last_undegraded":"2023-09-23T15:14:19.334525+0000","last_fullsized":"2023-09-23T15:14:19.334525+0000","mapping_epoch":171,"log_start":"177'2800","ondisk_log_start":"177'2800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5587,"log_dups_size":0,"ondisk_log_size":5587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280297472,"num_objects":4277,"num_object_clones":0,"num_object_copies":12831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8387,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920791+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:14:17.920791+0000","last_peered":"2023-09-23T15:14:17.920791+0000","last_clean":"2023-09-23T15:14:17.920791+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:14:17.920791+0000","last_undegraded":"2023-09-23T15:14:17.920791+0000","last_fullsized":"2023-09-23T15:14:17.920791+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904618+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:17.904618+0000","last_peered":"2023-09-23T15:14:17.904618+0000","last_clean":"2023-09-23T15:14:17.904618+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:14:17.904618+0000","last_undegraded":"2023-09-23T15:14:17.904618+0000","last_fullsized":"2023-09-23T15:14:17.904618+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922750+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:14:17.922750+0000","last_peered":"2023-09-23T15:14:17.922750+0000","last_clean":"2023-09-23T15:14:17.922750+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:14:17.922750+0000","last_undegraded":"2023-09-23T15:14:17.922750+0000","last_fullsized":"2023-09-23T15:14:17.922750+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"204'8529","reported_seq":8576,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:19.296236+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:14:19.296236+0000","last_peered":"2023-09-23T15:14:19.296236+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:14:19.296236+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"175'1000","ondisk_log_start":"175'1000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7529,"log_dups_size":0,"ondisk_log_size":7529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287244288,"num_objects":4383,"num_object_clones":0,"num_object_copies":13149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4383,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8529,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-627}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920216+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:14:17.920216+0000","last_peered":"2023-09-23T15:14:17.920216+0000","last_clean":"2023-09-23T15:14:17.920216+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:14:17.920216+0000","last_undegraded":"2023-09-23T15:14:17.920216+0000","last_fullsized":"2023-09-23T15:14:17.920216+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":166,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.940075+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:14:17.940075+0000","last_peered":"2023-09-23T15:14:17.940075+0000","last_clean":"2023-09-23T15:14:17.940075+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:14:17.940075+0000","last_undegraded":"2023-09-23T15:14:17.940075+0000","last_fullsized":"2023-09-23T15:14:17.940075+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":162,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902572+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:14:17.902572+0000","last_peered":"2023-09-23T15:14:17.902572+0000","last_clean":"2023-09-23T15:14:17.902572+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:14:17.902572+0000","last_undegraded":"2023-09-23T15:14:17.902572+0000","last_fullsized":"2023-09-23T15:14:17.902572+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"203'8354","reported_seq":8396,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.351923+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:14:18.351923+0000","last_peered":"2023-09-23T15:14:18.351923+0000","last_clean":"2023-09-23T15:14:18.351923+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:14:18.351923+0000","last_undegraded":"2023-09-23T15:14:18.351923+0000","last_fullsized":"2023-09-23T15:14:18.351923+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8354,"log_dups_size":0,"ondisk_log_size":8354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276824064,"num_objects":4224,"num_object_clones":0,"num_object_copies":12672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4224,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8354,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":497,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921296+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:14:17.921296+0000","last_peered":"2023-09-23T15:14:17.921296+0000","last_clean":"2023-09-23T15:14:17.921296+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:14:17.921296+0000","last_undegraded":"2023-09-23T15:14:17.921296+0000","last_fullsized":"2023-09-23T15:14:17.921296+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920787+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:14:17.920787+0000","last_peered":"2023-09-23T15:14:17.920787+0000","last_clean":"2023-09-23T15:14:17.920787+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:14:17.920787+0000","last_undegraded":"2023-09-23T15:14:17.920787+0000","last_fullsized":"2023-09-23T15:14:17.920787+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":196,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902467+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:14:17.902467+0000","last_peered":"2023-09-23T15:14:17.902467+0000","last_clean":"2023-09-23T15:14:17.902467+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:14:17.902467+0000","last_undegraded":"2023-09-23T15:14:17.902467+0000","last_fullsized":"2023-09-23T15:14:17.902467+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"204'8307","reported_seq":8350,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.436269+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:14:19.436269+0000","last_peered":"2023-09-23T15:14:19.436269+0000","last_clean":"2023-09-23T15:14:19.436269+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:14:19.436269+0000","last_undegraded":"2023-09-23T15:14:19.436269+0000","last_fullsized":"2023-09-23T15:14:19.436269+0000","mapping_epoch":171,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6407,"log_dups_size":0,"ondisk_log_size":6407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279904256,"num_objects":4271,"num_object_clones":0,"num_object_copies":12813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8307,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922571+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:14:17.922571+0000","last_peered":"2023-09-23T15:14:17.922571+0000","last_clean":"2023-09-23T15:14:17.922571+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:14:17.922571+0000","last_undegraded":"2023-09-23T15:14:17.922571+0000","last_fullsized":"2023-09-23T15:14:17.922571+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902116+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:14:17.902116+0000","last_peered":"2023-09-23T15:14:17.902116+0000","last_clean":"2023-09-23T15:14:17.902116+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:14:17.902116+0000","last_undegraded":"2023-09-23T15:14:17.902116+0000","last_fullsized":"2023-09-23T15:14:17.902116+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":179,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902923+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:14:17.902923+0000","last_peered":"2023-09-23T15:14:17.902923+0000","last_clean":"2023-09-23T15:14:17.902923+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:14:17.902923+0000","last_undegraded":"2023-09-23T15:14:17.902923+0000","last_fullsized":"2023-09-23T15:14:17.902923+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"204'8319","reported_seq":18181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.596285+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:14:19.596285+0000","last_peered":"2023-09-23T15:14:19.596285+0000","last_clean":"2023-09-23T15:14:19.596285+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:14:19.596285+0000","last_undegraded":"2023-09-23T15:14:19.596285+0000","last_fullsized":"2023-09-23T15:14:19.596285+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8319,"log_dups_size":0,"ondisk_log_size":8319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037656,"num_objects":4213,"num_object_clones":0,"num_object_copies":12639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4213,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8319,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921910+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:14:17.921910+0000","last_peered":"2023-09-23T15:14:17.921910+0000","last_clean":"2023-09-23T15:14:17.921910+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:14:17.921910+0000","last_undegraded":"2023-09-23T15:14:17.921910+0000","last_fullsized":"2023-09-23T15:14:17.921910+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":195,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.939518+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:14:17.939518+0000","last_peered":"2023-09-23T15:14:17.939518+0000","last_clean":"2023-09-23T15:14:17.939518+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:14:17.939518+0000","last_undegraded":"2023-09-23T15:14:17.939518+0000","last_fullsized":"2023-09-23T15:14:17.939518+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":80,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.909011+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:14:17.909011+0000","last_peered":"2023-09-23T15:14:17.909011+0000","last_clean":"2023-09-23T15:14:17.909011+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:14:17.909011+0000","last_undegraded":"2023-09-23T15:14:17.909011+0000","last_fullsized":"2023-09-23T15:14:17.909011+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"204'8348","reported_seq":8391,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.382917+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:14:19.382917+0000","last_peered":"2023-09-23T15:14:19.382917+0000","last_clean":"2023-09-23T15:14:19.382917+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:14:19.382917+0000","last_undegraded":"2023-09-23T15:14:19.382917+0000","last_fullsized":"2023-09-23T15:14:19.382917+0000","mapping_epoch":171,"log_start":"177'2700","ondisk_log_start":"177'2700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5648,"log_dups_size":0,"ondisk_log_size":5648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8348,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920150+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:14:17.920150+0000","last_peered":"2023-09-23T15:14:17.920150+0000","last_clean":"2023-09-23T15:14:17.920150+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:14:17.920150+0000","last_undegraded":"2023-09-23T15:14:17.920150+0000","last_fullsized":"2023-09-23T15:14:17.920150+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":379,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.913861+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:14:17.913861+0000","last_peered":"2023-09-23T15:14:17.913861+0000","last_clean":"2023-09-23T15:14:17.913861+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:14:17.913861+0000","last_undegraded":"2023-09-23T15:14:17.913861+0000","last_fullsized":"2023-09-23T15:14:17.913861+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":180,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904798+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:17.904798+0000","last_peered":"2023-09-23T15:14:17.904798+0000","last_clean":"2023-09-23T15:14:17.904798+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:14:17.904798+0000","last_undegraded":"2023-09-23T15:14:17.904798+0000","last_fullsized":"2023-09-23T15:14:17.904798+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.004236+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:14:09.004236+0000","last_peered":"2023-09-23T15:14:09.004236+0000","last_clean":"2023-09-23T15:14:09.004236+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:14:09.004236+0000","last_undegraded":"2023-09-23T15:14:09.004236+0000","last_fullsized":"2023-09-23T15:14:09.004236+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"204'8182","reported_seq":8224,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.422540+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:14:19.422540+0000","last_peered":"2023-09-23T15:14:19.422540+0000","last_clean":"2023-09-23T15:14:19.422540+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:14:19.422540+0000","last_undegraded":"2023-09-23T15:14:19.422540+0000","last_fullsized":"2023-09-23T15:14:19.422540+0000","mapping_epoch":171,"log_start":"175'1700","ondisk_log_start":"175'1700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6482,"log_dups_size":0,"ondisk_log_size":6482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273416192,"num_objects":4172,"num_object_clones":0,"num_object_copies":12516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8182,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920538+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:14:17.920538+0000","last_peered":"2023-09-23T15:14:17.920538+0000","last_clean":"2023-09-23T15:14:17.920538+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:14:17.920538+0000","last_undegraded":"2023-09-23T15:14:17.920538+0000","last_fullsized":"2023-09-23T15:14:17.920538+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":722,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904518+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:14:17.904518+0000","last_peered":"2023-09-23T15:14:17.904518+0000","last_clean":"2023-09-23T15:14:17.904518+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:14:17.904518+0000","last_undegraded":"2023-09-23T15:14:17.904518+0000","last_fullsized":"2023-09-23T15:14:17.904518+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":163,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902383+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:14:17.902383+0000","last_peered":"2023-09-23T15:14:17.902383+0000","last_clean":"2023-09-23T15:14:17.902383+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:14:17.902383+0000","last_undegraded":"2023-09-23T15:14:17.902383+0000","last_fullsized":"2023-09-23T15:14:17.902383+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":95,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.913244+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:14:17.913244+0000","last_peered":"2023-09-23T15:14:17.913244+0000","last_clean":"2023-09-23T15:14:17.913244+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:14:17.913244+0000","last_undegraded":"2023-09-23T15:14:17.913244+0000","last_fullsized":"2023-09-23T15:14:17.913244+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"204'8351","reported_seq":8392,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.058255+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:14:19.058255+0000","last_peered":"2023-09-23T15:14:19.058255+0000","last_clean":"2023-09-23T15:14:19.058255+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:14:19.058255+0000","last_undegraded":"2023-09-23T15:14:19.058255+0000","last_fullsized":"2023-09-23T15:14:19.058255+0000","mapping_epoch":171,"log_start":"177'3000","ondisk_log_start":"177'3000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5351,"log_dups_size":0,"ondisk_log_size":5351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280166400,"num_objects":4275,"num_object_clones":0,"num_object_copies":12825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8351,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921792+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:14:17.921792+0000","last_peered":"2023-09-23T15:14:17.921792+0000","last_clean":"2023-09-23T15:14:17.921792+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:14:17.921792+0000","last_undegraded":"2023-09-23T15:14:17.921792+0000","last_fullsized":"2023-09-23T15:14:17.921792+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":815,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.040447+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:14:09.040447+0000","last_peered":"2023-09-23T15:14:09.040447+0000","last_clean":"2023-09-23T15:14:09.040447+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:14:09.040447+0000","last_undegraded":"2023-09-23T15:14:09.040447+0000","last_fullsized":"2023-09-23T15:14:09.040447+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.908619+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:14:17.908619+0000","last_peered":"2023-09-23T15:14:17.908619+0000","last_clean":"2023-09-23T15:14:17.908619+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:14:17.908619+0000","last_undegraded":"2023-09-23T15:14:17.908619+0000","last_fullsized":"2023-09-23T15:14:17.908619+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":99,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.908947+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:14:17.908947+0000","last_peered":"2023-09-23T15:14:17.908947+0000","last_clean":"2023-09-23T15:14:17.908947+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:14:17.908947+0000","last_undegraded":"2023-09-23T15:14:17.908947+0000","last_fullsized":"2023-09-23T15:14:17.908947+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"204'8227","reported_seq":8268,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.581009+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:14:19.581009+0000","last_peered":"2023-09-23T15:14:19.581009+0000","last_clean":"2023-09-23T15:14:19.581009+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:14:19.581009+0000","last_undegraded":"2023-09-23T15:14:19.581009+0000","last_fullsized":"2023-09-23T15:14:19.581009+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8227,"log_dups_size":0,"ondisk_log_size":8227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272171008,"num_objects":4153,"num_object_clones":0,"num_object_copies":12459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4153,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8227,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":491,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904494+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:14:17.904494+0000","last_peered":"2023-09-23T15:14:17.904494+0000","last_clean":"2023-09-23T15:14:17.904494+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:14:17.904494+0000","last_undegraded":"2023-09-23T15:14:17.904494+0000","last_fullsized":"2023-09-23T15:14:17.904494+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":492,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.921428+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:14:17.921428+0000","last_peered":"2023-09-23T15:14:17.921428+0000","last_clean":"2023-09-23T15:14:17.921428+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:14:17.921428+0000","last_undegraded":"2023-09-23T15:14:17.921428+0000","last_fullsized":"2023-09-23T15:14:17.921428+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.918078+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:14:17.918078+0000","last_peered":"2023-09-23T15:14:17.918078+0000","last_clean":"2023-09-23T15:14:17.918078+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:14:17.918078+0000","last_undegraded":"2023-09-23T15:14:17.918078+0000","last_fullsized":"2023-09-23T15:14:17.918078+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.918516+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:14:17.918516+0000","last_peered":"2023-09-23T15:14:17.918516+0000","last_clean":"2023-09-23T15:14:17.918516+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:14:17.918516+0000","last_undegraded":"2023-09-23T15:14:17.918516+0000","last_fullsized":"2023-09-23T15:14:17.918516+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"204'8395","reported_seq":8438,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:19.400512+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:14:19.400512+0000","last_peered":"2023-09-23T15:14:19.400512+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:14:19.400512+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"176'2000","ondisk_log_start":"176'2000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6395,"log_dups_size":0,"ondisk_log_size":6395,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278462464,"num_objects":4249,"num_object_clones":0,"num_object_copies":12747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8395,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-703}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922664+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:14:17.922664+0000","last_peered":"2023-09-23T15:14:17.922664+0000","last_clean":"2023-09-23T15:14:17.922664+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:14:17.922664+0000","last_undegraded":"2023-09-23T15:14:17.922664+0000","last_fullsized":"2023-09-23T15:14:17.922664+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":286,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.908436+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:14:17.908436+0000","last_peered":"2023-09-23T15:14:17.908436+0000","last_clean":"2023-09-23T15:14:17.908436+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:14:17.908436+0000","last_undegraded":"2023-09-23T15:14:17.908436+0000","last_fullsized":"2023-09-23T15:14:17.908436+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":148,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920317+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:14:17.920317+0000","last_peered":"2023-09-23T15:14:17.920317+0000","last_clean":"2023-09-23T15:14:17.920317+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:14:17.920317+0000","last_undegraded":"2023-09-23T15:14:17.920317+0000","last_fullsized":"2023-09-23T15:14:17.920317+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":178,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920480+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:14:17.920480+0000","last_peered":"2023-09-23T15:14:17.920480+0000","last_clean":"2023-09-23T15:14:17.920480+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:14:17.920480+0000","last_undegraded":"2023-09-23T15:14:17.920480+0000","last_fullsized":"2023-09-23T15:14:17.920480+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"203'8236","reported_seq":8277,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:18.226438+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:14:18.226438+0000","last_peered":"2023-09-23T15:14:18.226438+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:14:18.226438+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8236,"log_dups_size":0,"ondisk_log_size":8236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279969792,"num_objects":4272,"num_object_clones":0,"num_object_copies":12816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8236,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-602}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920747+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:14:17.920747+0000","last_peered":"2023-09-23T15:14:17.920747+0000","last_clean":"2023-09-23T15:14:17.920747+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:14:17.920747+0000","last_undegraded":"2023-09-23T15:14:17.920747+0000","last_fullsized":"2023-09-23T15:14:17.920747+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":809,"reported_epoch":203,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:17.922940+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:14:17.922940+0000","last_peered":"2023-09-23T15:14:17.922940+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:14:17.922940+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":75,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.913769+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:14:17.913769+0000","last_peered":"2023-09-23T15:14:17.913769+0000","last_clean":"2023-09-23T15:14:17.913769+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:14:17.913769+0000","last_undegraded":"2023-09-23T15:14:17.913769+0000","last_fullsized":"2023-09-23T15:14:17.913769+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":153,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.923007+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:14:17.923007+0000","last_peered":"2023-09-23T15:14:17.923007+0000","last_clean":"2023-09-23T15:14:17.923007+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:14:17.923007+0000","last_undegraded":"2023-09-23T15:14:17.923007+0000","last_fullsized":"2023-09-23T15:14:17.923007+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"204'8128","reported_seq":8169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.090817+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:14:19.090817+0000","last_peered":"2023-09-23T15:14:19.090817+0000","last_clean":"2023-09-23T15:14:19.090817+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:14:19.090817+0000","last_undegraded":"2023-09-23T15:14:19.090817+0000","last_fullsized":"2023-09-23T15:14:19.090817+0000","mapping_epoch":171,"log_start":"175'600","ondisk_log_start":"175'600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7528,"log_dups_size":0,"ondisk_log_size":7528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":273547264,"num_objects":4174,"num_object_clones":0,"num_object_copies":12522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8128,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.923090+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:14:17.923090+0000","last_peered":"2023-09-23T15:14:17.923090+0000","last_clean":"2023-09-23T15:14:17.923090+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:14:17.923090+0000","last_undegraded":"2023-09-23T15:14:17.923090+0000","last_fullsized":"2023-09-23T15:14:17.923090+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":179,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.943123+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:14:17.943123+0000","last_peered":"2023-09-23T15:14:17.943123+0000","last_clean":"2023-09-23T15:14:17.943123+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:14:17.943123+0000","last_undegraded":"2023-09-23T15:14:17.943123+0000","last_fullsized":"2023-09-23T15:14:17.943123+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":663,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.909807+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:14:17.909807+0000","last_peered":"2023-09-23T15:14:17.909807+0000","last_clean":"2023-09-23T15:14:17.909807+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:14:17.909807+0000","last_undegraded":"2023-09-23T15:14:17.909807+0000","last_fullsized":"2023-09-23T15:14:17.909807+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":184,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.939771+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:14:17.939771+0000","last_peered":"2023-09-23T15:14:17.939771+0000","last_clean":"2023-09-23T15:14:17.939771+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:14:17.939771+0000","last_undegraded":"2023-09-23T15:14:17.939771+0000","last_fullsized":"2023-09-23T15:14:17.939771+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"204'8441","reported_seq":8489,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:19.051098+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:14:19.051098+0000","last_peered":"2023-09-23T15:14:19.051098+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:14:19.051098+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"177'3100","ondisk_log_start":"177'3100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5341,"log_dups_size":0,"ondisk_log_size":5341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274530304,"num_objects":4189,"num_object_clones":0,"num_object_copies":12567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8441,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-777}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":480,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.919219+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:14:17.919219+0000","last_peered":"2023-09-23T15:14:17.919219+0000","last_clean":"2023-09-23T15:14:17.919219+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:14:17.919219+0000","last_undegraded":"2023-09-23T15:14:17.919219+0000","last_fullsized":"2023-09-23T15:14:17.919219+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":680,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.918693+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:14:17.918693+0000","last_peered":"2023-09-23T15:14:17.918693+0000","last_clean":"2023-09-23T15:14:17.918693+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:14:17.918693+0000","last_undegraded":"2023-09-23T15:14:17.918693+0000","last_fullsized":"2023-09-23T15:14:17.918693+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":166,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.920553+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:14:17.920553+0000","last_peered":"2023-09-23T15:14:17.920553+0000","last_clean":"2023-09-23T15:14:17.920553+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:14:17.920553+0000","last_undegraded":"2023-09-23T15:14:17.920553+0000","last_fullsized":"2023-09-23T15:14:17.920553+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902228+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:14:17.902228+0000","last_peered":"2023-09-23T15:14:17.902228+0000","last_clean":"2023-09-23T15:14:17.902228+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:14:17.902228+0000","last_undegraded":"2023-09-23T15:14:17.902228+0000","last_fullsized":"2023-09-23T15:14:17.902228+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":155,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.922918+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:14:17.922918+0000","last_peered":"2023-09-23T15:14:17.922918+0000","last_clean":"2023-09-23T15:14:17.922918+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:14:17.922918+0000","last_undegraded":"2023-09-23T15:14:17.922918+0000","last_fullsized":"2023-09-23T15:14:17.922918+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":202,"reported_epoch":202,"state":"active+clean","last_fresh":"2023-09-23T15:14:09.040409+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:14:09.040409+0000","last_peered":"2023-09-23T15:14:09.040409+0000","last_clean":"2023-09-23T15:14:09.040409+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:14:09.040409+0000","last_undegraded":"2023-09-23T15:14:09.040409+0000","last_fullsized":"2023-09-23T15:14:09.040409+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":182,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904678+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:14:17.904678+0000","last_peered":"2023-09-23T15:14:17.904678+0000","last_clean":"2023-09-23T15:14:17.904678+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:14:17.904678+0000","last_undegraded":"2023-09-23T15:14:17.904678+0000","last_fullsized":"2023-09-23T15:14:17.904678+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":196,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902538+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:14:17.902538+0000","last_peered":"2023-09-23T15:14:17.902538+0000","last_clean":"2023-09-23T15:14:17.902538+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:14:17.902538+0000","last_undegraded":"2023-09-23T15:14:17.902538+0000","last_fullsized":"2023-09-23T15:14:17.902538+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":148,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.904711+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:14:17.904711+0000","last_peered":"2023-09-23T15:14:17.904711+0000","last_clean":"2023-09-23T15:14:17.904711+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:14:17.904711+0000","last_undegraded":"2023-09-23T15:14:17.904711+0000","last_fullsized":"2023-09-23T15:14:17.904711+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":179,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902269+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:14:17.902269+0000","last_peered":"2023-09-23T15:14:17.902269+0000","last_clean":"2023-09-23T15:14:17.902269+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:14:17.902269+0000","last_undegraded":"2023-09-23T15:14:17.902269+0000","last_fullsized":"2023-09-23T15:14:17.902269+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":175,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.917464+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:14:17.917464+0000","last_peered":"2023-09-23T15:14:17.917464+0000","last_clean":"2023-09-23T15:14:17.917464+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:14:17.917464+0000","last_undegraded":"2023-09-23T15:14:17.917464+0000","last_fullsized":"2023-09-23T15:14:17.917464+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":189,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.917021+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:14:17.917021+0000","last_peered":"2023-09-23T15:14:17.917021+0000","last_clean":"2023-09-23T15:14:17.917021+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:14:17.917021+0000","last_undegraded":"2023-09-23T15:14:17.917021+0000","last_fullsized":"2023-09-23T15:14:17.917021+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":151,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.924282+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:14:17.924282+0000","last_peered":"2023-09-23T15:14:17.924282+0000","last_clean":"2023-09-23T15:14:17.924282+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:14:17.924282+0000","last_undegraded":"2023-09-23T15:14:17.924282+0000","last_fullsized":"2023-09-23T15:14:17.924282+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":172,"reported_epoch":203,"state":"active+clean","last_fresh":"2023-09-23T15:14:17.902098+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:14:17.902098+0000","last_peered":"2023-09-23T15:14:17.902098+0000","last_clean":"2023-09-23T15:14:17.902098+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:14:17.902098+0000","last_undegraded":"2023-09-23T15:14:17.902098+0000","last_fullsized":"2023-09-23T15:14:17.902098+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4435673112,"num_objects":67684,"num_object_clones":0,"num_object_copies":203052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24709,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":67684,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":133094,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4991,"num_bytes_recovered":327090176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12361871360,"data_stored":12361859144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112294,"ondisk_log_size":112294,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201515,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463151,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2307624,"kb_used_data":2170272,"kb_used_omap":0,"kb_used_meta":137344,"kb_avail":91437528,"statfs":{"total":95995035648,"available":93632028672,"internally_reserved":0,"allocated":2222358528,"data_stored":2221686272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":140640256},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":4.2370000000000001,"15min":2.633},"min":{"1min":0.23699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.926,"5min":95.721000000000004,"15min":108.39400000000001},"last":6.367},{"interface":"front","average":{"1min":2.863,"5min":4.4660000000000002,"15min":2.7400000000000002},"min":{"1min":0.23400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.614000000000001,"5min":96.019000000000005,"15min":108.452},"last":0.77600000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":3.371,"15min":3.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":3.3860000000000001},{"interface":"front","average":{"1min":3.6070000000000002,"5min":3.6070000000000002,"15min":3.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":20.984000000000002,"5min":20.984000000000002,"15min":20.984000000000002},"last":15.164}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":6.6059999999999999,"15min":4.8700000000000001},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.664000000000001,"5min":139.17500000000001,"15min":180.374},"last":2.2090000000000001},{"interface":"front","average":{"1min":3.3690000000000002,"5min":6.4640000000000004,"15min":4.8700000000000001},"min":{"1min":0.33300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":16.552,"5min":146.999,"15min":181.685},"last":0.65800000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":3.052,"5min":2.2149999999999999,"15min":1.169},"min":{"1min":0.34499999999999997,"5min":0.249,"15min":0.249},"max":{"1min":7.9329999999999998,"5min":17.084,"15min":17.084},"last":1.9410000000000001},{"interface":"front","average":{"1min":3.4940000000000002,"5min":2.2240000000000002,"15min":1.1679999999999999},"min":{"1min":0.42899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.6769999999999996,"5min":8.6769999999999996,"15min":8.6769999999999996},"last":9.9860000000000007}]},{"osd":4,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.8999999999999999,"15min":1.0329999999999999},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5119999999999996,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.998,"15min":1.0980000000000001},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.5760000000000001,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.46700000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.8399999999999999,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":5.5880000000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":2.0339999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9470000000000001,"15min":1.431},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4550000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.671,"15min":0.98199999999999998},"min":{"1min":0.36099999999999999,"5min":0.28999999999999998,"15min":0.28000000000000003},"max":{"1min":3.7490000000000001,"5min":17,"15min":17},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.4750000000000001,"15min":0.95599999999999996},"min":{"1min":0.33500000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.496,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.497}]}]},{"osd":1,"up_from":183,"seq":785979015189,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":924096,"kb_used_data":797624,"kb_used_omap":6,"kb_used_meta":126457,"kb_avail":92821056,"statfs":{"total":95995035648,"available":95048761344,"internally_reserved":0,"allocated":816766976,"data_stored":815793816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":129492043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.8039999999999998,"15min":3.8039999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.626000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":4.6100000000000003,"5min":4.6100000000000003,"15min":4.6100000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.131,"5min":16.131,"15min":16.131},"last":0.24399999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":13.843,"5min":13.843,"15min":13.843},"last":0.30499999999999999},{"interface":"front","average":{"1min":4.2229999999999999,"5min":4.2229999999999999,"15min":4.2229999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.658000000000001,"5min":16.658000000000001,"15min":16.658000000000001},"last":0.26200000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":123.426,"5min":123.426,"15min":123.426},"last":11.291},{"interface":"front","average":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":123.32899999999999,"5min":123.32899999999999,"15min":123.32899999999999},"last":9.1010000000000009}]},{"osd":4,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":0.68799999999999994},{"interface":"front","average":{"1min":3.5579999999999998,"5min":3.5579999999999998,"15min":3.5579999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.544,"5min":16.544,"15min":16.544},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":16.166,"5min":16.166,"15min":16.166},"last":0.63},{"interface":"front","average":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":0.51600000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":16.074000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":0.60199999999999998},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":7.7910000000000004,"5min":7.7910000000000004,"15min":7.7910000000000004},"last":0.72199999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.157,"15min":3.157},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":0.64600000000000002},{"interface":"front","average":{"1min":3.3540000000000001,"5min":3.3540000000000001,"15min":3.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":16.594000000000001,"5min":16.594000000000001,"15min":16.594000000000001},"last":0.55000000000000004}]}]},{"osd":0,"up_from":95,"seq":408021893238,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1339524,"kb_used_data":1090236,"kb_used_omap":8,"kb_used_meta":249271,"kb_avail":92405628,"statfs":{"total":95995035648,"available":94623363072,"internally_reserved":0,"allocated":1116401664,"data_stored":1115533522,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":255254100},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":31.510999999999999,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":4.71,"5min":4.71,"15min":4.71},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.378,"5min":31.378,"15min":31.378},"last":3.5209999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":5.359,"15min":2.375},"min":{"1min":0.46000000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":31.271999999999998,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.95999999999999996},{"interface":"front","average":{"1min":4.9180000000000001,"5min":4.4139999999999997,"15min":1.9850000000000001},"min":{"1min":0.51500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":31.542999999999999,"5min":122.071,"15min":122.071},"last":0.33100000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":12.659000000000001,"5min":5.0529999999999999,"15min":2.0529999999999999},"min":{"1min":0.28299999999999997,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":84.935000000000002,"5min":99.204999999999998,"15min":99.204999999999998},"last":4.4950000000000001},{"interface":"front","average":{"1min":12.394,"5min":4.1619999999999999,"15min":2.085},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":78.902000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":10.757}]},{"osd":4,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":3.0129999999999999,"15min":1.5209999999999999},"min":{"1min":0.51200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":31.341999999999999,"5min":31.341999999999999,"15min":31.341999999999999},"last":3.0470000000000002},{"interface":"front","average":{"1min":4.556,"5min":2.9670000000000001,"15min":1.538},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":31.059999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":3.0569999999999999,"15min":1.6180000000000001},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":31.25,"5min":109.369,"15min":109.369},"last":0.34799999999999998},{"interface":"front","average":{"1min":4.032,"5min":3.532,"15min":1.7350000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":31.295000000000002,"5min":109.387,"15min":109.387},"last":3.085}]},{"osd":6,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":1.5780000000000001},"min":{"1min":0.41299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.893999999999998,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":4.0350000000000001,"5min":2.9220000000000002,"15min":1.526},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":31.431000000000001,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.313}]},{"osd":7,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":3.2130000000000001,"15min":1.6259999999999999},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":31.472000000000001,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.36799999999999999},{"interface":"front","average":{"1min":4.5099999999999998,"5min":2.4510000000000001,"15min":1.4139999999999999},"min":{"1min":0.53700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":31.225999999999999,"5min":31.225999999999999,"15min":31.225999999999999},"last":3.1139999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566185,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2755020,"kb_used_data":2454276,"kb_used_omap":11,"kb_used_meta":300724,"kb_avail":90990132,"statfs":{"total":95995035648,"available":93173895168,"internally_reserved":0,"allocated":2513178624,"data_stored":2512306678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":307942160},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":5.8540000000000001,"5min":2.7320000000000002,"15min":1.25},"min":{"1min":0.23000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":38.274000000000001,"5min":38.274000000000001,"15min":38.274000000000001},"last":8.843},{"interface":"front","average":{"1min":4.6130000000000004,"5min":2.3580000000000001,"15min":1.111},"min":{"1min":0.307,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":33.026000000000003,"5min":33.026000000000003,"15min":33.026000000000003},"last":6.6859999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":5.04,"5min":5.04,"15min":5.04},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":6.2649999999999997},{"interface":"front","average":{"1min":2.5489999999999999,"5min":2.5489999999999999,"15min":2.5489999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":7.6669999999999998},"last":6.6600000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:13:22 2023","interfaces":[{"interface":"back","average":{"1min":3.8879999999999999,"5min":1.8420000000000001,"15min":1.1060000000000001},"min":{"1min":0.24299999999999999,"5min":0.219,"15min":0.219},"max":{"1min":22.233000000000001,"5min":22.233000000000001,"15min":57.695},"last":8.1969999999999992},{"interface":"front","average":{"1min":3.968,"5min":1.895,"15min":1.1519999999999999},"min":{"1min":0.24399999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":22.254999999999999,"5min":22.254999999999999,"15min":58.387999999999998},"last":6.3109999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.6190000000000002,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.6589999999999998,"5min":52.795999999999999,"15min":52.795999999999999},"last":6.5279999999999996},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.6280000000000001,"15min":1.7090000000000001},"min":{"1min":0.434,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.478999999999999,"5min":68.894999999999996,"15min":68.894999999999996},"last":8.7710000000000008}]},{"osd":5,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":2.7719999999999998,"15min":1.4790000000000001},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.161000000000001,"5min":41.723999999999997,"15min":41.723999999999997},"last":6.3419999999999996},{"interface":"front","average":{"1min":2.7200000000000002,"5min":2.7469999999999999,"15min":1.4410000000000001},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.694,"5min":39.822000000000003,"15min":39.822000000000003},"last":6.5640000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.9239999999999999,"15min":1.5329999999999999},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5800000000000001,"5min":37.834000000000003,"15min":37.834000000000003},"last":6.6349999999999998},{"interface":"front","average":{"1min":2.4910000000000001,"5min":2.3959999999999999,"15min":1.387},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.019,"5min":37.749000000000002,"15min":37.749000000000002},"last":8.8109999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.383,"15min":1.3260000000000001},"min":{"1min":0.42399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.207999999999998,"5min":18.207999999999998,"15min":18.207999999999998},"last":6.3719999999999999},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.2669999999999999,"15min":1.3149999999999999},"min":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6299999999999999,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":6.6040000000000001}]}]},{"osd":3,"up_from":110,"seq":472446402659,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3268068,"kb_used_data":2980636,"kb_used_omap":3,"kb_used_meta":287292,"kb_avail":90477084,"statfs":{"total":95995035648,"available":92648534016,"internally_reserved":0,"allocated":3052171264,"data_stored":3051337772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":294187177},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.694,"5min":3.694,"15min":1.518},"min":{"1min":0.26000000000000001,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":11.276,"5min":79.611000000000004,"15min":79.611000000000004},"last":1.2250000000000001},{"interface":"front","average":{"1min":3.0499999999999998,"5min":3.831,"15min":1.5940000000000001},"min":{"1min":0.21099999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.0809999999999995,"5min":79.656000000000006,"15min":79.656000000000006},"last":0.91500000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":4.5860000000000003,"15min":4.5860000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.315000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":1.3979999999999999},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.427,"5min":14.427,"15min":14.427},"last":1.1950000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.9769999999999999,"5min":7.5570000000000004,"15min":2.952},"min":{"1min":0.255,"5min":0.22,"15min":0.22},"max":{"1min":10.467000000000001,"5min":109.077,"15min":109.077},"last":1.1539999999999999},{"interface":"front","average":{"1min":4.0830000000000002,"5min":8.4160000000000004,"15min":3.1589999999999998},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":11.265000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":1.1040000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8929999999999998,"5min":1.8009999999999999,"15min":1.026},"min":{"1min":0.312,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":9.0399999999999991,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":1.167},{"interface":"front","average":{"1min":2.1859999999999999,"5min":1.9670000000000001,"15min":1.1950000000000001},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.6580000000000004,"5min":8.9009999999999998,"15min":8.9009999999999998},"last":1.532}]},{"osd":5,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.1099999999999999,"5min":2.6669999999999998,"15min":1.3420000000000001},"min":{"1min":0.28100000000000003,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.9450000000000003,"5min":56.960999999999999,"15min":56.960999999999999},"last":1.1379999999999999},{"interface":"front","average":{"1min":2.2330000000000001,"5min":2.9209999999999998,"15min":1.5389999999999999},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":7.593,"5min":56.981999999999999,"15min":56.981999999999999},"last":1.498}]},{"osd":6,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":2.3069999999999999,"15min":1.2290000000000001},"min":{"1min":0.34599999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.4749999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":1.2050000000000001},{"interface":"front","average":{"1min":2.2610000000000001,"5min":2.5990000000000002,"15min":1.448},"min":{"1min":0.38700000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6890000000000001,"5min":19.355,"15min":19.355},"last":1.5589999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.2469999999999999,"5min":2.6240000000000001,"15min":1.448},"min":{"1min":0.433,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":7.7489999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.589},{"interface":"front","average":{"1min":2.8100000000000001,"5min":1.827,"15min":1.0680000000000001},"min":{"1min":0.47899999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":10.406000000000001,"5min":13.403,"15min":13.403},"last":0.56000000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986424,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1545316,"kb_used_data":1356764,"kb_used_omap":0,"kb_used_meta":188544,"kb_avail":92199836,"statfs":{"total":95995035648,"available":94412632064,"internally_reserved":0,"allocated":1389326336,"data_stored":1388657655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":193069056},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":4.0780000000000003,"15min":2.2869999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":6.6799999999999997,"5min":108.148,"15min":122.69199999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.1679999999999999,"5min":3.903,"15min":2.306},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6719999999999997,"5min":108.124,"15min":122.745},"last":7.976}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":4.5490000000000004,"15min":4.5490000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":53.832000000000001,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":5.077,"5min":5.077,"15min":5.077},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.078000000000003,"5min":53.078000000000003,"15min":53.078000000000003},"last":2.7719999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":6.2439999999999998,"15min":2.7719999999999998},"min":{"1min":0.41099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":6.96,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.27700000000000002},{"interface":"front","average":{"1min":4.1719999999999997,"5min":6.6150000000000002,"15min":3.0299999999999998},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":34.798999999999999,"5min":147.88900000000001,"15min":147.88900000000001},"last":2.347}]},{"osd":3,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":20.986999999999998,"5min":7.718,"15min":3.0219999999999998},"min":{"1min":0.28899999999999998,"5min":0.252,"15min":0.252},"max":{"1min":135.607,"5min":163.97499999999999,"15min":163.97499999999999},"last":2.851},{"interface":"front","average":{"1min":20.789000000000001,"5min":7.1319999999999997,"15min":2.8220000000000001},"min":{"1min":0.38100000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":135.33600000000001,"5min":164.447,"15min":164.447},"last":3.1400000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:38 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.3660000000000001,"15min":1.198},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.998,"5min":28.891999999999999,"15min":28.891999999999999},"last":0.25900000000000001},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.8340000000000001,"15min":1.0449999999999999},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9489999999999998,"5min":32.100999999999999,"15min":32.100999999999999},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 15:13:19 2023","interfaces":[{"interface":"back","average":{"1min":0.68400000000000005,"5min":3.9449999999999998,"15min":1.73},"min":{"1min":0.34899999999999998,"5min":0.318,"15min":0.318},"max":{"1min":2.927,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.74399999999999999,"5min":3.5070000000000001,"15min":1.5980000000000001},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":2.5659999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.32400000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:13:24 2023","interfaces":[{"interface":"back","average":{"1min":0.80500000000000005,"5min":2.1059999999999999,"15min":1.1379999999999999},"min":{"1min":0.33500000000000002,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":4.5869999999999997,"5min":18.23,"15min":18.23},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.78800000000000003,"5min":2.0270000000000001,"15min":1.149},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.879,"5min":14.653,"15min":14.653},"last":0.42099999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724787,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1538408,"kb_used_data":1352352,"kb_used_omap":0,"kb_used_meta":186048,"kb_avail":92206744,"statfs":{"total":95995035648,"available":94419705856,"internally_reserved":0,"allocated":1384808448,"data_stored":1384139731,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":190513152},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":2.0859999999999999,"15min":1.2190000000000001},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.968,"5min":17.044,"15min":24.962},"last":0.40500000000000003},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.823,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.95,"5min":18.814,"15min":25.317},"last":0.42399999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.673,"15min":1.673},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.717000000000001,"5min":11.717000000000001,"15min":11.717000000000001},"last":0.49399999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":4.7000000000000002,"15min":2.097},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.7859999999999996,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.47499999999999998},{"interface":"front","average":{"1min":4.5129999999999999,"5min":4.4930000000000003,"15min":2.0619999999999998},"min":{"1min":0.33400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":13.332000000000001,"5min":97.305999999999997,"15min":97.305999999999997},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":2.3519999999999999,"5min":3.2109999999999999,"15min":1.5129999999999999},"min":{"1min":0.34399999999999997,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.7770000000000001,"5min":53.088999999999999,"15min":53.088999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.093,"5min":2.8260000000000001,"15min":1.4350000000000001},"min":{"1min":0.47299999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":11.121,"5min":53.764000000000003,"15min":53.764000000000003},"last":0.68000000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.298,"15min":1.163},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7669999999999999,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.622,"5min":2.5449999999999999,"15min":1.242},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.1850000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":2.0539999999999998,"15min":1.107},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8760000000000003,"5min":15.196,"15min":15.196},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.752,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":1.6419999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.69299999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:13:30 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.3519999999999999,"15min":1.2270000000000001},"min":{"1min":0.38300000000000001,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":5.4260000000000002,"5min":15.301,"15min":15.301},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.95799999999999996,"5min":2.2410000000000001,"15min":1.2070000000000001},"min":{"1min":0.40999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":3.0249999999999999,"5min":15.762,"15min":15.762},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1105596416,"data_stored":1105592344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":727060480,"data_stored":727056408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2509111296,"data_stored":2509111296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3043098624,"data_stored":3043098624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1383727104,"data_stored":1383727104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1376129024,"data_stored":1376124952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2217148416,"data_stored":2217148416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:14:25.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:14:25.523 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:25.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:25.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:25.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:14:25.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:25.781 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:25.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:26.000 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:26.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:26.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:26.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:26.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:26 smithi017 ceph-mon[222878]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:26.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:26 smithi017 ceph-mon[222878]: pgmap v534: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 24712/199509 objects degraded (12.386%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T15:14:26.198 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:26 smithi017 ceph-mon[215646]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:26.198 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:26 smithi017 ceph-mon[215646]: pgmap v534: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 24712/199509 objects degraded (12.386%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T15:14:26.201 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:26.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:26.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:26.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:26.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:26 smithi027 ceph-mon[195233]: from='client.45269 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:26.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:26 smithi027 ceph-mon[195233]: pgmap v534: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 449 op/s; 24712/199509 objects degraded (12.386%); 2.3 MiB/s, 37 objects/s recovering 2023-09-23T15:14:26.402 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:26.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:26.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:26.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:26.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:26.904 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:27.026 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:27.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:27.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:27.134 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:27.166 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:27.231 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:27.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:27.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:27.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:27.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24712/199509 objects degraded (12.386%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:27.384 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:27.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24712/199509 objects degraded (12.386%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:27.444 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:27.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24712/199509 objects degraded (12.386%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:27.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:27.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:27.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:14:27.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:27.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:27.722 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:27.735 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:27.864 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:27.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:27.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:28.129 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:28 smithi027 ceph-mon[195233]: pgmap v535: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 24589/194700 objects degraded (12.629%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:14:28.383 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:28.386 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:28 smithi017 ceph-mon[215646]: pgmap v535: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 24589/194700 objects degraded (12.629%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:14:28.387 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:28] "GET /metrics HTTP/1.1" 200 36324 "" "Prometheus/2.43.0" 2023-09-23T15:14:28.389 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:28 smithi017 ceph-mon[222878]: pgmap v535: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 615 op/s; 24589/194700 objects degraded (12.629%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:14:28.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:28.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:28.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:28.523 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:28.587 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:28.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:28.598 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:28.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:28.794 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:29.364 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:29.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:29.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:29.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:29.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:29.478 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:29.518 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:29.576 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:29.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:29.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:29.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:29.770 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:30.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:30 smithi017 ceph-mon[215646]: pgmap v536: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 24589/194700 objects degraded (12.629%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T15:14:30.078 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:30.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:30.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:30.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:30.287 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:30 smithi027 ceph-mon[195233]: pgmap v536: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 24589/194700 objects degraded (12.629%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T15:14:30.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:30 smithi017 ceph-mon[222878]: pgmap v536: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 24589/194700 objects degraded (12.629%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T15:14:30.527 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:30.558 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:30.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:30.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:30.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:30.700 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:30.753 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:30.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:30.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:30.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:30.990 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:31.045 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:14:31.049 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:14:31.110 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:14:31.141 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:14:31.142 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:14:31.142 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:14:31.142 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:14:31.142 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:14:31.322 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:31.323 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:14:31.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:31.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:31.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:31.567 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:32.057 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:32.058 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 24480/186381 objects degraded (13.134%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":679,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":205,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":62127,"data_bytes":4074477788,"bytes_used":13003415552,"bytes_avail":658961833984,"bytes_total":671965249536,"degraded_objects":24480,"degraded_total":186381,"degraded_ratio":0.13134386015741947,"recovering_objects_per_sec":46,"recovering_bytes_per_sec":3043806,"recovering_keys_per_sec":0,"num_objects_recovered":532,"num_bytes_recovered":34865152,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":652},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (5m)\n [==========================..] (remaining: 22s)","progress":0.93577980995178223,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:14:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: pgmap v537: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 24480/186381 objects degraded (13.134%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24480/186381 objects degraded (13.134%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: pgmap v537: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 24480/186381 objects degraded (13.134%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:14:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24480/186381 objects degraded (13.134%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: pgmap v537: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s; 24480/186381 objects degraded (13.134%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2650569998' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:14:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: osdmap e205: 8 total, 8 up, 7 in 2023-09-23T15:14:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: failed to encode map e205 with expected crc 2023-09-23T15:14:32.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24480/186381 objects degraded (13.134%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:32.917 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:32.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:32.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:33.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:33.068 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:33.114 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:33.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:33.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:33.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1060824772' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:33.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:33 smithi017 ceph-mon[215646]: failed to encode map e205 with expected crc 2023-09-23T15:14:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1060824772' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:33 smithi017 ceph-mon[222878]: failed to encode map e205 with expected crc 2023-09-23T15:14:33.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:33.316 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:33.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1060824772' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:14:33.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:33 smithi027 ceph-mon[195233]: failed to encode map e205 with expected crc 2023-09-23T15:14:33.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:14:33.625 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:33.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:33.630 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:33.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:33.820 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:34 smithi027 ceph-mon[195233]: pgmap v539: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 24480/186381 objects degraded (13.134%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:34.402 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:34.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:34 smithi017 ceph-mon[222878]: pgmap v539: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 24480/186381 objects degraded (13.134%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:34.406 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:34 smithi017 ceph-mon[215646]: pgmap v539: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 558 op/s; 24480/186381 objects degraded (13.134%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:34.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:34.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:34.443 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:14:34.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:14:34.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:34.649 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:34.816 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:34.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:34.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:34.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:34.926 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:34.952 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:35.012 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:35.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:35.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:35.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:35.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:35 smithi017 ceph-mon[215646]: pgmap v540: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 24480/182892 objects degraded (13.385%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:35.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/177548324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:35.105 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:35 smithi017 ceph-mon[222878]: pgmap v540: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 24480/182892 objects degraded (13.385%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:35.105 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/177548324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:35.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:35.134 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:35.158 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:35.184 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:35.209 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:35.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:35.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:35.217 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:35 smithi027 ceph-mon[195233]: pgmap v540: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 553 op/s; 24480/182892 objects degraded (13.385%); 1.4 MiB/s, 23 objects/s recovering 2023-09-23T15:14:35.217 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/177548324' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:35.227 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:35.282 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:35.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:14:35.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:35.336 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:35.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:35.421 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:35.440 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:14:35.475 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:14:36.494 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:36.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:36.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:36.601 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:36.677 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:37.334 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:37.366 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:37.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:37.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:37.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:37.512 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:37.553 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:37.563 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:37.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:37.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:37.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:37 smithi017 ceph-mon[222878]: pgmap v541: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:37.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24371/178449 objects degraded (13.657%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:37.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:37 smithi017 ceph-mon[215646]: pgmap v541: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:37.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24371/178449 objects degraded (13.657%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:37.815 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:37.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:37 smithi027 ceph-mon[195233]: pgmap v541: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:37.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24371/178449 objects degraded (13.657%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:38.062 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:38.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:38.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:38.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:38.285 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:38] "GET /metrics HTTP/1.1" 200 36325 "" "Prometheus/2.43.0" 2023-09-23T15:14:38.313 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:38.354 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:14:38.354 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:14:38.469 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:38.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:38.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:38.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:38.643 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:39.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:14:39.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:39 smithi017 ceph-mon[215646]: pgmap v542: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:39.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:39 smithi017 ceph-mon[222878]: pgmap v542: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:39.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:39 smithi027 ceph-mon[195233]: pgmap v542: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 24371/178449 objects degraded (13.657%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T15:14:40.696 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:14:40.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:40.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:40.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:40.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:40.889 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:40.953 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:40.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:40.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:41.061 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:41.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:41.155 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:41.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:41.407 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:41.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:41.416 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:41.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:41.625 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:41.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:41 smithi017 ceph-mon[222878]: pgmap v543: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24252/170403 objects degraded (14.232%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:41.704 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:41 smithi017 ceph-mon[215646]: pgmap v543: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24252/170403 objects degraded (14.232%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:41.723 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:41 smithi027 ceph-mon[195233]: pgmap v543: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 24252/170403 objects degraded (14.232%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:41.758 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:41.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:41.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:41.976 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:42.044 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:42.086 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:14:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24252/170403 objects degraded (14.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[222878]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2232296540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24252/170403 objects degraded (14.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:42.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[215646]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:42.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2232296540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24252/170403 objects degraded (14.232%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:42 smithi027 ceph-mon[195233]: from='client.45275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2232296540' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:14:42.954 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: pgmap v544: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 24252/170403 objects degraded (14.232%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:43.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: pgmap v544: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 24252/170403 objects degraded (14.232%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T15:14:43.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:14:43.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:43.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T15:14:43.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:43.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:43.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: pgmap v544: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 470 op/s; 24252/170403 objects degraded (14.232%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T15:14:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:14:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:14:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: osdmap e206: 8 total, 8 up, 7 in 2023-09-23T15:14:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:44.213 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:44.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:44.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:44.305 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":543,"stamp":"2023-09-23T15:14:40.365621+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3725433052,"num_objects":56801,"num_object_clones":0,"num_object_copies":170403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24252,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56801,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":145032,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5510,"num_bytes_recovered":362584231,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120717,"ondisk_log_size":120717,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11829624,"kb_used_data":10295424,"kb_used_omap":30,"kb_used_meta":1534113,"kb_avail":644386440,"statfs":{"total":671965249536,"available":659851714560,"internally_reserved":0,"allocated":10542514176,"data_stored":10536902322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1570932312},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-349044736,"num_objects":-5326,"num_object_clones":0,"num_object_copies":-15978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-228,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":228,"num_bytes_recovered":14942208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009364"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":199,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965498+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:14:18.965498+0000","last_peered":"2023-09-23T15:14:18.965498+0000","last_clean":"2023-09-23T15:14:18.965498+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:14:18.965498+0000","last_undegraded":"2023-09-23T15:14:18.965498+0000","last_fullsized":"2023-09-23T15:14:18.965498+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":133,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918568+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:14:18.918568+0000","last_peered":"2023-09-23T15:14:18.918568+0000","last_clean":"2023-09-23T15:14:18.918568+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:14:18.918568+0000","last_undegraded":"2023-09-23T15:14:18.918568+0000","last_fullsized":"2023-09-23T15:14:18.918568+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910954+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:14:18.910954+0000","last_peered":"2023-09-23T15:14:18.910954+0000","last_clean":"2023-09-23T15:14:18.910954+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:14:18.910954+0000","last_undegraded":"2023-09-23T15:14:18.910954+0000","last_fullsized":"2023-09-23T15:14:18.910954+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086695+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:14:31.086695+0000","last_peered":"2023-09-23T15:14:31.086695+0000","last_clean":"2023-09-23T15:14:31.086695+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:14:31.086695+0000","last_undegraded":"2023-09-23T15:14:31.086695+0000","last_fullsized":"2023-09-23T15:14:31.086695+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":202,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918175+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:14:18.918175+0000","last_peered":"2023-09-23T15:14:18.918175+0000","last_clean":"2023-09-23T15:14:18.918175+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:14:18.918175+0000","last_undegraded":"2023-09-23T15:14:18.918175+0000","last_fullsized":"2023-09-23T15:14:18.918175+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":179,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.964060+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:14:18.964060+0000","last_peered":"2023-09-23T15:14:18.964060+0000","last_clean":"2023-09-23T15:14:18.964060+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:14:18.964060+0000","last_undegraded":"2023-09-23T15:14:18.964060+0000","last_fullsized":"2023-09-23T15:14:18.964060+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.409644+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:14:19.409644+0000","last_peered":"2023-09-23T15:14:19.409644+0000","last_clean":"2023-09-23T15:14:19.409644+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:14:19.409644+0000","last_undegraded":"2023-09-23T15:14:19.409644+0000","last_fullsized":"2023-09-23T15:14:19.409644+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":184,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918176+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:14:18.918176+0000","last_peered":"2023-09-23T15:14:18.918176+0000","last_clean":"2023-09-23T15:14:18.918176+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:14:18.918176+0000","last_undegraded":"2023-09-23T15:14:18.918176+0000","last_fullsized":"2023-09-23T15:14:18.918176+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909042+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:14:18.909042+0000","last_peered":"2023-09-23T15:14:18.909042+0000","last_clean":"2023-09-23T15:14:18.909042+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:14:18.909042+0000","last_undegraded":"2023-09-23T15:14:18.909042+0000","last_fullsized":"2023-09-23T15:14:18.909042+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906891+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:14:18.906891+0000","last_peered":"2023-09-23T15:14:18.906891+0000","last_clean":"2023-09-23T15:14:18.906891+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:14:18.906891+0000","last_undegraded":"2023-09-23T15:14:18.906891+0000","last_fullsized":"2023-09-23T15:14:18.906891+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":127,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918239+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:14:18.918239+0000","last_peered":"2023-09-23T15:14:18.918239+0000","last_clean":"2023-09-23T15:14:18.918239+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:14:18.918239+0000","last_undegraded":"2023-09-23T15:14:18.918239+0000","last_fullsized":"2023-09-23T15:14:18.918239+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":67,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918492+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:14:18.918492+0000","last_peered":"2023-09-23T15:14:18.918492+0000","last_clean":"2023-09-23T15:14:18.918492+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:14:18.918492+0000","last_undegraded":"2023-09-23T15:14:18.918492+0000","last_fullsized":"2023-09-23T15:14:18.918492+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":76,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911155+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:14:18.911155+0000","last_peered":"2023-09-23T15:14:18.911155+0000","last_clean":"2023-09-23T15:14:18.911155+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:14:18.911155+0000","last_undegraded":"2023-09-23T15:14:18.911155+0000","last_fullsized":"2023-09-23T15:14:18.911155+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":189,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965407+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:14:18.965407+0000","last_peered":"2023-09-23T15:14:18.965407+0000","last_clean":"2023-09-23T15:14:18.965407+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:14:18.965407+0000","last_undegraded":"2023-09-23T15:14:18.965407+0000","last_fullsized":"2023-09-23T15:14:18.965407+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965627+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:14:18.965627+0000","last_peered":"2023-09-23T15:14:18.965627+0000","last_clean":"2023-09-23T15:14:18.965627+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:14:18.965627+0000","last_undegraded":"2023-09-23T15:14:18.965627+0000","last_fullsized":"2023-09-23T15:14:18.965627+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910580+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:14:18.910580+0000","last_peered":"2023-09-23T15:14:18.910580+0000","last_clean":"2023-09-23T15:14:18.910580+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:14:18.910580+0000","last_undegraded":"2023-09-23T15:14:18.910580+0000","last_fullsized":"2023-09-23T15:14:18.910580+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":67,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965713+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:14:18.965713+0000","last_peered":"2023-09-23T15:14:18.965713+0000","last_clean":"2023-09-23T15:14:18.965713+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:14:18.965713+0000","last_undegraded":"2023-09-23T15:14:18.965713+0000","last_fullsized":"2023-09-23T15:14:18.965713+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.919470+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:14:18.919470+0000","last_peered":"2023-09-23T15:14:18.919470+0000","last_clean":"2023-09-23T15:14:18.919470+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:14:18.919470+0000","last_undegraded":"2023-09-23T15:14:18.919470+0000","last_fullsized":"2023-09-23T15:14:18.919470+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909634+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:14:18.909634+0000","last_peered":"2023-09-23T15:14:18.909634+0000","last_clean":"2023-09-23T15:14:18.909634+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:14:18.909634+0000","last_undegraded":"2023-09-23T15:14:18.909634+0000","last_fullsized":"2023-09-23T15:14:18.909634+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":169,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910842+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:14:18.910842+0000","last_peered":"2023-09-23T15:14:18.910842+0000","last_clean":"2023-09-23T15:14:18.910842+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:14:18.910842+0000","last_undegraded":"2023-09-23T15:14:18.910842+0000","last_fullsized":"2023-09-23T15:14:18.910842+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.907097+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:14:18.907097+0000","last_peered":"2023-09-23T15:14:18.907097+0000","last_clean":"2023-09-23T15:14:18.907097+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:14:18.907097+0000","last_undegraded":"2023-09-23T15:14:18.907097+0000","last_fullsized":"2023-09-23T15:14:18.907097+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908801+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:14:18.908801+0000","last_peered":"2023-09-23T15:14:18.908801+0000","last_clean":"2023-09-23T15:14:18.908801+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:14:18.908801+0000","last_undegraded":"2023-09-23T15:14:18.908801+0000","last_fullsized":"2023-09-23T15:14:18.908801+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":51,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965346+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:14:18.965346+0000","last_peered":"2023-09-23T15:14:18.965346+0000","last_clean":"2023-09-23T15:14:18.965346+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:14:18.965346+0000","last_undegraded":"2023-09-23T15:14:18.965346+0000","last_fullsized":"2023-09-23T15:14:18.965346+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":158,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:19.409599+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:14:19.409599+0000","last_peered":"2023-09-23T15:14:19.409599+0000","last_clean":"2023-09-23T15:14:19.409599+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:14:19.409599+0000","last_undegraded":"2023-09-23T15:14:19.409599+0000","last_fullsized":"2023-09-23T15:14:19.409599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911097+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:14:18.911097+0000","last_peered":"2023-09-23T15:14:18.911097+0000","last_clean":"2023-09-23T15:14:18.911097+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:14:18.911097+0000","last_undegraded":"2023-09-23T15:14:18.911097+0000","last_fullsized":"2023-09-23T15:14:18.911097+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":171,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.963077+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:14:18.963077+0000","last_peered":"2023-09-23T15:14:18.963077+0000","last_clean":"2023-09-23T15:14:18.963077+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:14:18.963077+0000","last_undegraded":"2023-09-23T15:14:18.963077+0000","last_fullsized":"2023-09-23T15:14:18.963077+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":195,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918733+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:14:18.918733+0000","last_peered":"2023-09-23T15:14:18.918733+0000","last_clean":"2023-09-23T15:14:18.918733+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:14:18.918733+0000","last_undegraded":"2023-09-23T15:14:18.918733+0000","last_fullsized":"2023-09-23T15:14:18.918733+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"205'8948","reported_seq":8990,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.642582+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:14:39.642582+0000","last_peered":"2023-09-23T15:14:39.642582+0000","last_clean":"2023-09-23T15:14:39.642582+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:14:39.642582+0000","last_undegraded":"2023-09-23T15:14:39.642582+0000","last_fullsized":"2023-09-23T15:14:39.642582+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8948,"log_dups_size":0,"ondisk_log_size":8948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225837056,"num_objects":3446,"num_object_clones":0,"num_object_copies":10338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8948,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909114+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:14:18.909114+0000","last_peered":"2023-09-23T15:14:18.909114+0000","last_clean":"2023-09-23T15:14:18.909114+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:14:18.909114+0000","last_undegraded":"2023-09-23T15:14:18.909114+0000","last_fullsized":"2023-09-23T15:14:18.909114+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":79,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.966324+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:14:18.966324+0000","last_peered":"2023-09-23T15:14:18.966324+0000","last_clean":"2023-09-23T15:14:18.966324+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:14:18.966324+0000","last_undegraded":"2023-09-23T15:14:18.966324+0000","last_fullsized":"2023-09-23T15:14:18.966324+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":156,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906611+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:14:18.906611+0000","last_peered":"2023-09-23T15:14:18.906611+0000","last_clean":"2023-09-23T15:14:18.906611+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:14:18.906611+0000","last_undegraded":"2023-09-23T15:14:18.906611+0000","last_fullsized":"2023-09-23T15:14:18.906611+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"205'9112","reported_seq":10265,"reported_epoch":205,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:39.499449+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:14:39.499449+0000","last_peered":"2023-09-23T15:14:39.499449+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:14:39.499449+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"177'2700","ondisk_log_start":"177'2700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6412,"log_dups_size":0,"ondisk_log_size":6412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232914944,"num_objects":3554,"num_object_clones":0,"num_object_copies":10662,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4447,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3554,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9112,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":551,"num_bytes_recovered":36110336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-893}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":472,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:18.908977+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:14:18.908977+0000","last_peered":"2023-09-23T15:14:18.908977+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:14:18.908977+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":79,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.966306+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:14:18.966306+0000","last_peered":"2023-09-23T15:14:18.966306+0000","last_clean":"2023-09-23T15:14:18.966306+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:14:18.966306+0000","last_undegraded":"2023-09-23T15:14:18.966306+0000","last_fullsized":"2023-09-23T15:14:18.966306+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965221+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:14:18.965221+0000","last_peered":"2023-09-23T15:14:18.965221+0000","last_clean":"2023-09-23T15:14:18.965221+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:14:18.965221+0000","last_undegraded":"2023-09-23T15:14:18.965221+0000","last_fullsized":"2023-09-23T15:14:18.965221+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"205'8869","reported_seq":8913,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:38.468964+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:14:38.468964+0000","last_peered":"2023-09-23T15:14:38.468964+0000","last_clean":"2023-09-23T15:14:38.468964+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:14:38.468964+0000","last_undegraded":"2023-09-23T15:14:38.468964+0000","last_fullsized":"2023-09-23T15:14:38.468964+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8869,"log_dups_size":0,"ondisk_log_size":8869,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234160128,"num_objects":3573,"num_object_clones":0,"num_object_copies":10719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8869,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908989+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:14:18.908989+0000","last_peered":"2023-09-23T15:14:18.908989+0000","last_clean":"2023-09-23T15:14:18.908989+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:14:18.908989+0000","last_undegraded":"2023-09-23T15:14:18.908989+0000","last_fullsized":"2023-09-23T15:14:18.908989+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":65,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.907049+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:14:18.907049+0000","last_peered":"2023-09-23T15:14:18.907049+0000","last_clean":"2023-09-23T15:14:18.907049+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:14:18.907049+0000","last_undegraded":"2023-09-23T15:14:18.907049+0000","last_fullsized":"2023-09-23T15:14:18.907049+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":193,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.963644+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:14:18.963644+0000","last_peered":"2023-09-23T15:14:18.963644+0000","last_clean":"2023-09-23T15:14:18.963644+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:14:18.963644+0000","last_undegraded":"2023-09-23T15:14:18.963644+0000","last_fullsized":"2023-09-23T15:14:18.963644+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"205'9046","reported_seq":9090,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.398731+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:14:39.398731+0000","last_peered":"2023-09-23T15:14:39.398731+0000","last_clean":"2023-09-23T15:14:39.398731+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:14:39.398731+0000","last_undegraded":"2023-09-23T15:14:39.398731+0000","last_fullsized":"2023-09-23T15:14:39.398731+0000","mapping_epoch":171,"log_start":"178'3400","ondisk_log_start":"178'3400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5646,"log_dups_size":0,"ondisk_log_size":5646,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9046,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908973+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:14:18.908973+0000","last_peered":"2023-09-23T15:14:18.908973+0000","last_clean":"2023-09-23T15:14:18.908973+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:14:18.908973+0000","last_undegraded":"2023-09-23T15:14:18.908973+0000","last_fullsized":"2023-09-23T15:14:18.908973+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.907281+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:18.907281+0000","last_peered":"2023-09-23T15:14:18.907281+0000","last_clean":"2023-09-23T15:14:18.907281+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:14:18.907281+0000","last_undegraded":"2023-09-23T15:14:18.907281+0000","last_fullsized":"2023-09-23T15:14:18.907281+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908969+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:14:18.908969+0000","last_peered":"2023-09-23T15:14:18.908969+0000","last_clean":"2023-09-23T15:14:18.908969+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:14:18.908969+0000","last_undegraded":"2023-09-23T15:14:18.908969+0000","last_fullsized":"2023-09-23T15:14:18.908969+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"205'9215","reported_seq":9263,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:39.340083+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:14:39.340083+0000","last_peered":"2023-09-23T15:14:39.340083+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:14:39.340083+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"175'1700","ondisk_log_start":"175'1700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7515,"log_dups_size":0,"ondisk_log_size":7515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242286592,"num_objects":3697,"num_object_clones":0,"num_object_copies":11091,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3697,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9215,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-1313}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910375+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:14:18.910375+0000","last_peered":"2023-09-23T15:14:18.910375+0000","last_clean":"2023-09-23T15:14:18.910375+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:14:18.910375+0000","last_undegraded":"2023-09-23T15:14:18.910375+0000","last_fullsized":"2023-09-23T15:14:18.910375+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.921568+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:14:18.921568+0000","last_peered":"2023-09-23T15:14:18.921568+0000","last_clean":"2023-09-23T15:14:18.921568+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:14:18.921568+0000","last_undegraded":"2023-09-23T15:14:18.921568+0000","last_fullsized":"2023-09-23T15:14:18.921568+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965491+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:14:18.965491+0000","last_peered":"2023-09-23T15:14:18.965491+0000","last_clean":"2023-09-23T15:14:18.965491+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:14:18.965491+0000","last_undegraded":"2023-09-23T15:14:18.965491+0000","last_fullsized":"2023-09-23T15:14:18.965491+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"205'9051","reported_seq":9095,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:38.468038+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:14:38.468038+0000","last_peered":"2023-09-23T15:14:38.468038+0000","last_clean":"2023-09-23T15:14:38.468038+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:14:38.468038+0000","last_undegraded":"2023-09-23T15:14:38.468038+0000","last_fullsized":"2023-09-23T15:14:38.468038+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9051,"log_dups_size":0,"ondisk_log_size":9051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231145472,"num_objects":3527,"num_object_clones":0,"num_object_copies":10581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3527,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9051,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":498,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909703+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:14:18.909703+0000","last_peered":"2023-09-23T15:14:18.909703+0000","last_clean":"2023-09-23T15:14:18.909703+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:14:18.909703+0000","last_undegraded":"2023-09-23T15:14:18.909703+0000","last_fullsized":"2023-09-23T15:14:18.909703+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.963556+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:14:18.963556+0000","last_peered":"2023-09-23T15:14:18.963556+0000","last_clean":"2023-09-23T15:14:18.963556+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:14:18.963556+0000","last_undegraded":"2023-09-23T15:14:18.963556+0000","last_fullsized":"2023-09-23T15:14:18.963556+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965480+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:14:18.965480+0000","last_peered":"2023-09-23T15:14:18.965480+0000","last_clean":"2023-09-23T15:14:18.965480+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:14:18.965480+0000","last_undegraded":"2023-09-23T15:14:18.965480+0000","last_fullsized":"2023-09-23T15:14:18.965480+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"205'9021","reported_seq":9064,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.515068+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:14:39.515068+0000","last_peered":"2023-09-23T15:14:39.515068+0000","last_clean":"2023-09-23T15:14:39.515068+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:14:39.515068+0000","last_undegraded":"2023-09-23T15:14:39.515068+0000","last_fullsized":"2023-09-23T15:14:39.515068+0000","mapping_epoch":171,"log_start":"177'2600","ondisk_log_start":"177'2600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233111552,"num_objects":3557,"num_object_clones":0,"num_object_copies":10671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9021,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908882+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:14:18.908882+0000","last_peered":"2023-09-23T15:14:18.908882+0000","last_clean":"2023-09-23T15:14:18.908882+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:14:18.908882+0000","last_undegraded":"2023-09-23T15:14:18.908882+0000","last_fullsized":"2023-09-23T15:14:18.908882+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":188,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965117+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:14:18.965117+0000","last_peered":"2023-09-23T15:14:18.965117+0000","last_clean":"2023-09-23T15:14:18.965117+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:14:18.965117+0000","last_undegraded":"2023-09-23T15:14:18.965117+0000","last_fullsized":"2023-09-23T15:14:18.965117+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.966364+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:14:18.966364+0000","last_peered":"2023-09-23T15:14:18.966364+0000","last_clean":"2023-09-23T15:14:18.966364+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:14:18.966364+0000","last_undegraded":"2023-09-23T15:14:18.966364+0000","last_fullsized":"2023-09-23T15:14:18.966364+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"205'8986","reported_seq":18849,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.546943+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:14:39.546943+0000","last_peered":"2023-09-23T15:14:39.546943+0000","last_clean":"2023-09-23T15:14:39.546943+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:14:39.546943+0000","last_undegraded":"2023-09-23T15:14:39.546943+0000","last_fullsized":"2023-09-23T15:14:39.546943+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8986,"log_dups_size":0,"ondisk_log_size":8986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232325144,"num_objects":3546,"num_object_clones":0,"num_object_copies":10638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3546,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8986,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911031+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:14:18.911031+0000","last_peered":"2023-09-23T15:14:18.911031+0000","last_clean":"2023-09-23T15:14:18.911031+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:14:18.911031+0000","last_undegraded":"2023-09-23T15:14:18.911031+0000","last_fullsized":"2023-09-23T15:14:18.911031+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":196,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918098+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:14:18.918098+0000","last_peered":"2023-09-23T15:14:18.918098+0000","last_clean":"2023-09-23T15:14:18.918098+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:14:18.918098+0000","last_undegraded":"2023-09-23T15:14:18.918098+0000","last_fullsized":"2023-09-23T15:14:18.918098+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":81,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.912115+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:14:18.912115+0000","last_peered":"2023-09-23T15:14:18.912115+0000","last_clean":"2023-09-23T15:14:18.912115+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:14:18.912115+0000","last_undegraded":"2023-09-23T15:14:18.912115+0000","last_fullsized":"2023-09-23T15:14:18.912115+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"205'9058","reported_seq":9102,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.356337+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:14:39.356337+0000","last_peered":"2023-09-23T15:14:39.356337+0000","last_clean":"2023-09-23T15:14:39.356337+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:14:39.356337+0000","last_undegraded":"2023-09-23T15:14:39.356337+0000","last_fullsized":"2023-09-23T15:14:39.356337+0000","mapping_epoch":171,"log_start":"178'3500","ondisk_log_start":"178'3500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5558,"log_dups_size":0,"ondisk_log_size":5558,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230031360,"num_objects":3510,"num_object_clones":0,"num_object_copies":10530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9058,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909079+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:14:18.909079+0000","last_peered":"2023-09-23T15:14:18.909079+0000","last_clean":"2023-09-23T15:14:18.909079+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:14:18.909079+0000","last_undegraded":"2023-09-23T15:14:18.909079+0000","last_fullsized":"2023-09-23T15:14:18.909079+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":380,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911808+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:14:18.911808+0000","last_peered":"2023-09-23T15:14:18.911808+0000","last_clean":"2023-09-23T15:14:18.911808+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:14:18.911808+0000","last_undegraded":"2023-09-23T15:14:18.911808+0000","last_fullsized":"2023-09-23T15:14:18.911808+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":181,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906983+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:18.906983+0000","last_peered":"2023-09-23T15:14:18.906983+0000","last_clean":"2023-09-23T15:14:18.906983+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:14:18.906983+0000","last_undegraded":"2023-09-23T15:14:18.906983+0000","last_fullsized":"2023-09-23T15:14:18.906983+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.969169+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:14:18.969169+0000","last_peered":"2023-09-23T15:14:18.969169+0000","last_clean":"2023-09-23T15:14:18.969169+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:14:18.969169+0000","last_undegraded":"2023-09-23T15:14:18.969169+0000","last_fullsized":"2023-09-23T15:14:18.969169+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"205'8858","reported_seq":8902,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.470328+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:14:39.470328+0000","last_peered":"2023-09-23T15:14:39.470328+0000","last_clean":"2023-09-23T15:14:39.470328+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:14:39.470328+0000","last_undegraded":"2023-09-23T15:14:39.470328+0000","last_fullsized":"2023-09-23T15:14:39.470328+0000","mapping_epoch":171,"log_start":"176'2400","ondisk_log_start":"176'2400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6458,"log_dups_size":0,"ondisk_log_size":6458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229113856,"num_objects":3496,"num_object_clones":0,"num_object_copies":10488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3496,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8858,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910670+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:14:18.910670+0000","last_peered":"2023-09-23T15:14:18.910670+0000","last_clean":"2023-09-23T15:14:18.910670+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:14:18.910670+0000","last_undegraded":"2023-09-23T15:14:18.910670+0000","last_fullsized":"2023-09-23T15:14:18.910670+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":723,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906732+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:14:18.906732+0000","last_peered":"2023-09-23T15:14:18.906732+0000","last_clean":"2023-09-23T15:14:18.906732+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:14:18.906732+0000","last_undegraded":"2023-09-23T15:14:18.906732+0000","last_fullsized":"2023-09-23T15:14:18.906732+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":164,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965542+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:14:18.965542+0000","last_peered":"2023-09-23T15:14:18.965542+0000","last_clean":"2023-09-23T15:14:18.965542+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:14:18.965542+0000","last_undegraded":"2023-09-23T15:14:18.965542+0000","last_fullsized":"2023-09-23T15:14:18.965542+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911596+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:14:18.911596+0000","last_peered":"2023-09-23T15:14:18.911596+0000","last_clean":"2023-09-23T15:14:18.911596+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:14:18.911596+0000","last_undegraded":"2023-09-23T15:14:18.911596+0000","last_fullsized":"2023-09-23T15:14:18.911596+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"205'9054","reported_seq":9096,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.180221+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:14:39.180221+0000","last_peered":"2023-09-23T15:14:39.180221+0000","last_clean":"2023-09-23T15:14:39.180221+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:14:39.180221+0000","last_undegraded":"2023-09-23T15:14:39.180221+0000","last_fullsized":"2023-09-23T15:14:39.180221+0000","mapping_epoch":171,"log_start":"178'3700","ondisk_log_start":"178'3700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5354,"log_dups_size":0,"ondisk_log_size":5354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234094592,"num_objects":3572,"num_object_clones":0,"num_object_copies":10716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9054,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910940+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:14:18.910940+0000","last_peered":"2023-09-23T15:14:18.910940+0000","last_clean":"2023-09-23T15:14:18.910940+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:14:18.910940+0000","last_undegraded":"2023-09-23T15:14:18.910940+0000","last_fullsized":"2023-09-23T15:14:18.910940+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":817,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.970325+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:14:18.970325+0000","last_peered":"2023-09-23T15:14:18.970325+0000","last_clean":"2023-09-23T15:14:18.970325+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:14:18.970325+0000","last_undegraded":"2023-09-23T15:14:18.970325+0000","last_fullsized":"2023-09-23T15:14:18.970325+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":76,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910757+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:14:18.910757+0000","last_peered":"2023-09-23T15:14:18.910757+0000","last_clean":"2023-09-23T15:14:18.910757+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:14:18.910757+0000","last_undegraded":"2023-09-23T15:14:18.910757+0000","last_fullsized":"2023-09-23T15:14:18.910757+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":100,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.912105+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:14:18.912105+0000","last_peered":"2023-09-23T15:14:18.912105+0000","last_clean":"2023-09-23T15:14:18.912105+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:14:18.912105+0000","last_undegraded":"2023-09-23T15:14:18.912105+0000","last_fullsized":"2023-09-23T15:14:18.912105+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"205'8900","reported_seq":8942,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.678590+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:14:39.678590+0000","last_peered":"2023-09-23T15:14:39.678590+0000","last_clean":"2023-09-23T15:14:39.678590+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:14:39.678590+0000","last_undegraded":"2023-09-23T15:14:39.678590+0000","last_fullsized":"2023-09-23T15:14:39.678590+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8900,"log_dups_size":0,"ondisk_log_size":8900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228065280,"num_objects":3480,"num_object_clones":0,"num_object_copies":10440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8900,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":492,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906758+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:14:18.906758+0000","last_peered":"2023-09-23T15:14:18.906758+0000","last_clean":"2023-09-23T15:14:18.906758+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:14:18.906758+0000","last_undegraded":"2023-09-23T15:14:18.906758+0000","last_fullsized":"2023-09-23T15:14:18.906758+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":493,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909817+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:14:18.909817+0000","last_peered":"2023-09-23T15:14:18.909817+0000","last_clean":"2023-09-23T15:14:18.909817+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:14:18.909817+0000","last_undegraded":"2023-09-23T15:14:18.909817+0000","last_fullsized":"2023-09-23T15:14:18.909817+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.104692+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:14:31.104692+0000","last_peered":"2023-09-23T15:14:31.104692+0000","last_clean":"2023-09-23T15:14:31.104692+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:14:31.104692+0000","last_undegraded":"2023-09-23T15:14:31.104692+0000","last_fullsized":"2023-09-23T15:14:31.104692+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.963749+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:14:18.963749+0000","last_peered":"2023-09-23T15:14:18.963749+0000","last_clean":"2023-09-23T15:14:18.963749+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:14:18.963749+0000","last_undegraded":"2023-09-23T15:14:18.963749+0000","last_fullsized":"2023-09-23T15:14:18.963749+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"205'9099","reported_seq":9143,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:39.488550+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:14:39.488550+0000","last_peered":"2023-09-23T15:14:39.488550+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:14:39.488550+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"177'2700","ondisk_log_start":"177'2700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232325120,"num_objects":3545,"num_object_clones":0,"num_object_copies":10635,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3545,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9099,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1407}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909264+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:14:18.909264+0000","last_peered":"2023-09-23T15:14:18.909264+0000","last_clean":"2023-09-23T15:14:18.909264+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:14:18.909264+0000","last_undegraded":"2023-09-23T15:14:18.909264+0000","last_fullsized":"2023-09-23T15:14:18.909264+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":287,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910575+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:14:18.910575+0000","last_peered":"2023-09-23T15:14:18.910575+0000","last_clean":"2023-09-23T15:14:18.910575+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:14:18.910575+0000","last_undegraded":"2023-09-23T15:14:18.910575+0000","last_fullsized":"2023-09-23T15:14:18.910575+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":149,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.910492+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:14:18.910492+0000","last_peered":"2023-09-23T15:14:18.910492+0000","last_clean":"2023-09-23T15:14:18.910492+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:14:18.910492+0000","last_undegraded":"2023-09-23T15:14:18.910492+0000","last_fullsized":"2023-09-23T15:14:18.910492+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":179,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909062+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:14:18.909062+0000","last_peered":"2023-09-23T15:14:18.909062+0000","last_clean":"2023-09-23T15:14:18.909062+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:14:18.909062+0000","last_undegraded":"2023-09-23T15:14:18.909062+0000","last_fullsized":"2023-09-23T15:14:18.909062+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"205'8915","reported_seq":8957,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:38.532541+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:14:38.532541+0000","last_peered":"2023-09-23T15:14:38.532541+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:14:38.532541+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":8915,"log_dups_size":0,"ondisk_log_size":8915,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235470848,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8915,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-1281}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909266+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:14:18.909266+0000","last_peered":"2023-09-23T15:14:18.909266+0000","last_clean":"2023-09-23T15:14:18.909266+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:14:18.909266+0000","last_undegraded":"2023-09-23T15:14:18.909266+0000","last_fullsized":"2023-09-23T15:14:18.909266+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":810,"reported_epoch":204,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:18.908607+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:14:18.908607+0000","last_peered":"2023-09-23T15:14:18.908607+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:14:18.908607+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":76,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.911740+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:14:18.911740+0000","last_peered":"2023-09-23T15:14:18.911740+0000","last_clean":"2023-09-23T15:14:18.911740+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:14:18.911740+0000","last_undegraded":"2023-09-23T15:14:18.911740+0000","last_fullsized":"2023-09-23T15:14:18.911740+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":154,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909204+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:14:18.909204+0000","last_peered":"2023-09-23T15:14:18.909204+0000","last_clean":"2023-09-23T15:14:18.909204+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:14:18.909204+0000","last_undegraded":"2023-09-23T15:14:18.909204+0000","last_fullsized":"2023-09-23T15:14:18.909204+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"205'8756","reported_seq":8798,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.273440+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:14:39.273440+0000","last_peered":"2023-09-23T15:14:39.273440+0000","last_clean":"2023-09-23T15:14:39.273440+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:14:39.273440+0000","last_undegraded":"2023-09-23T15:14:39.273440+0000","last_fullsized":"2023-09-23T15:14:39.273440+0000","mapping_epoch":171,"log_start":"175'1200","ondisk_log_start":"175'1200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":232390656,"num_objects":3546,"num_object_clones":0,"num_object_copies":10638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8756,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909023+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:14:18.909023+0000","last_peered":"2023-09-23T15:14:18.909023+0000","last_clean":"2023-09-23T15:14:18.909023+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:14:18.909023+0000","last_undegraded":"2023-09-23T15:14:18.909023+0000","last_fullsized":"2023-09-23T15:14:18.909023+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918650+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:14:18.918650+0000","last_peered":"2023-09-23T15:14:18.918650+0000","last_clean":"2023-09-23T15:14:18.918650+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:14:18.918650+0000","last_undegraded":"2023-09-23T15:14:18.918650+0000","last_fullsized":"2023-09-23T15:14:18.918650+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":664,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.912070+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:14:18.912070+0000","last_peered":"2023-09-23T15:14:18.912070+0000","last_clean":"2023-09-23T15:14:18.912070+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:14:18.912070+0000","last_undegraded":"2023-09-23T15:14:18.912070+0000","last_fullsized":"2023-09-23T15:14:18.912070+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":185,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918261+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:14:18.918261+0000","last_peered":"2023-09-23T15:14:18.918261+0000","last_clean":"2023-09-23T15:14:18.918261+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:14:18.918261+0000","last_undegraded":"2023-09-23T15:14:18.918261+0000","last_fullsized":"2023-09-23T15:14:18.918261+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"205'9116","reported_seq":9165,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:39.200989+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:14:39.200989+0000","last_peered":"2023-09-23T15:14:39.200989+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:14:39.200989+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"179'3800","ondisk_log_start":"179'3800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5316,"log_dups_size":0,"ondisk_log_size":5316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230293504,"num_objects":3514,"num_object_clones":0,"num_object_copies":10542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9116,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-1452}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":481,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.908949+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:14:18.908949+0000","last_peered":"2023-09-23T15:14:18.908949+0000","last_clean":"2023-09-23T15:14:18.908949+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:14:18.908949+0000","last_undegraded":"2023-09-23T15:14:18.908949+0000","last_fullsized":"2023-09-23T15:14:18.908949+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":681,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.963916+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:14:18.963916+0000","last_peered":"2023-09-23T15:14:18.963916+0000","last_clean":"2023-09-23T15:14:18.963916+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:14:18.963916+0000","last_undegraded":"2023-09-23T15:14:18.963916+0000","last_fullsized":"2023-09-23T15:14:18.963916+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":167,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909155+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:14:18.909155+0000","last_peered":"2023-09-23T15:14:18.909155+0000","last_clean":"2023-09-23T15:14:18.909155+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:14:18.909155+0000","last_undegraded":"2023-09-23T15:14:18.909155+0000","last_fullsized":"2023-09-23T15:14:18.909155+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":146,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965227+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:14:18.965227+0000","last_peered":"2023-09-23T15:14:18.965227+0000","last_clean":"2023-09-23T15:14:18.965227+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:14:18.965227+0000","last_undegraded":"2023-09-23T15:14:18.965227+0000","last_fullsized":"2023-09-23T15:14:18.965227+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":157,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085854+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:14:31.085854+0000","last_peered":"2023-09-23T15:14:31.085854+0000","last_clean":"2023-09-23T15:14:31.085854+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:14:31.085854+0000","last_undegraded":"2023-09-23T15:14:31.085854+0000","last_fullsized":"2023-09-23T15:14:31.085854+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":204,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.969050+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:14:18.969050+0000","last_peered":"2023-09-23T15:14:18.969050+0000","last_clean":"2023-09-23T15:14:18.969050+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:14:18.969050+0000","last_undegraded":"2023-09-23T15:14:18.969050+0000","last_fullsized":"2023-09-23T15:14:18.969050+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":183,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.907403+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:14:18.907403+0000","last_peered":"2023-09-23T15:14:18.907403+0000","last_clean":"2023-09-23T15:14:18.907403+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:14:18.907403+0000","last_undegraded":"2023-09-23T15:14:18.907403+0000","last_fullsized":"2023-09-23T15:14:18.907403+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":197,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965766+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:14:18.965766+0000","last_peered":"2023-09-23T15:14:18.965766+0000","last_clean":"2023-09-23T15:14:18.965766+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:14:18.965766+0000","last_undegraded":"2023-09-23T15:14:18.965766+0000","last_fullsized":"2023-09-23T15:14:18.965766+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":149,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.906887+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:14:18.906887+0000","last_peered":"2023-09-23T15:14:18.906887+0000","last_clean":"2023-09-23T15:14:18.906887+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:14:18.906887+0000","last_undegraded":"2023-09-23T15:14:18.906887+0000","last_fullsized":"2023-09-23T15:14:18.906887+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":180,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965311+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:14:18.965311+0000","last_peered":"2023-09-23T15:14:18.965311+0000","last_clean":"2023-09-23T15:14:18.965311+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:14:18.965311+0000","last_undegraded":"2023-09-23T15:14:18.965311+0000","last_fullsized":"2023-09-23T15:14:18.965311+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":176,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918606+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:14:18.918606+0000","last_peered":"2023-09-23T15:14:18.918606+0000","last_clean":"2023-09-23T15:14:18.918606+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:14:18.918606+0000","last_undegraded":"2023-09-23T15:14:18.918606+0000","last_fullsized":"2023-09-23T15:14:18.918606+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":190,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.918163+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:14:18.918163+0000","last_peered":"2023-09-23T15:14:18.918163+0000","last_clean":"2023-09-23T15:14:18.918163+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:14:18.918163+0000","last_undegraded":"2023-09-23T15:14:18.918163+0000","last_fullsized":"2023-09-23T15:14:18.918163+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":152,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.909411+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:14:18.909411+0000","last_peered":"2023-09-23T15:14:18.909411+0000","last_clean":"2023-09-23T15:14:18.909411+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:14:18.909411+0000","last_undegraded":"2023-09-23T15:14:18.909411+0000","last_fullsized":"2023-09-23T15:14:18.909411+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":173,"reported_epoch":204,"state":"active+clean","last_fresh":"2023-09-23T15:14:18.965170+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:14:18.965170+0000","last_peered":"2023-09-23T15:14:18.965170+0000","last_clean":"2023-09-23T15:14:18.965170+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:14:18.965170+0000","last_undegraded":"2023-09-23T15:14:18.965170+0000","last_fullsized":"2023-09-23T15:14:18.965170+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3720675352,"num_objects":56774,"num_object_clones":0,"num_object_copies":170322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24249,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":56774,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":144004,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5451,"num_bytes_recovered":357236736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10450186240,"data_stored":10450174024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":116304,"ondisk_log_size":116304,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201519,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463155,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969600,"kb_used_data":1819576,"kb_used_omap":0,"kb_used_meta":150016,"kb_avail":91775552,"statfs":{"total":95995035648,"available":93978165248,"internally_reserved":0,"allocated":1863245824,"data_stored":1862565538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":153616384},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":4.2370000000000001,"15min":2.633},"min":{"1min":0.23699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.926,"5min":95.721000000000004,"15min":108.39400000000001},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.863,"5min":4.4660000000000002,"15min":2.7400000000000002},"min":{"1min":0.23400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.614000000000001,"5min":96.019000000000005,"15min":108.452},"last":1.2849999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":3.371,"5min":3.371,"15min":3.371},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":12.534000000000001,"5min":12.534000000000001,"15min":12.534000000000001},"last":2.899},{"interface":"front","average":{"1min":3.6070000000000002,"5min":3.6070000000000002,"15min":3.6070000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":20.984000000000002,"5min":20.984000000000002,"15min":20.984000000000002},"last":2.2599999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":6.6059999999999999,"15min":4.8700000000000001},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.664000000000001,"5min":139.17500000000001,"15min":180.374},"last":0.77500000000000002},{"interface":"front","average":{"1min":3.3690000000000002,"5min":6.4640000000000004,"15min":4.8700000000000001},"min":{"1min":0.33300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":16.552,"5min":146.999,"15min":181.685},"last":1.923}]},{"osd":3,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":12.955,"5min":4.3739999999999997,"15min":1.992},"min":{"1min":0.441,"5min":0.249,"15min":0.249},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":3.1600000000000001},{"interface":"front","average":{"1min":12.819000000000001,"5min":4.4420000000000002,"15min":1.982},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":80.977999999999994,"5min":80.977999999999994,"15min":80.977999999999994},"last":1.4239999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.8999999999999999,"15min":1.0329999999999999},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5119999999999996,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.998,"15min":1.0980000000000001},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.5760000000000001,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.81100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.8399999999999999,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":5.5880000000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9470000000000001,"15min":1.431},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4550000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.83999999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.381,"15min":1},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":7.5910000000000002,"15min":17},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3939999999999999,"15min":0.97499999999999998},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":3.754,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.53700000000000003}]}]},{"osd":1,"up_from":183,"seq":785979015192,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":817420,"kb_used_data":682756,"kb_used_omap":6,"kb_used_meta":134649,"kb_avail":92927732,"statfs":{"total":95995035648,"available":95157997568,"internally_reserved":0,"allocated":699142144,"data_stored":698164969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":137880651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8039999999999998,"5min":3.8039999999999998,"15min":3.8039999999999998},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.626000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":3.1640000000000001},{"interface":"front","average":{"1min":4.6100000000000003,"5min":4.6100000000000003,"15min":4.6100000000000003},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":16.131,"5min":16.131,"15min":16.131},"last":3.9529999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.1280000000000001,"5min":4.1280000000000001,"15min":4.1280000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":13.843,"5min":13.843,"15min":13.843},"last":2.1269999999999998},{"interface":"front","average":{"1min":4.2229999999999999,"5min":4.2229999999999999,"15min":4.2229999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":16.658000000000001,"5min":16.658000000000001,"15min":16.658000000000001},"last":2.379}]},{"osd":3,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":10.795999999999999,"5min":10.795999999999999,"15min":10.795999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":123.426,"5min":123.426,"15min":123.426},"last":2.3399999999999999},{"interface":"front","average":{"1min":12.566000000000001,"5min":12.566000000000001,"15min":12.566000000000001},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":123.32899999999999,"5min":123.32899999999999,"15min":123.32899999999999},"last":0.33300000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":2.6960000000000002,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":7.8330000000000002,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":1.9710000000000001},{"interface":"front","average":{"1min":3.5579999999999998,"5min":3.5579999999999998,"15min":3.5579999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":16.544,"5min":16.544,"15min":16.544},"last":2.3159999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.7789999999999999,"5min":3.7789999999999999,"15min":3.7789999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":16.166,"5min":16.166,"15min":16.166},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.8969999999999998,"5min":2.8969999999999998,"15min":2.8969999999999998},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.8600000000000003,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":2.0489999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.8100000000000001,"5min":3.8100000000000001,"15min":3.8100000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":16.074000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":3.9159999999999999},{"interface":"front","average":{"1min":2.7490000000000001,"5min":2.7490000000000001,"15min":2.7490000000000001},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":7.7910000000000004,"5min":7.7910000000000004,"15min":7.7910000000000004},"last":2.0150000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.157,"15min":3.157},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":7.8890000000000002,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":2.0910000000000002},{"interface":"front","average":{"1min":3.3540000000000001,"5min":3.3540000000000001,"15min":3.3540000000000001},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":16.594000000000001,"5min":16.594000000000001,"15min":16.594000000000001},"last":2.3570000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893242,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1178336,"kb_used_data":916760,"kb_used_omap":8,"kb_used_meta":261559,"kb_avail":92566816,"statfs":{"total":95995035648,"available":94788419584,"internally_reserved":0,"allocated":938762240,"data_stored":937882303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":267837012},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.2919999999999998,"5min":4.2919999999999998,"15min":4.2919999999999998},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":31.510999999999999,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":4.71,"5min":4.71,"15min":4.71},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":31.378,"5min":31.378,"15min":31.378},"last":0.53700000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":5.359,"15min":2.375},"min":{"1min":0.46000000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":31.271999999999998,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.438},{"interface":"front","average":{"1min":4.9180000000000001,"5min":4.4139999999999997,"15min":1.9850000000000001},"min":{"1min":0.51500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":31.542999999999999,"5min":122.071,"15min":122.071},"last":0.26100000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8159999999999998,"5min":5.5419999999999998,"15min":2.3420000000000001},"min":{"1min":0.35299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.643000000000001,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":4.9729999999999999,"5min":4.6059999999999999,"15min":2.3490000000000002},"min":{"1min":0.40699999999999997,"5min":0.252,"15min":0.252},"max":{"1min":25.632000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":0.625}]},{"osd":4,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":3.0129999999999999,"15min":1.5209999999999999},"min":{"1min":0.51200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":31.341999999999999,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":4.556,"5min":2.9670000000000001,"15min":1.538},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":31.059999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":3.0569999999999999,"15min":1.6180000000000001},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":31.25,"5min":109.369,"15min":109.369},"last":0.503},{"interface":"front","average":{"1min":4.032,"5min":3.532,"15min":1.7350000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":31.295000000000002,"5min":109.387,"15min":109.387},"last":0.68100000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":1.5780000000000001},"min":{"1min":0.41299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.893999999999998,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":4.0350000000000001,"5min":2.9220000000000002,"15min":1.526},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":31.431000000000001,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.55000000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":3.2130000000000001,"15min":1.6259999999999999},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":31.472000000000001,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.58299999999999996},{"interface":"front","average":{"1min":4.5099999999999998,"5min":2.4510000000000001,"15min":1.4139999999999999},"min":{"1min":0.53700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":31.225999999999999,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.71499999999999997}]}]},{"osd":2,"up_from":105,"seq":450971566189,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2380392,"kb_used_data":2075552,"kb_used_omap":11,"kb_used_meta":304820,"kb_avail":91364760,"statfs":{"total":95995035648,"available":93557514240,"internally_reserved":0,"allocated":2125365248,"data_stored":2124481507,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":312136464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":5.8540000000000001,"5min":2.7320000000000002,"15min":1.25},"min":{"1min":0.23000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":38.274000000000001,"5min":38.274000000000001,"15min":38.274000000000001},"last":28.969000000000001},{"interface":"front","average":{"1min":4.6130000000000004,"5min":2.3580000000000001,"15min":1.111},"min":{"1min":0.307,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":33.026000000000003,"5min":33.026000000000003,"15min":33.026000000000003},"last":4.7450000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":5.04,"5min":5.04,"15min":5.04},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":18.239000000000001,"5min":18.239000000000001,"15min":18.239000000000001},"last":5.2400000000000002},{"interface":"front","average":{"1min":2.5489999999999999,"5min":2.5489999999999999,"15min":2.5489999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":7.6669999999999998,"5min":7.6669999999999998,"15min":7.6669999999999998},"last":4.9130000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:14:27 2023","interfaces":[{"interface":"back","average":{"1min":10.016,"5min":3.4990000000000001,"15min":1.748},"min":{"1min":0.44700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":87.108000000000004,"5min":87.108000000000004,"15min":87.108000000000004},"last":42.107999999999997},{"interface":"front","average":{"1min":9.4209999999999994,"5min":3.3980000000000001,"15min":1.754},"min":{"1min":0.59599999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":84.739000000000004,"5min":84.739000000000004,"15min":84.739000000000004},"last":11.954000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.6190000000000002,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.6589999999999998,"5min":52.795999999999999,"15min":52.795999999999999},"last":4.6550000000000002},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.6280000000000001,"15min":1.7090000000000001},"min":{"1min":0.434,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.478999999999999,"5min":68.894999999999996,"15min":68.894999999999996},"last":32.542999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":2.7719999999999998,"15min":1.4790000000000001},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.161000000000001,"5min":41.723999999999997,"15min":41.723999999999997},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.7200000000000002,"5min":2.7469999999999999,"15min":1.4410000000000001},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.694,"5min":39.822000000000003,"15min":39.822000000000003},"last":4.7130000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.9239999999999999,"15min":1.5329999999999999},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5800000000000001,"5min":37.834000000000003,"15min":37.834000000000003},"last":4.6859999999999999},{"interface":"front","average":{"1min":2.4910000000000001,"5min":2.3959999999999999,"15min":1.387},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.019,"5min":37.749000000000002,"15min":37.749000000000002},"last":32.487000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.383,"15min":1.3260000000000001},"min":{"1min":0.42399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.207999999999998,"5min":18.207999999999998,"15min":18.207999999999998},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.2669999999999999,"15min":1.3149999999999999},"min":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6299999999999999,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":4.766}]}]},{"osd":3,"up_from":110,"seq":472446402663,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2813428,"kb_used_data":2522028,"kb_used_omap":3,"kb_used_meta":291388,"kb_avail":90931724,"statfs":{"total":95995035648,"available":93114085376,"internally_reserved":0,"allocated":2582556672,"data_stored":2581715400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":298381481},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":4.0629999999999997,"5min":3.0760000000000001,"15min":1.764},"min":{"1min":0.36499999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":25.111999999999998,"5min":61.744,"15min":79.611000000000004},"last":4.8929999999999998},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.0870000000000002,"15min":1.788},"min":{"1min":0.29599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.6270000000000007,"5min":61.686,"15min":79.656000000000006},"last":0.40200000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:13:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5860000000000003,"5min":4.5860000000000003,"15min":4.5860000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":28.315000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":4.625},{"interface":"front","average":{"1min":4.8460000000000001,"5min":4.8460000000000001,"15min":4.8460000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.427,"5min":14.427,"15min":14.427},"last":3.9159999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":5.4690000000000003,"5min":5.5890000000000004,"15min":3.2749999999999999},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":25.187000000000001,"5min":109.077,"15min":109.077},"last":3.956},{"interface":"front","average":{"1min":4.4580000000000002,"5min":6.0449999999999999,"15min":3.4279999999999999},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.873000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":0.47999999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.9910000000000001,"15min":1.1299999999999999},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4169999999999998,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":0.36199999999999999},{"interface":"front","average":{"1min":2.802,"5min":2.1509999999999998,"15min":1.3320000000000001},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.632999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":4.6719999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.8170000000000002,"15min":1.46},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":6.3869999999999996,"5min":56.960999999999999,"15min":56.960999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.839,"5min":3.0630000000000002,"15min":1.675},"min":{"1min":0.36399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":10.738,"5min":56.981999999999999,"15min":56.981999999999999},"last":4.7690000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.355,"15min":1.3260000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.3419999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.6219999999999999,"15min":1.5780000000000001},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":10.561999999999999,"5min":19.355,"15min":19.355},"last":4.7389999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.726,"15min":1.585},"min":{"1min":0.42599999999999999,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":10.603,"5min":14.885999999999999,"15min":14.885999999999999},"last":4.798},{"interface":"front","average":{"1min":2.702,"5min":2.113,"15min":1.2050000000000001},"min":{"1min":0.437,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.351000000000001,"5min":14.351000000000001,"15min":14.351000000000001},"last":3.9940000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986428,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1335868,"kb_used_data":1149108,"kb_used_omap":0,"kb_used_meta":186752,"kb_avail":92409284,"statfs":{"total":95995035648,"available":94627106816,"internally_reserved":0,"allocated":1176686592,"data_stored":1176009881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":191234048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":4.0780000000000003,"15min":2.2869999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":6.6799999999999997,"5min":108.148,"15min":122.69199999999999},"last":65.988},{"interface":"front","average":{"1min":1.1679999999999999,"5min":3.903,"15min":2.306},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6719999999999997,"5min":108.124,"15min":122.745},"last":65.396000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":4.5490000000000004,"5min":4.5490000000000004,"15min":4.5490000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":53.832000000000001,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":5.077,"5min":5.077,"15min":5.077},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":53.078000000000003,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.70999999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":6.2439999999999998,"15min":2.7719999999999998},"min":{"1min":0.41099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":6.96,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":4.1719999999999997,"5min":6.6150000000000002,"15min":3.0299999999999998},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":34.798999999999999,"5min":147.88900000000001,"15min":147.88900000000001},"last":2.0169999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":4.8849999999999998,"5min":8.2460000000000004,"15min":3.3100000000000001},"min":{"1min":0.32400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":38.685000000000002,"5min":163.97499999999999,"15min":163.97499999999999},"last":0.73999999999999999},{"interface":"front","average":{"1min":5.5129999999999999,"5min":8.0139999999999993,"15min":3.149},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":38.767000000000003,"5min":164.447,"15min":164.447},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:13:38 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":2.3660000000000001,"15min":1.198},"min":{"1min":0.25700000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.998,"5min":28.891999999999999,"15min":28.891999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.56999999999999995,"5min":1.8340000000000001,"15min":1.0449999999999999},"min":{"1min":0.27200000000000002,"5min":0.252,"15min":0.252},"max":{"1min":2.9489999999999998,"5min":32.100999999999999,"15min":32.100999999999999},"last":0.54900000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:14:22 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":3.6800000000000002,"15min":1.7350000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3319999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.69099999999999995,"5min":3.323,"15min":1.6040000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.5179999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.502,"15min":1.1319999999999999},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":1.6419999999999999,"5min":18.23,"15min":18.23},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.48,"15min":1.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":2.214,"5min":14.044,"15min":14.653},"last":0.80600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724791,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1334580,"kb_used_data":1129644,"kb_used_omap":0,"kb_used_meta":204928,"kb_avail":92410572,"statfs":{"total":95995035648,"available":94628425728,"internally_reserved":0,"allocated":1156755456,"data_stored":1156082724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":209846272},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":2.0859999999999999,"15min":1.2190000000000001},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.968,"5min":17.044,"15min":24.962},"last":2.1110000000000002},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.823,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.95,"5min":18.814,"15min":25.317},"last":1.6359999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.673,"15min":1.673},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.2530000000000001,"5min":9.2530000000000001,"15min":9.2530000000000001},"last":1.843},{"interface":"front","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":11.717000000000001,"5min":11.717000000000001,"15min":11.717000000000001},"last":0.436}]},{"osd":2,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":4.7000000000000002,"15min":2.097},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.7859999999999996,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.30099999999999999},{"interface":"front","average":{"1min":4.5129999999999999,"5min":4.4930000000000003,"15min":2.0619999999999998},"min":{"1min":0.33400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":13.332000000000001,"5min":97.305999999999997,"15min":97.305999999999997},"last":2.464}]},{"osd":3,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":5.4550000000000001,"5min":3.7829999999999999,"15min":1.8360000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":32.305999999999997,"5min":53.088999999999999,"15min":53.088999999999999},"last":8.6600000000000001},{"interface":"front","average":{"1min":5.0970000000000004,"5min":3.5249999999999999,"15min":1.734},"min":{"1min":0.46400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":24.052,"5min":53.764000000000003,"15min":53.764000000000003},"last":10.246}]},{"osd":4,"last update":"Sat Sep 23 15:13:43 2023","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":2.298,"15min":1.163},"min":{"1min":0.26900000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.7669999999999999,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.622,"5min":2.5449999999999999,"15min":1.242},"min":{"1min":0.28599999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.1850000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.88200000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":2.0539999999999998,"15min":1.107},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8760000000000003,"5min":15.196,"15min":15.196},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.752,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":1.6419999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.39000000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.8580000000000001,"15min":1.228},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":1.331,"5min":11.353999999999999,"15min":15.301},"last":1.554},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.9119999999999999,"15min":1.212},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.8460000000000001,"5min":15.762,"15min":15.762},"last":0.41499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":926355456,"data_stored":926351384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":690032640,"data_stored":690028568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2117861376,"data_stored":2117861376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2547843072,"data_stored":2547843072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1161232384,"data_stored":1161232384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1150554112,"data_stored":1150550040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1856307200,"data_stored":1856307200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:14:44.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:44.394 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:44.488 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[215646]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.514 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[215646]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.514 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[215646]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.514 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[222878]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.514 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[222878]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.514 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:44 smithi017 ceph-mon[222878]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:44 smithi027 ceph-mon[195233]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:44 smithi027 ceph-mon[195233]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:44 smithi027 ceph-mon[195233]: failed to encode map e206 with expected crc 2023-09-23T15:14:44.634 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:44.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:44.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:44.739 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:44.781 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:44.838 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:44.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:44.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:44.849 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:14:44.849 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:14:44.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:45.032 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:45.057 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:45.083 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:45.128 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:45.163 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:45.163 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:45.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:45.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:45.202 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:45.265 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:45.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:45.304 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:45.314 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:45.363 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:45.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:45.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:45.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.383 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:14:45.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:45.503 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:45 smithi027 ceph-mon[195233]: pgmap v546: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 24252/166941 objects degraded (14.527%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1936026327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:45.622 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:45.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:45.684 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:14:45.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:45 smithi017 ceph-mon[215646]: pgmap v546: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 24252/166941 objects degraded (14.527%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:45.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1936026327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:45 smithi017 ceph-mon[222878]: pgmap v546: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 531 op/s; 24252/166941 objects degraded (14.527%); 1.4 MiB/s, 22 objects/s recovering 2023-09-23T15:14:45.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1936026327' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:45.725 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:45.768 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:45.851 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:45.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:45.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:45.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:45.989 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:46.033 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:46.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:46.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:46.107 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:46.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:46.166 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:46.234 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:46.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:46.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:46.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:46.438 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:46.562 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:46.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:14:46.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:46.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:46.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:46.800 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:47.013 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:47.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:47.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:47.137 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:47.252 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:47.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:14:47.409 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:47.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:47.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:47.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:47.551 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:47.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:47 smithi027 ceph-mon[195233]: pgmap v547: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:47.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24127/162336 objects degraded (14.862%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:47.563 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:47.629 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:47.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:47.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:47.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:47 smithi017 ceph-mon[215646]: pgmap v547: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:47.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24127/162336 objects degraded (14.862%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:47.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:47 smithi017 ceph-mon[222878]: pgmap v547: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:47.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24127/162336 objects degraded (14.862%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:47.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:47.771 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:47.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:47.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:47.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:47.858 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:14:47.859 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:14:47.859 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:14:47.859 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:14:47.859 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:14:47.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:47.987 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:48.052 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:48.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:48.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:48.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:48.191 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:48.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:48] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.43.0" 2023-09-23T15:14:48.248 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:48.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:48.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:48.505 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:48.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:14:48.632 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:48.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:14:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:48.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:48.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:48.789 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:48.867 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:48.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:48.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:48.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:49.024 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:49.076 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:49.093 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:49.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:49.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:49.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:49.315 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:49.470 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:49.478 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:49.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:49.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:49.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:49 smithi027 ceph-mon[195233]: pgmap v548: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:49.636 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:49.689 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:49.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:49 smithi017 ceph-mon[215646]: pgmap v548: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:49 smithi017 ceph-mon[222878]: pgmap v548: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 24127/162336 objects degraded (14.862%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:49.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:49.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:49.781 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:49.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:49.826 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:49.896 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:49.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:49.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:49.999 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:50.027 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:50.230 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:50.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:50.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:50.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:50.481 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:50.643 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:50.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:50.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106167 5.c 5:3b2646bb:::benchmark_data_smithi017_288048_object106166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.01396242599998, 2023-09-23T15:14:50.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26369647800000001, 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106167 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:14:50.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710224+0000", 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710278+0000", 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4126000000000003e-05 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710288+0000", 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4380000000000001e-06 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710293+0000", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9969999999999998e-06 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712898+0000", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026053460000000001 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713002+0000", 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103749 2023-09-23T15:14:50.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713127+0000", 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012499599999999999 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719754+0000", 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066268890000000004 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973752+0000", 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25399865999999999 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973807+0000", 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4224999999999997e-05 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973921+0000", 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114744 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106172 5.c 5:3a30f215:::benchmark_data_smithi017_288048_object106171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.01121755200001, 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26115257600000003, 2023-09-23T15:14:50.734 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106172 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712969+0000", 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713027+0000", 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7998999999999998e-05 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713038+0000", 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1790999999999999e-05 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713046+0000", 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.323e-06 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716305+0000", 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032597160000000002 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716430+0000", 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124528 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716566+0000", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135735 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723883+0000", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073173500000000002 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973975+0000", 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.250091695 2023-09-23T15:14:50.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974026+0000", 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1768000000000003e-05 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974122+0000", 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5847999999999994e-05 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.740 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106171 5.a e134/114 5:591ee17c:::benchmark_data_smithi017_288048_object106170:head v 134'6617, mlcod=134'6615)", 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.00464271999999, 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.264097527, 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719544+0000", 2023-09-23T15:14:50.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719581+0000", 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6968999999999998e-05 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719591+0000", 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9680000000000001e-06 2023-09-23T15:14:50.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719596+0000", 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4990000000000002e-06 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719639+0000", 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2762000000000002e-05 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719657+0000", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8224999999999999e-05 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983596+0000", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26393868399999998 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983642+0000", 2023-09-23T15:14:50.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6028e-05 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106177 5.a e134/114 5:561ec9dc:::benchmark_data_smithi017_288048_object106176:head v 134'6618, mlcod=134'6615)", 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 350.00027578300001, 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25975826499999999, 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.746 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723910+0000", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723946+0000", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6072000000000003e-05 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723957+0000", 2023-09-23T15:14:50.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0461e-05 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723963+0000", 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9569999999999999e-06 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725450+0000", 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001486769 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725471+0000", 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1826000000000001e-05 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983647+0000", 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25817548600000001 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983670+0000", 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2768e-05 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106178 5.a e134/114 5:56b4f3e4:::benchmark_data_smithi017_288048_object106177:head v 134'6619, mlcod=134'6615)", 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 349.99894617299998, 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25845162399999999, 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.751 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725276+0000", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5089000000000002e-05 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725287+0000", 2023-09-23T15:14:50.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1552999999999999e-05 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725293+0000", 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8449999999999999e-06 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977314+0000", 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.252021461 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977334+0000", 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9160999999999998e-05 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983673+0000", 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006339028 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983693+0000", 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0047e-05 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166941 5.1 5:8b30db7e:::benchmark_data_smithi017_288048_object55172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.63783507199997, 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40214831299999998, 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.756 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166941 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086351+0000", 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086356+0000", 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1039999999999998e-06 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086368+0000", 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1618e-05 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086374+0000", 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5490000000000003e-06 2023-09-23T15:14:50.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086536+0000", 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161743 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086652+0000", 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011594 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086770+0000", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011798600000000001 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091315+0000", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045446310000000004 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488382+0000", 2023-09-23T15:14:50.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39706780000000003 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488406+0000", 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3218e-05 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488454+0000", 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8301999999999999e-05 2023-09-23T15:14:50.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488501+0000", 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6573000000000003e-05 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166942 5.e 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.63780073800001, 2023-09-23T15:14:50.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40238844899999998, 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166942 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:14:50.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086386+0000", 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086389+0000", 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5830000000000001e-06 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086395+0000", 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086399+0000", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2300000000000002e-06 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086603+0000", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020372899999999999 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086701+0000", 2023-09-23T15:14:50.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8332000000000007e-05 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086804+0000", 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103294 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093768+0000", 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006964034 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488259+0000", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39449091200000003 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488702+0000", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00044246200000000002 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488734+0000", 2023-09-23T15:14:50.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2069000000000001e-05 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488775+0000", 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1278e-05 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166947 5.e 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:14:50.770 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.63733707599999, 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40205635099999998, 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166947 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086849+0000", 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086854+0000", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6290000000000001e-06 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086866+0000", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2478e-05 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086873+0000", 2023-09-23T15:14:50.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1299999999999998e-06 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090167+0000", 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032944319999999999 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090246+0000", 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9369999999999994e-05 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090328+0000", 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1706999999999995e-05 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488279+0000", 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397950631 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488321+0000", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2725999999999997e-05 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488813+0000", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000491991 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488869+0000", 2023-09-23T15:14:50.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5350999999999999e-05 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488907+0000", 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7762999999999997e-05 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166944 5.0 e147/132 5:0733dbd2:::benchmark_data_smithi017_288048_object55175:head v 147'10299, mlcod=147'10297)", 2023-09-23T15:14:50.777 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.63710667800001, 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40144166599999997, 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087080+0000", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087086+0000", 2023-09-23T15:14:50.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5890000000000002e-06 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087096+0000", 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0434999999999999e-05 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087101+0000", 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4890000000000003e-06 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087141+0000", 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9147999999999999e-05 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087162+0000", 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0928e-05 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099400+0000", 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012238123 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488522+0000", 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38912261599999998 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.782 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166953 5.c 5:3faa0b3c:::benchmark_data_smithi017_288048_object55184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.63306030400003, 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39750094200000002, 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166953 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091126+0000", 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091130+0000", 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.014e-06 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091144+0000", 2023-09-23T15:14:50.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4717e-05 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091149+0000", 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9590000000000003e-06 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091258+0000", 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108806 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091386+0000", 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012736 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091496+0000", 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109987 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488306+0000", 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39681003300000001 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488328+0000", 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2175999999999999e-05 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488380+0000", 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1913999999999998e-05 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488628+0000", 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024825900000000002 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166945 5.0 e147/132 5:0c8902eb:::benchmark_data_smithi017_288048_object55176:head v 147'10300, mlcod=147'10297)", 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.632003881, 2023-09-23T15:14:50.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39637069699999999, 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092188+0000", 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9760000000000003e-06 2023-09-23T15:14:50.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092197+0000", 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2019999999999993e-06 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092203+0000", 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8019999999999997e-06 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092271+0000", 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8647999999999997e-05 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092292+0000", 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0990999999999999e-05 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488527+0000", 2023-09-23T15:14:50.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39623427999999999 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488554+0000", 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7458e-05 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166939 5.f e147/114 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head v 147'10605, mlcod=147'10602)", 2023-09-23T15:14:50.794 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.62783210499998, 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40201396099999998, 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096354+0000", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096363+0000", 2023-09-23T15:14:50.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2139999999999996e-06 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096374+0000", 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1022000000000001e-05 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096379+0000", 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488409+0000", 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39202973600000002 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488494+0000", 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5271000000000004e-05 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498336+0000", 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098418459999999996 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498369+0000", 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2783000000000001e-05 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.799 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166950 5.f e147/114 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head v 147'10606, mlcod=147'10602)", 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.62778431100003, 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40582205900000001, 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096406+0000", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4649999999999999e-06 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096412+0000", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7450000000000001e-06 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096415+0000", 2023-09-23T15:14:50.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5650000000000002e-06 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493677+0000", 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397262008 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493697+0000", 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9083e-05 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502182+0000", 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084849629999999999 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502225+0000", 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.35e-05 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166955 5.5 e147/132 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head v 147'10623, mlcod=147'10621)", 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.627747097, 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.410760976, 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:14:50.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096443+0000", 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1700000000000001e-06 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096449+0000", 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2023-09-23T15:14:50.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096453+0000", 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4520000000000002e-06 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488252+0000", 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39179915599999998 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488294+0000", 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2344e-05 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507165+0000", 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018870505999999999 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507201+0000", 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6368999999999997e-05 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166947 5.e e147/114 ondisk, result = 0)", 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.62772139399999, 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39185894700000001, 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.810 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096468+0000", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.357e-06 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096473+0000", 2023-09-23T15:14:50.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2220000000000001e-06 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096477+0000", 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.399e-06 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488307+0000", 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39183000600000001 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488315+0000", 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488325+0000", 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0294e-05 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166953 5.c e147/132 ondisk, result = 0)", 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.62770333200001, 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39182831099999998, 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:14:50.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096485+0000", 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6139999999999999e-06 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096490+0000", 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8550000000000001e-06 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096493+0000", 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9909999999999998e-06 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488288+0000", 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.391795069 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488298+0000", 2023-09-23T15:14:50.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4709999999999996e-06 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488312+0000", 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4516e-05 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166941 5.1 e147/114 ondisk, result = 0)", 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:14:50.819 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 272.62381066, 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38801138200000002, 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:14:50.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100376+0000", 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100382+0000", 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0780000000000001e-06 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488291+0000", 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38790906000000003 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488299+0000", 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8660000000000006e-06 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488367+0000", 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8133000000000006e-05 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488376+0000", 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.935e-06 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488388+0000", 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2139e-05 2023-09-23T15:14:50.823 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78602 6.e 6:7eeb81e7:::benchmark_data_smithi017_350392_object78601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 129.17558747699999, 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6491542219999999, 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:14:50.824 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78602 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548599+0000", 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548629+0000", 2023-09-23T15:14:50.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9839999999999999e-05 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548639+0000", 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6940000000000007e-06 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548644+0000", 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8779999999999997e-06 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548691+0000", 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7481000000000002e-05 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548837+0000", 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014598700000000001 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:14:50.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548993+0000", 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015569999999999999 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565316+0000", 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016322705 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565327+0000", 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0994e-05 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847765+0000", 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28243886899999998 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847776+0000", 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0519999999999999e-05 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197689+0000", 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3499132 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197754+0000", 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.831 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78611 6.e 6:7d1942e2:::benchmark_data_smithi017_350392_object78610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 129.16662814399999, 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733330400000002, 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78611 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.832 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557557+0000", 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557602+0000", 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4189e-05 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557615+0000", 2023-09-23T15:14:50.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3869999999999999e-05 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557627+0000", 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1229999999999999e-05 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565352+0000", 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077249190000000002 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565359+0000", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0609999999999998e-06 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847801+0000", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28244210600000003 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847808+0000", 2023-09-23T15:14:50.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5240000000000003e-06 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201122+0000", 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.353314205 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201310+0000", 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018789999999999999 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201482+0000", 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017193500000000001 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.204567+0000", 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030849520000000002 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.210867+0000", 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062997979999999997 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228500+0000", 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017632618999999999 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228573+0000", 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3799000000000003e-05 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.230892+0000", 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023187149999999998 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:147647 6.2 6:4814bf33:::benchmark_data_smithi017_350392_object47256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4.7852668359999999, 2023-09-23T15:14:50.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25844827100000001, 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 147647 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:14:50.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938919+0000", 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:14:50.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938928+0000", 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938946+0000", 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8111e-05 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:50.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938954+0000", 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1780000000000003e-06 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938996+0000", 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2521999999999997e-05 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939088+0000", 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1113000000000007e-05 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939182+0000", 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4487000000000005e-05 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.942153+0000", 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029708239999999999 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.143408+0000", 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20125490800000001 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197272+0000", 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053864243999999999 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197325+0000", 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2802e-05 2023-09-23T15:14:50.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197369+0000", 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3869000000000003e-05 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:50.847 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:14:50.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:50.945 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:51.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:14:51.539 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:14:51.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[215646]: pgmap v549: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 24009/154104 objects degraded (15.580%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:51.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/776634063' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.797 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:51.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:51.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:51.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[222878]: pgmap v549: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 24009/154104 objects degraded (15.580%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:51.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/776634063' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:51 smithi027 ceph-mon[195233]: pgmap v549: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 24009/154104 objects degraded (15.580%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/776634063' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:14:51.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:51.945 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:51.957 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:52.027 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:52.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:52.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:52.169 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:52.242 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:52.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:52.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:52.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:52.379 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:52.446 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:52.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:52.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:52.553 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:52.553 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.553 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:52.554 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T15:14:52.554 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.554 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24009/154104 objects degraded (15.580%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:52.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:52.590 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:52.647 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:52.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:52.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:52.752 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24009/154104 objects degraded (15.580%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:52.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:14:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:14:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: osdmap e207: 8 total, 8 up, 7 in 2023-09-23T15:14:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: failed to encode map e207 with expected crc 2023-09-23T15:14:52.829 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24009/154104 objects degraded (15.580%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:52.839 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:53.247 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:53.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:53.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:53.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:53.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:53.448 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:53.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:53.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:53.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:53.592 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:53.661 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:53.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:53.674 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:14:53.674 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:14:53.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:151880 6.d 6:b8e5e477:::benchmark_data_smithi017_350392_object51489:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:53.660566+0000", 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.014066453, 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014090432, 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 151880 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:53.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660566+0000", 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660566+0000", 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:53.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660564+0000", 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660571+0000", 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9979999999999997e-06 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660586+0000", 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5101e-05 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660592+0000", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1109999999999996e-06 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660639+0000", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6647999999999998e-05 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660777+0000", 2023-09-23T15:14:53.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138431 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.660912+0000", 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013443600000000001 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.666081+0000", 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051687449999999998 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.668146+0000", 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002065009 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:151869 6.d 6:b40107d3:::benchmark_data_smithi017_350392_object51478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e207)", 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:53.649557+0000", 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.025074841, 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025187620000000001, 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:14:53.680 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 151869 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649557+0000", 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649557+0000", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649555+0000", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649565+0000", 2023-09-23T15:14:53.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6350000000000001e-06 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649588+0000", 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3155000000000001e-05 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649596+0000", 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3459999999999994e-06 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649689+0000", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2404999999999994e-05 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649851+0000", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016249999999999999 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.649968+0000", 2023-09-23T15:14:53.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011645 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.655040+0000", 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050720130000000002 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:53.658264+0000", 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032242130000000001 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:14:53.685 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:53.686 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:14:53.686 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:14:53.686 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 2 2023-09-23T15:14:53.686 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:14:53.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:53.804 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:53.908 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:53.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:53.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:53.940 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:53 smithi027 ceph-mon[195233]: pgmap v551: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 24009/154104 objects degraded (15.580%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:54.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:14:54.132 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:53 smithi017 ceph-mon[215646]: pgmap v551: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 24009/154104 objects degraded (15.580%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:54.132 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:53 smithi017 ceph-mon[222878]: pgmap v551: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 575 op/s; 24009/154104 objects degraded (15.580%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:54.147 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:54.215 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:14:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:14:54.290 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:14:54.308 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:14:54.323 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:14:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:54.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:14:54.362 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:14:54.423 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:14:54.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:54.464 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:14:54.469 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:54.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:14:54.520 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:14:54.531 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:54.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:54.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:54.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:14:54.617 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:14:54.632 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:14:54.634 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.635 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:14:54.639 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:54.721 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:54.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:54.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:54.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:54.982 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:55.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:55.078 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:55.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:55.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:55.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:55.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1500256597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:55.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1500256597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:55.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1500256597' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:14:55.270 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:55.845 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:55.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:55.848 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:55.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:55.977 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:56.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:55 smithi017 ceph-mon[222878]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:56.107 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:55 smithi017 ceph-mon[222878]: pgmap v552: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 24009/150531 objects degraded (15.950%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:56.109 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:55 smithi017 ceph-mon[215646]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:56.109 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:55 smithi017 ceph-mon[215646]: pgmap v552: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 24009/150531 objects degraded (15.950%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:56.137 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":551,"stamp":"2023-09-23T15:14:52.376347+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3369375964,"num_objects":51368,"num_object_clones":0,"num_object_copies":154104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24009,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51368,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":150465,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5753,"num_bytes_recovered":378509479,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122450,"ondisk_log_size":122450,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10839244,"kb_used_data":9321556,"kb_used_omap":30,"kb_used_meta":1517601,"kb_avail":645376820,"statfs":{"total":671965249536,"available":660865863680,"internally_reserved":0,"allocated":9545273344,"data_stored":9539633385,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1554024024},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,4,2,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-356057088,"num_objects":-5433,"num_object_clones":0,"num_object_copies":-16299,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-243,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5433,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5433,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":243,"num_bytes_recovered":15925248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.437611"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":200,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.092434+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:14:31.092434+0000","last_peered":"2023-09-23T15:14:31.092434+0000","last_clean":"2023-09-23T15:14:31.092434+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:14:31.092434+0000","last_undegraded":"2023-09-23T15:14:31.092434+0000","last_fullsized":"2023-09-23T15:14:31.092434+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":134,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.108635+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:14:31.108635+0000","last_peered":"2023-09-23T15:14:31.108635+0000","last_clean":"2023-09-23T15:14:31.108635+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:14:31.108635+0000","last_undegraded":"2023-09-23T15:14:31.108635+0000","last_fullsized":"2023-09-23T15:14:31.108635+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":82,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.094710+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:14:31.094710+0000","last_peered":"2023-09-23T15:14:31.094710+0000","last_clean":"2023-09-23T15:14:31.094710+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:14:31.094710+0000","last_undegraded":"2023-09-23T15:14:31.094710+0000","last_fullsized":"2023-09-23T15:14:31.094710+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":171,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.419368+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:14:39.419368+0000","last_peered":"2023-09-23T15:14:39.419368+0000","last_clean":"2023-09-23T15:14:39.419368+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:14:39.419368+0000","last_undegraded":"2023-09-23T15:14:39.419368+0000","last_fullsized":"2023-09-23T15:14:39.419368+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":203,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.095710+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:14:31.095710+0000","last_peered":"2023-09-23T15:14:31.095710+0000","last_clean":"2023-09-23T15:14:31.095710+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:14:31.095710+0000","last_undegraded":"2023-09-23T15:14:31.095710+0000","last_fullsized":"2023-09-23T15:14:31.095710+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.080334+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:14:31.080334+0000","last_peered":"2023-09-23T15:14:31.080334+0000","last_clean":"2023-09-23T15:14:31.080334+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:14:31.080334+0000","last_undegraded":"2023-09-23T15:14:31.080334+0000","last_fullsized":"2023-09-23T15:14:31.080334+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":150,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087445+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:14:31.087445+0000","last_peered":"2023-09-23T15:14:31.087445+0000","last_clean":"2023-09-23T15:14:31.087445+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:14:31.087445+0000","last_undegraded":"2023-09-23T15:14:31.087445+0000","last_fullsized":"2023-09-23T15:14:31.087445+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":185,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.095777+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:14:31.095777+0000","last_peered":"2023-09-23T15:14:31.095777+0000","last_clean":"2023-09-23T15:14:31.095777+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:14:31.095777+0000","last_undegraded":"2023-09-23T15:14:31.095777+0000","last_fullsized":"2023-09-23T15:14:31.095777+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086268+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:14:31.086268+0000","last_peered":"2023-09-23T15:14:31.086268+0000","last_clean":"2023-09-23T15:14:31.086268+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:14:31.086268+0000","last_undegraded":"2023-09-23T15:14:31.086268+0000","last_fullsized":"2023-09-23T15:14:31.086268+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":196,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088762+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:14:31.088762+0000","last_peered":"2023-09-23T15:14:31.088762+0000","last_clean":"2023-09-23T15:14:31.088762+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:14:31.088762+0000","last_undegraded":"2023-09-23T15:14:31.088762+0000","last_fullsized":"2023-09-23T15:14:31.088762+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":128,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.100395+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:14:31.100395+0000","last_peered":"2023-09-23T15:14:31.100395+0000","last_clean":"2023-09-23T15:14:31.100395+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:14:31.100395+0000","last_undegraded":"2023-09-23T15:14:31.100395+0000","last_fullsized":"2023-09-23T15:14:31.100395+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":68,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.108855+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:14:31.108855+0000","last_peered":"2023-09-23T15:14:31.108855+0000","last_clean":"2023-09-23T15:14:31.108855+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:14:31.108855+0000","last_undegraded":"2023-09-23T15:14:31.108855+0000","last_fullsized":"2023-09-23T15:14:31.108855+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":77,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.092107+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:14:31.092107+0000","last_peered":"2023-09-23T15:14:31.092107+0000","last_clean":"2023-09-23T15:14:31.092107+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:14:31.092107+0000","last_undegraded":"2023-09-23T15:14:31.092107+0000","last_fullsized":"2023-09-23T15:14:31.092107+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":190,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085600+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:14:31.085600+0000","last_peered":"2023-09-23T15:14:31.085600+0000","last_clean":"2023-09-23T15:14:31.085600+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:14:31.085600+0000","last_undegraded":"2023-09-23T15:14:31.085600+0000","last_fullsized":"2023-09-23T15:14:31.085600+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085372+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:14:31.085372+0000","last_peered":"2023-09-23T15:14:31.085372+0000","last_clean":"2023-09-23T15:14:31.085372+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:14:31.085372+0000","last_undegraded":"2023-09-23T15:14:31.085372+0000","last_fullsized":"2023-09-23T15:14:31.085372+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087650+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:14:31.087650+0000","last_peered":"2023-09-23T15:14:31.087650+0000","last_clean":"2023-09-23T15:14:31.087650+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:14:31.087650+0000","last_undegraded":"2023-09-23T15:14:31.087650+0000","last_fullsized":"2023-09-23T15:14:31.087650+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":68,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085428+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:14:31.085428+0000","last_peered":"2023-09-23T15:14:31.085428+0000","last_clean":"2023-09-23T15:14:31.085428+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:14:31.085428+0000","last_undegraded":"2023-09-23T15:14:31.085428+0000","last_fullsized":"2023-09-23T15:14:31.085428+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":196,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.117943+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:14:31.117943+0000","last_peered":"2023-09-23T15:14:31.117943+0000","last_clean":"2023-09-23T15:14:31.117943+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:14:31.117943+0000","last_undegraded":"2023-09-23T15:14:31.117943+0000","last_fullsized":"2023-09-23T15:14:31.117943+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085702+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:14:31.085702+0000","last_peered":"2023-09-23T15:14:31.085702+0000","last_clean":"2023-09-23T15:14:31.085702+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:14:31.085702+0000","last_undegraded":"2023-09-23T15:14:31.085702+0000","last_fullsized":"2023-09-23T15:14:31.085702+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":170,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086920+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:14:31.086920+0000","last_peered":"2023-09-23T15:14:31.086920+0000","last_clean":"2023-09-23T15:14:31.086920+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:14:31.086920+0000","last_undegraded":"2023-09-23T15:14:31.086920+0000","last_fullsized":"2023-09-23T15:14:31.086920+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":174,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088776+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:14:31.088776+0000","last_peered":"2023-09-23T15:14:31.088776+0000","last_clean":"2023-09-23T15:14:31.088776+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:14:31.088776+0000","last_undegraded":"2023-09-23T15:14:31.088776+0000","last_fullsized":"2023-09-23T15:14:31.088776+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085436+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:14:31.085436+0000","last_peered":"2023-09-23T15:14:31.085436+0000","last_clean":"2023-09-23T15:14:31.085436+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:14:31.085436+0000","last_undegraded":"2023-09-23T15:14:31.085436+0000","last_fullsized":"2023-09-23T15:14:31.085436+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":52,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085314+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:14:31.085314+0000","last_peered":"2023-09-23T15:14:31.085314+0000","last_clean":"2023-09-23T15:14:31.085314+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:14:31.085314+0000","last_undegraded":"2023-09-23T15:14:31.085314+0000","last_fullsized":"2023-09-23T15:14:31.085314+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":159,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086767+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:14:31.086767+0000","last_peered":"2023-09-23T15:14:31.086767+0000","last_clean":"2023-09-23T15:14:31.086767+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:14:31.086767+0000","last_undegraded":"2023-09-23T15:14:31.086767+0000","last_fullsized":"2023-09-23T15:14:31.086767+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087261+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:14:31.087261+0000","last_peered":"2023-09-23T15:14:31.087261+0000","last_clean":"2023-09-23T15:14:31.087261+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:14:31.087261+0000","last_undegraded":"2023-09-23T15:14:31.087261+0000","last_fullsized":"2023-09-23T15:14:31.087261+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":172,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.079025+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:14:31.079025+0000","last_peered":"2023-09-23T15:14:31.079025+0000","last_clean":"2023-09-23T15:14:31.079025+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:14:31.079025+0000","last_undegraded":"2023-09-23T15:14:31.079025+0000","last_fullsized":"2023-09-23T15:14:31.079025+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":196,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.109459+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:14:31.109459+0000","last_peered":"2023-09-23T15:14:31.109459+0000","last_clean":"2023-09-23T15:14:31.109459+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:14:31.109459+0000","last_undegraded":"2023-09-23T15:14:31.109459+0000","last_fullsized":"2023-09-23T15:14:31.109459+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"206'9268","reported_seq":9311,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.679392+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:14:49.679392+0000","last_peered":"2023-09-23T15:14:49.679392+0000","last_clean":"2023-09-23T15:14:49.679392+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:14:49.679392+0000","last_undegraded":"2023-09-23T15:14:49.679392+0000","last_fullsized":"2023-09-23T15:14:49.679392+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9268,"log_dups_size":0,"ondisk_log_size":9268,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":204865536,"num_objects":3126,"num_object_clones":0,"num_object_copies":9378,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3126,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9268,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086431+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:14:31.086431+0000","last_peered":"2023-09-23T15:14:31.086431+0000","last_clean":"2023-09-23T15:14:31.086431+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:14:31.086431+0000","last_undegraded":"2023-09-23T15:14:31.086431+0000","last_fullsized":"2023-09-23T15:14:31.086431+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":80,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086184+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:14:31.086184+0000","last_peered":"2023-09-23T15:14:31.086184+0000","last_clean":"2023-09-23T15:14:31.086184+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:14:31.086184+0000","last_undegraded":"2023-09-23T15:14:31.086184+0000","last_fullsized":"2023-09-23T15:14:31.086184+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":157,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088266+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:14:31.088266+0000","last_peered":"2023-09-23T15:14:31.088266+0000","last_clean":"2023-09-23T15:14:31.088266+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:14:31.088266+0000","last_undegraded":"2023-09-23T15:14:31.088266+0000","last_fullsized":"2023-09-23T15:14:31.088266+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"206'9452","reported_seq":11092,"reported_epoch":206,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:49.556597+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:14:49.556597+0000","last_peered":"2023-09-23T15:14:49.556597+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:14:49.556597+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"177'3000","ondisk_log_start":"177'3000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6452,"log_dups_size":0,"ondisk_log_size":6452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210632704,"num_objects":3214,"num_object_clones":0,"num_object_copies":9642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9452,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":794,"num_bytes_recovered":52035584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-990}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":473,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:31.086602+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:14:31.086602+0000","last_peered":"2023-09-23T15:14:31.086602+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:14:31.086602+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":80,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086730+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:14:31.086730+0000","last_peered":"2023-09-23T15:14:31.086730+0000","last_clean":"2023-09-23T15:14:31.086730+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:14:31.086730+0000","last_undegraded":"2023-09-23T15:14:31.086730+0000","last_fullsized":"2023-09-23T15:14:31.086730+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":198,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085164+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:14:31.085164+0000","last_peered":"2023-09-23T15:14:31.085164+0000","last_clean":"2023-09-23T15:14:31.085164+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:14:31.085164+0000","last_undegraded":"2023-09-23T15:14:31.085164+0000","last_fullsized":"2023-09-23T15:14:31.085164+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"206'9192","reported_seq":9235,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:48.542721+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:14:48.542721+0000","last_peered":"2023-09-23T15:14:48.542721+0000","last_clean":"2023-09-23T15:14:48.542721+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:14:48.542721+0000","last_undegraded":"2023-09-23T15:14:48.542721+0000","last_fullsized":"2023-09-23T15:14:48.542721+0000","mapping_epoch":171,"log_start":"174'100","ondisk_log_start":"174'100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9092,"log_dups_size":0,"ondisk_log_size":9092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212992000,"num_objects":3250,"num_object_clones":0,"num_object_copies":9750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9192,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087139+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:14:31.087139+0000","last_peered":"2023-09-23T15:14:31.087139+0000","last_clean":"2023-09-23T15:14:31.087139+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:14:31.087139+0000","last_undegraded":"2023-09-23T15:14:31.087139+0000","last_fullsized":"2023-09-23T15:14:31.087139+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":66,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088695+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:14:31.088695+0000","last_peered":"2023-09-23T15:14:31.088695+0000","last_clean":"2023-09-23T15:14:31.088695+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:14:31.088695+0000","last_undegraded":"2023-09-23T15:14:31.088695+0000","last_fullsized":"2023-09-23T15:14:31.088695+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":194,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.079377+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:14:31.079377+0000","last_peered":"2023-09-23T15:14:31.079377+0000","last_clean":"2023-09-23T15:14:31.079377+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:14:31.079377+0000","last_undegraded":"2023-09-23T15:14:31.079377+0000","last_fullsized":"2023-09-23T15:14:31.079377+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"206'9389","reported_seq":9434,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.334203+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:14:49.334203+0000","last_peered":"2023-09-23T15:14:49.334203+0000","last_clean":"2023-09-23T15:14:49.334203+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:14:49.334203+0000","last_undegraded":"2023-09-23T15:14:49.334203+0000","last_fullsized":"2023-09-23T15:14:49.334203+0000","mapping_epoch":171,"log_start":"179'3800","ondisk_log_start":"179'3800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5589,"log_dups_size":0,"ondisk_log_size":5589,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214630400,"num_objects":3275,"num_object_clones":0,"num_object_copies":9825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9389,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086445+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:14:31.086445+0000","last_peered":"2023-09-23T15:14:31.086445+0000","last_clean":"2023-09-23T15:14:31.086445+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:14:31.086445+0000","last_undegraded":"2023-09-23T15:14:31.086445+0000","last_fullsized":"2023-09-23T15:14:31.086445+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":165,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.090466+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:31.090466+0000","last_peered":"2023-09-23T15:14:31.090466+0000","last_clean":"2023-09-23T15:14:31.090466+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:14:31.090466+0000","last_undegraded":"2023-09-23T15:14:31.090466+0000","last_fullsized":"2023-09-23T15:14:31.090466+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":165,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085705+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:14:31.085705+0000","last_peered":"2023-09-23T15:14:31.085705+0000","last_clean":"2023-09-23T15:14:31.085705+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:14:31.085705+0000","last_undegraded":"2023-09-23T15:14:31.085705+0000","last_fullsized":"2023-09-23T15:14:31.085705+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"206'9580","reported_seq":9629,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:49.307532+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:14:49.307532+0000","last_peered":"2023-09-23T15:14:49.307532+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:14:49.307532+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"175'2000","ondisk_log_start":"175'2000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7580,"log_dups_size":0,"ondisk_log_size":7580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218365952,"num_objects":3332,"num_object_clones":0,"num_object_copies":9996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9580,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-1678}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087469+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:14:31.087469+0000","last_peered":"2023-09-23T15:14:31.087469+0000","last_clean":"2023-09-23T15:14:31.087469+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:14:31.087469+0000","last_undegraded":"2023-09-23T15:14:31.087469+0000","last_fullsized":"2023-09-23T15:14:31.087469+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":168,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.104031+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:14:31.104031+0000","last_peered":"2023-09-23T15:14:31.104031+0000","last_clean":"2023-09-23T15:14:31.104031+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:14:31.104031+0000","last_undegraded":"2023-09-23T15:14:31.104031+0000","last_fullsized":"2023-09-23T15:14:31.104031+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":164,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085670+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:14:31.085670+0000","last_peered":"2023-09-23T15:14:31.085670+0000","last_clean":"2023-09-23T15:14:31.085670+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:14:31.085670+0000","last_undegraded":"2023-09-23T15:14:31.085670+0000","last_fullsized":"2023-09-23T15:14:31.085670+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"206'9358","reported_seq":9403,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:48.434688+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:14:48.434688+0000","last_peered":"2023-09-23T15:14:48.434688+0000","last_clean":"2023-09-23T15:14:48.434688+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:14:48.434688+0000","last_undegraded":"2023-09-23T15:14:48.434688+0000","last_fullsized":"2023-09-23T15:14:48.434688+0000","mapping_epoch":171,"log_start":"175'200","ondisk_log_start":"175'200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9158,"log_dups_size":0,"ondisk_log_size":9158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211025920,"num_objects":3220,"num_object_clones":0,"num_object_copies":9660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9358,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":499,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085773+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:14:31.085773+0000","last_peered":"2023-09-23T15:14:31.085773+0000","last_clean":"2023-09-23T15:14:31.085773+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:14:31.085773+0000","last_undegraded":"2023-09-23T15:14:31.085773+0000","last_fullsized":"2023-09-23T15:14:31.085773+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.079327+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:14:31.079327+0000","last_peered":"2023-09-23T15:14:31.079327+0000","last_clean":"2023-09-23T15:14:31.079327+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:14:31.079327+0000","last_undegraded":"2023-09-23T15:14:31.079327+0000","last_fullsized":"2023-09-23T15:14:31.079327+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":198,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085258+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:14:31.085258+0000","last_peered":"2023-09-23T15:14:31.085258+0000","last_clean":"2023-09-23T15:14:31.085258+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:14:31.085258+0000","last_undegraded":"2023-09-23T15:14:31.085258+0000","last_fullsized":"2023-09-23T15:14:31.085258+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"206'9376","reported_seq":9421,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.553578+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:14:49.553578+0000","last_peered":"2023-09-23T15:14:49.553578+0000","last_clean":"2023-09-23T15:14:49.553578+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:14:49.553578+0000","last_undegraded":"2023-09-23T15:14:49.553578+0000","last_fullsized":"2023-09-23T15:14:49.553578+0000","mapping_epoch":171,"log_start":"177'2900","ondisk_log_start":"177'2900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209846272,"num_objects":3202,"num_object_clones":0,"num_object_copies":9606,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3202,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9376,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085529+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:14:31.085529+0000","last_peered":"2023-09-23T15:14:31.085529+0000","last_clean":"2023-09-23T15:14:31.085529+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:14:31.085529+0000","last_undegraded":"2023-09-23T15:14:31.085529+0000","last_fullsized":"2023-09-23T15:14:31.085529+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":189,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085067+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:14:31.085067+0000","last_peered":"2023-09-23T15:14:31.085067+0000","last_clean":"2023-09-23T15:14:31.085067+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:14:31.085067+0000","last_undegraded":"2023-09-23T15:14:31.085067+0000","last_fullsized":"2023-09-23T15:14:31.085067+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086772+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:14:31.086772+0000","last_peered":"2023-09-23T15:14:31.086772+0000","last_clean":"2023-09-23T15:14:31.086772+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:14:31.086772+0000","last_undegraded":"2023-09-23T15:14:31.086772+0000","last_fullsized":"2023-09-23T15:14:31.086772+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"206'9353","reported_seq":19217,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.630697+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:14:49.630697+0000","last_peered":"2023-09-23T15:14:49.630697+0000","last_clean":"2023-09-23T15:14:49.630697+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:14:49.630697+0000","last_undegraded":"2023-09-23T15:14:49.630697+0000","last_fullsized":"2023-09-23T15:14:49.630697+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9353,"log_dups_size":0,"ondisk_log_size":9353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":208273432,"num_objects":3179,"num_object_clones":0,"num_object_copies":9537,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3179,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9353,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087167+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:14:31.087167+0000","last_peered":"2023-09-23T15:14:31.087167+0000","last_clean":"2023-09-23T15:14:31.087167+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:14:31.087167+0000","last_undegraded":"2023-09-23T15:14:31.087167+0000","last_fullsized":"2023-09-23T15:14:31.087167+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":197,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.095634+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:14:31.095634+0000","last_peered":"2023-09-23T15:14:31.095634+0000","last_clean":"2023-09-23T15:14:31.095634+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:14:31.095634+0000","last_undegraded":"2023-09-23T15:14:31.095634+0000","last_fullsized":"2023-09-23T15:14:31.095634+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":82,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.092862+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:14:31.092862+0000","last_peered":"2023-09-23T15:14:31.092862+0000","last_clean":"2023-09-23T15:14:31.092862+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:14:31.092862+0000","last_undegraded":"2023-09-23T15:14:31.092862+0000","last_fullsized":"2023-09-23T15:14:31.092862+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"206'9403","reported_seq":9448,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.350587+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:14:49.350587+0000","last_peered":"2023-09-23T15:14:49.350587+0000","last_clean":"2023-09-23T15:14:49.350587+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:14:49.350587+0000","last_undegraded":"2023-09-23T15:14:49.350587+0000","last_fullsized":"2023-09-23T15:14:49.350587+0000","mapping_epoch":171,"log_start":"179'3800","ondisk_log_start":"179'3800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5603,"log_dups_size":0,"ondisk_log_size":5603,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207421440,"num_objects":3165,"num_object_clones":0,"num_object_copies":9495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9403,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087231+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:14:31.087231+0000","last_peered":"2023-09-23T15:14:31.087231+0000","last_clean":"2023-09-23T15:14:31.087231+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:14:31.087231+0000","last_undegraded":"2023-09-23T15:14:31.087231+0000","last_fullsized":"2023-09-23T15:14:31.087231+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":382,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:42.951509+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:14:42.951509+0000","last_peered":"2023-09-23T15:14:42.951509+0000","last_clean":"2023-09-23T15:14:42.951509+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:14:42.951509+0000","last_undegraded":"2023-09-23T15:14:42.951509+0000","last_fullsized":"2023-09-23T15:14:42.951509+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":182,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088602+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:31.088602+0000","last_peered":"2023-09-23T15:14:31.088602+0000","last_clean":"2023-09-23T15:14:31.088602+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:14:31.088602+0000","last_undegraded":"2023-09-23T15:14:31.088602+0000","last_fullsized":"2023-09-23T15:14:31.088602+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.071135+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:14:31.071135+0000","last_peered":"2023-09-23T15:14:31.071135+0000","last_clean":"2023-09-23T15:14:31.071135+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:14:31.071135+0000","last_undegraded":"2023-09-23T15:14:31.071135+0000","last_fullsized":"2023-09-23T15:14:31.071135+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"206'9198","reported_seq":9243,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.450441+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:14:49.450441+0000","last_peered":"2023-09-23T15:14:49.450441+0000","last_clean":"2023-09-23T15:14:49.450441+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:14:49.450441+0000","last_undegraded":"2023-09-23T15:14:49.450441+0000","last_fullsized":"2023-09-23T15:14:49.450441+0000","mapping_epoch":171,"log_start":"177'2800","ondisk_log_start":"177'2800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6398,"log_dups_size":0,"ondisk_log_size":6398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":206831616,"num_objects":3156,"num_object_clones":0,"num_object_copies":9468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9198,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087710+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:14:31.087710+0000","last_peered":"2023-09-23T15:14:31.087710+0000","last_clean":"2023-09-23T15:14:31.087710+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:14:31.087710+0000","last_undegraded":"2023-09-23T15:14:31.087710+0000","last_fullsized":"2023-09-23T15:14:31.087710+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":724,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.089389+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:14:31.089389+0000","last_peered":"2023-09-23T15:14:31.089389+0000","last_clean":"2023-09-23T15:14:31.089389+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:14:31.089389+0000","last_undegraded":"2023-09-23T15:14:31.089389+0000","last_fullsized":"2023-09-23T15:14:31.089389+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":165,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085309+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:14:31.085309+0000","last_peered":"2023-09-23T15:14:31.085309+0000","last_clean":"2023-09-23T15:14:31.085309+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:14:31.085309+0000","last_undegraded":"2023-09-23T15:14:31.085309+0000","last_fullsized":"2023-09-23T15:14:31.085309+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":98,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:42.951177+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:14:42.951177+0000","last_peered":"2023-09-23T15:14:42.951177+0000","last_clean":"2023-09-23T15:14:42.951177+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:14:42.951177+0000","last_undegraded":"2023-09-23T15:14:42.951177+0000","last_fullsized":"2023-09-23T15:14:42.951177+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"206'9394","reported_seq":9437,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.202556+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:14:49.202556+0000","last_peered":"2023-09-23T15:14:49.202556+0000","last_clean":"2023-09-23T15:14:49.202556+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:14:49.202556+0000","last_undegraded":"2023-09-23T15:14:49.202556+0000","last_fullsized":"2023-09-23T15:14:49.202556+0000","mapping_epoch":171,"log_start":"179'4100","ondisk_log_start":"179'4100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5294,"log_dups_size":0,"ondisk_log_size":5294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211812352,"num_objects":3232,"num_object_clones":0,"num_object_copies":9696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9394,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087012+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:14:31.087012+0000","last_peered":"2023-09-23T15:14:31.087012+0000","last_clean":"2023-09-23T15:14:31.087012+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:14:31.087012+0000","last_undegraded":"2023-09-23T15:14:31.087012+0000","last_fullsized":"2023-09-23T15:14:31.087012+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":818,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.074537+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:14:31.074537+0000","last_peered":"2023-09-23T15:14:31.074537+0000","last_clean":"2023-09-23T15:14:31.074537+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:14:31.074537+0000","last_undegraded":"2023-09-23T15:14:31.074537+0000","last_fullsized":"2023-09-23T15:14:31.074537+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":78,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:42.950500+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:14:42.950500+0000","last_peered":"2023-09-23T15:14:42.950500+0000","last_clean":"2023-09-23T15:14:42.950500+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:14:42.950500+0000","last_undegraded":"2023-09-23T15:14:42.950500+0000","last_fullsized":"2023-09-23T15:14:42.950500+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":101,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.092789+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:14:31.092789+0000","last_peered":"2023-09-23T15:14:31.092789+0000","last_clean":"2023-09-23T15:14:31.092789+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:14:31.092789+0000","last_undegraded":"2023-09-23T15:14:31.092789+0000","last_fullsized":"2023-09-23T15:14:31.092789+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"206'9244","reported_seq":9287,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.634438+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:14:49.634438+0000","last_peered":"2023-09-23T15:14:49.634438+0000","last_clean":"2023-09-23T15:14:49.634438+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:14:49.634438+0000","last_undegraded":"2023-09-23T15:14:49.634438+0000","last_fullsized":"2023-09-23T15:14:49.634438+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9244,"log_dups_size":0,"ondisk_log_size":9244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":205520896,"num_objects":3136,"num_object_clones":0,"num_object_copies":9408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9244,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":493,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088659+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:14:31.088659+0000","last_peered":"2023-09-23T15:14:31.088659+0000","last_clean":"2023-09-23T15:14:31.088659+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:14:31.088659+0000","last_undegraded":"2023-09-23T15:14:31.088659+0000","last_fullsized":"2023-09-23T15:14:31.088659+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":494,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085899+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:14:31.085899+0000","last_peered":"2023-09-23T15:14:31.085899+0000","last_clean":"2023-09-23T15:14:31.085899+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:14:31.085899+0000","last_undegraded":"2023-09-23T15:14:31.085899+0000","last_fullsized":"2023-09-23T15:14:31.085899+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":120,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:34.520698+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:14:34.520698+0000","last_peered":"2023-09-23T15:14:34.520698+0000","last_clean":"2023-09-23T15:14:34.520698+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:14:34.520698+0000","last_undegraded":"2023-09-23T15:14:34.520698+0000","last_fullsized":"2023-09-23T15:14:34.520698+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":156,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.079999+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:14:31.079999+0000","last_peered":"2023-09-23T15:14:31.079999+0000","last_clean":"2023-09-23T15:14:31.079999+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:14:31.079999+0000","last_undegraded":"2023-09-23T15:14:31.079999+0000","last_fullsized":"2023-09-23T15:14:31.079999+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"206'9425","reported_seq":9471,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:49.553727+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:14:49.553727+0000","last_peered":"2023-09-23T15:14:49.553727+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:14:49.553727+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"177'3000","ondisk_log_start":"177'3000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6425,"log_dups_size":0,"ondisk_log_size":6425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":210960384,"num_objects":3219,"num_object_clones":0,"num_object_copies":9657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9425,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-1733}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085623+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:14:31.085623+0000","last_peered":"2023-09-23T15:14:31.085623+0000","last_clean":"2023-09-23T15:14:31.085623+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:14:31.085623+0000","last_undegraded":"2023-09-23T15:14:31.085623+0000","last_fullsized":"2023-09-23T15:14:31.085623+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":289,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:42.950277+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:14:42.950277+0000","last_peered":"2023-09-23T15:14:42.950277+0000","last_clean":"2023-09-23T15:14:42.950277+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:14:42.950277+0000","last_undegraded":"2023-09-23T15:14:42.950277+0000","last_fullsized":"2023-09-23T15:14:42.950277+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":150,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087561+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:14:31.087561+0000","last_peered":"2023-09-23T15:14:31.087561+0000","last_clean":"2023-09-23T15:14:31.087561+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:14:31.087561+0000","last_undegraded":"2023-09-23T15:14:31.087561+0000","last_fullsized":"2023-09-23T15:14:31.087561+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":180,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086530+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:14:31.086530+0000","last_peered":"2023-09-23T15:14:31.086530+0000","last_clean":"2023-09-23T15:14:31.086530+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:14:31.086530+0000","last_undegraded":"2023-09-23T15:14:31.086530+0000","last_fullsized":"2023-09-23T15:14:31.086530+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"206'9270","reported_seq":9315,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:48.528245+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:14:48.528245+0000","last_peered":"2023-09-23T15:14:48.528245+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:14:48.528245+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"174'100","ondisk_log_start":"174'100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9170,"log_dups_size":0,"ondisk_log_size":9170,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212205568,"num_objects":3238,"num_object_clones":0,"num_object_copies":9714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9270,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-1636}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086852+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:14:31.086852+0000","last_peered":"2023-09-23T15:14:31.086852+0000","last_clean":"2023-09-23T15:14:31.086852+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:14:31.086852+0000","last_undegraded":"2023-09-23T15:14:31.086852+0000","last_fullsized":"2023-09-23T15:14:31.086852+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":811,"reported_epoch":205,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:31.086127+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:14:31.086127+0000","last_peered":"2023-09-23T15:14:31.086127+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:14:31.086127+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":78,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:42.951367+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:14:42.951367+0000","last_peered":"2023-09-23T15:14:42.951367+0000","last_clean":"2023-09-23T15:14:42.951367+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:14:42.951367+0000","last_undegraded":"2023-09-23T15:14:42.951367+0000","last_fullsized":"2023-09-23T15:14:42.951367+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":155,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085858+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:14:31.085858+0000","last_peered":"2023-09-23T15:14:31.085858+0000","last_clean":"2023-09-23T15:14:31.085858+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:14:31.085858+0000","last_undegraded":"2023-09-23T15:14:31.085858+0000","last_fullsized":"2023-09-23T15:14:31.085858+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"206'9099","reported_seq":9142,"reported_epoch":206,"state":"active+clean","last_fresh":"2023-09-23T15:14:49.328723+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:14:49.328723+0000","last_peered":"2023-09-23T15:14:49.328723+0000","last_clean":"2023-09-23T15:14:49.328723+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:14:49.328723+0000","last_undegraded":"2023-09-23T15:14:49.328723+0000","last_fullsized":"2023-09-23T15:14:49.328723+0000","mapping_epoch":171,"log_start":"175'1500","ondisk_log_start":"175'1500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209911808,"num_objects":3203,"num_object_clones":0,"num_object_copies":9609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9099,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086309+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:14:31.086309+0000","last_peered":"2023-09-23T15:14:31.086309+0000","last_clean":"2023-09-23T15:14:31.086309+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:14:31.086309+0000","last_undegraded":"2023-09-23T15:14:31.086309+0000","last_fullsized":"2023-09-23T15:14:31.086309+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.109373+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:14:31.109373+0000","last_peered":"2023-09-23T15:14:31.109373+0000","last_clean":"2023-09-23T15:14:31.109373+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:14:31.109373+0000","last_undegraded":"2023-09-23T15:14:31.109373+0000","last_fullsized":"2023-09-23T15:14:31.109373+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":665,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.093552+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:14:31.093552+0000","last_peered":"2023-09-23T15:14:31.093552+0000","last_clean":"2023-09-23T15:14:31.093552+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:14:31.093552+0000","last_undegraded":"2023-09-23T15:14:31.093552+0000","last_fullsized":"2023-09-23T15:14:31.093552+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.095846+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:14:31.095846+0000","last_peered":"2023-09-23T15:14:31.095846+0000","last_clean":"2023-09-23T15:14:31.095846+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:14:31.095846+0000","last_undegraded":"2023-09-23T15:14:31.095846+0000","last_fullsized":"2023-09-23T15:14:31.095846+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"206'9436","reported_seq":9486,"reported_epoch":206,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:49.195614+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:14:49.195614+0000","last_peered":"2023-09-23T15:14:49.195614+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:14:49.195614+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"179'4100","ondisk_log_start":"179'4100","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5336,"log_dups_size":0,"ondisk_log_size":5336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209321984,"num_objects":3194,"num_object_clones":0,"num_object_copies":9582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9436,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-1772}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":482,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086181+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:14:31.086181+0000","last_peered":"2023-09-23T15:14:31.086181+0000","last_clean":"2023-09-23T15:14:31.086181+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:14:31.086181+0000","last_undegraded":"2023-09-23T15:14:31.086181+0000","last_fullsized":"2023-09-23T15:14:31.086181+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":682,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.080209+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:14:31.080209+0000","last_peered":"2023-09-23T15:14:31.080209+0000","last_clean":"2023-09-23T15:14:31.080209+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:14:31.080209+0000","last_undegraded":"2023-09-23T15:14:31.080209+0000","last_fullsized":"2023-09-23T15:14:31.080209+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":168,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.086616+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:14:31.086616+0000","last_peered":"2023-09-23T15:14:31.086616+0000","last_clean":"2023-09-23T15:14:31.086616+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:14:31.086616+0000","last_undegraded":"2023-09-23T15:14:31.086616+0000","last_fullsized":"2023-09-23T15:14:31.086616+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":147,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085383+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:14:31.085383+0000","last_peered":"2023-09-23T15:14:31.085383+0000","last_clean":"2023-09-23T15:14:31.085383+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:14:31.085383+0000","last_undegraded":"2023-09-23T15:14:31.085383+0000","last_fullsized":"2023-09-23T15:14:31.085383+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":158,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:39.419251+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:14:39.419251+0000","last_peered":"2023-09-23T15:14:39.419251+0000","last_clean":"2023-09-23T15:14:39.419251+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:14:39.419251+0000","last_undegraded":"2023-09-23T15:14:39.419251+0000","last_fullsized":"2023-09-23T15:14:39.419251+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":205,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.070976+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:14:31.070976+0000","last_peered":"2023-09-23T15:14:31.070976+0000","last_clean":"2023-09-23T15:14:31.070976+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:14:31.070976+0000","last_undegraded":"2023-09-23T15:14:31.070976+0000","last_fullsized":"2023-09-23T15:14:31.070976+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":184,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.090600+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:14:31.090600+0000","last_peered":"2023-09-23T15:14:31.090600+0000","last_clean":"2023-09-23T15:14:31.090600+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:14:31.090600+0000","last_undegraded":"2023-09-23T15:14:31.090600+0000","last_fullsized":"2023-09-23T15:14:31.090600+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":198,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.085479+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:14:31.085479+0000","last_peered":"2023-09-23T15:14:31.085479+0000","last_clean":"2023-09-23T15:14:31.085479+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:14:31.085479+0000","last_undegraded":"2023-09-23T15:14:31.085479+0000","last_fullsized":"2023-09-23T15:14:31.085479+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":150,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088520+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:14:31.088520+0000","last_peered":"2023-09-23T15:14:31.088520+0000","last_clean":"2023-09-23T15:14:31.088520+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:14:31.088520+0000","last_undegraded":"2023-09-23T15:14:31.088520+0000","last_fullsized":"2023-09-23T15:14:31.088520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":181,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.088683+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:14:31.088683+0000","last_peered":"2023-09-23T15:14:31.088683+0000","last_clean":"2023-09-23T15:14:31.088683+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:14:31.088683+0000","last_undegraded":"2023-09-23T15:14:31.088683+0000","last_fullsized":"2023-09-23T15:14:31.088683+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":177,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.108993+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:14:31.108993+0000","last_peered":"2023-09-23T15:14:31.108993+0000","last_clean":"2023-09-23T15:14:31.108993+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:14:31.108993+0000","last_undegraded":"2023-09-23T15:14:31.108993+0000","last_fullsized":"2023-09-23T15:14:31.108993+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":191,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.100304+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:14:31.100304+0000","last_peered":"2023-09-23T15:14:31.100304+0000","last_clean":"2023-09-23T15:14:31.100304+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:14:31.100304+0000","last_undegraded":"2023-09-23T15:14:31.100304+0000","last_fullsized":"2023-09-23T15:14:31.100304+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":153,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.087534+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:14:31.087534+0000","last_peered":"2023-09-23T15:14:31.087534+0000","last_clean":"2023-09-23T15:14:31.087534+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:14:31.087534+0000","last_undegraded":"2023-09-23T15:14:31.087534+0000","last_fullsized":"2023-09-23T15:14:31.087534+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":174,"reported_epoch":205,"state":"active+clean","last_fresh":"2023-09-23T15:14:31.084931+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:14:31.084931+0000","last_peered":"2023-09-23T15:14:31.084931+0000","last_clean":"2023-09-23T15:14:31.084931+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:14:31.084931+0000","last_undegraded":"2023-09-23T15:14:31.084931+0000","last_fullsized":"2023-09-23T15:14:31.084931+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3364618264,"num_objects":51341,"num_object_clones":0,"num_object_copies":154023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24006,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":51341,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149437,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5694,"num_bytes_recovered":373161984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9484840960,"data_stored":9484828744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":118037,"ondisk_log_size":118037,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201521,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463157,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1793612,"kb_used_data":1649604,"kb_used_omap":0,"kb_used_meta":144000,"kb_avail":91951540,"statfs":{"total":95995035648,"available":94158376960,"internally_reserved":0,"allocated":1689194496,"data_stored":1688510195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":147456000},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.54,"5min":4.2370000000000001,"15min":2.633},"min":{"1min":0.23699999999999999,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":12.926,"5min":95.721000000000004,"15min":108.39400000000001},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.863,"5min":4.4660000000000002,"15min":2.7400000000000002},"min":{"1min":0.23400000000000001,"5min":0.223,"15min":0.223},"max":{"1min":11.614000000000001,"5min":96.019000000000005,"15min":108.452},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":4.5350000000000001,"5min":3.6030000000000002,"15min":3.448},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":43.377000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":7.2919999999999998,"5min":4.3440000000000003,"15min":3.8519999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":42.704000000000001,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":3.4239999999999999,"5min":6.6059999999999999,"15min":4.8700000000000001},"min":{"1min":0.33500000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.664000000000001,"5min":139.17500000000001,"15min":180.374},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.3690000000000002,"5min":6.4640000000000004,"15min":4.8700000000000001},"min":{"1min":0.33300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":16.552,"5min":146.999,"15min":181.685},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":12.955,"5min":4.3739999999999997,"15min":1.992},"min":{"1min":0.441,"5min":0.249,"15min":0.249},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":0.627},{"interface":"front","average":{"1min":12.819000000000001,"5min":4.4420000000000002,"15min":1.982},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":80.977999999999994,"5min":80.977999999999994,"15min":80.977999999999994},"last":0.77900000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.8999999999999999,"15min":1.0329999999999999},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5119999999999996,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.248},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.998,"15min":1.0980000000000001},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.5760000000000001,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.30599999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.8399999999999999,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":5.5880000000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9470000000000001,"15min":1.431},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4550000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.36399999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.381,"15min":1},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":7.5910000000000002,"15min":17},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3939999999999999,"15min":0.97499999999999998},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":3.754,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.35199999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015195,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":797784,"kb_used_data":649168,"kb_used_omap":6,"kb_used_meta":148601,"kb_avail":92947368,"statfs":{"total":95995035648,"available":95178104832,"internally_reserved":0,"allocated":664748032,"data_stored":663766842,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":152167499},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":3.6080000000000001,"15min":3.738},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.0839999999999996,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":3.5609999999999999,"5min":4.4000000000000004,"15min":4.54},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.558,"5min":16.131,"15min":16.131},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0229999999999997,"5min":4.3070000000000004,"15min":4.1870000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.757999999999999,"5min":13.843,"15min":13.843},"last":0.34899999999999998},{"interface":"front","average":{"1min":4.3380000000000001,"5min":4.2460000000000004,"15min":4.2300000000000004},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.44}]},{"osd":3,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9539999999999997,"5min":9.6270000000000007,"15min":10.406000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.27,"5min":123.426,"15min":123.426},"last":0.88200000000000001},{"interface":"front","average":{"1min":3.5499999999999998,"5min":10.762,"15min":11.964},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1010000000000009,"5min":123.32899999999999,"15min":123.32899999999999},"last":5.7510000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.7090000000000001,"15min":2.7000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.7199999999999998,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.129,"5min":3.472,"15min":3.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.779,"5min":16.779,"15min":16.779},"last":3.7000000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":3.419,"15min":3.6589999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8259999999999996,"5min":16.166,"15min":16.166},"last":5.827},{"interface":"front","average":{"1min":3.0190000000000001,"5min":2.9209999999999998,"15min":2.9049999999999998},"min":{"1min":0.35399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7640000000000002,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":4.8920000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":3.488,"15min":3.702},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.8010000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":5.7060000000000004},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.8050000000000002,"15min":2.7669999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.7930000000000001,"5min":7.7930000000000001,"15min":7.7930000000000001},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.1749999999999998,"15min":3.1629999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.8179999999999996,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2909999999999999,"15min":3.3330000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":6.8170000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893244,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078956,"kb_used_data":829668,"kb_used_omap":8,"kb_used_meta":249271,"kb_avail":92666196,"statfs":{"total":95995035648,"available":94890184704,"internally_reserved":0,"allocated":849580032,"data_stored":848696080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":255254100},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.875,"15min":4.1529999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.5099999999999998,"5min":31.510999999999999,"15min":31.510999999999999},"last":2.4209999999999998},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.274,"15min":4.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4249999999999998,"5min":31.378,"15min":31.378},"last":1.611}]},{"osd":2,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":7.093,"5min":5.359,"15min":2.375},"min":{"1min":0.46000000000000002,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":31.271999999999998,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.86599999999999999},{"interface":"front","average":{"1min":4.9180000000000001,"5min":4.4139999999999997,"15min":1.9850000000000001},"min":{"1min":0.51500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":31.542999999999999,"5min":122.071,"15min":122.071},"last":2.6280000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8159999999999998,"5min":5.5419999999999998,"15min":2.3420000000000001},"min":{"1min":0.35299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.643000000000001,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.497},{"interface":"front","average":{"1min":4.9729999999999999,"5min":4.6059999999999999,"15min":2.3490000000000002},"min":{"1min":0.40699999999999997,"5min":0.252,"15min":0.252},"max":{"1min":25.632000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":5.5730000000000004,"5min":3.0129999999999999,"15min":1.5209999999999999},"min":{"1min":0.51200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":31.341999999999999,"5min":31.341999999999999,"15min":31.341999999999999},"last":1.496},{"interface":"front","average":{"1min":4.556,"5min":2.9670000000000001,"15min":1.538},"min":{"1min":0.35999999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":31.059999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":2.1320000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.1289999999999996,"5min":3.0569999999999999,"15min":1.6180000000000001},"min":{"1min":0.33500000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":31.25,"5min":109.369,"15min":109.369},"last":0.59599999999999997},{"interface":"front","average":{"1min":4.032,"5min":3.532,"15min":1.7350000000000001},"min":{"1min":0.36399999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":31.295000000000002,"5min":109.387,"15min":109.387},"last":1.534}]},{"osd":6,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":3.0259999999999998,"5min":3.0259999999999998,"15min":1.5780000000000001},"min":{"1min":0.41299999999999998,"5min":0.309,"15min":0.309},"max":{"1min":16.893999999999998,"5min":99.114999999999995,"15min":99.114999999999995},"last":1.304},{"interface":"front","average":{"1min":4.0350000000000001,"5min":2.9220000000000002,"15min":1.526},"min":{"1min":0.36899999999999999,"5min":0.314,"15min":0.314},"max":{"1min":31.431000000000001,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.55700000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:13:59 2023","interfaces":[{"interface":"back","average":{"1min":4.6189999999999998,"5min":3.2130000000000001,"15min":1.6259999999999999},"min":{"1min":0.40400000000000003,"5min":0.372,"15min":0.372},"max":{"1min":31.472000000000001,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.626},{"interface":"front","average":{"1min":4.5099999999999998,"5min":2.4510000000000001,"15min":1.4139999999999999},"min":{"1min":0.53700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":31.225999999999999,"5min":31.225999999999999,"15min":31.225999999999999},"last":1.5620000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566191,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2195316,"kb_used_data":1880620,"kb_used_omap":11,"kb_used_meta":314676,"kb_avail":91549836,"statfs":{"total":95995035648,"available":93747032064,"internally_reserved":0,"allocated":1925754880,"data_stored":1924867124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":322229008},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":5.8540000000000001,"5min":2.7320000000000002,"15min":1.25},"min":{"1min":0.23000000000000001,"5min":0.184,"15min":0.184},"max":{"1min":38.274000000000001,"5min":38.274000000000001,"15min":38.274000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":4.6130000000000004,"5min":2.3580000000000001,"15min":1.111},"min":{"1min":0.307,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":33.026000000000003,"5min":33.026000000000003,"15min":33.026000000000003},"last":4.0410000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:14:47 2023","interfaces":[{"interface":"back","average":{"1min":9.2129999999999992,"5min":5.8739999999999997,"15min":5.3179999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":84.641000000000005,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.57099999999999995},{"interface":"front","average":{"1min":9.2040000000000006,"5min":3.8799999999999999,"15min":2.992},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":84.751999999999995,"5min":84.751999999999995,"15min":84.751999999999995},"last":8.9779999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:14:27 2023","interfaces":[{"interface":"back","average":{"1min":10.016,"5min":3.4990000000000001,"15min":1.748},"min":{"1min":0.44700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":87.108000000000004,"5min":87.108000000000004,"15min":87.108000000000004},"last":2.0680000000000001},{"interface":"front","average":{"1min":9.4209999999999994,"5min":3.3980000000000001,"15min":1.754},"min":{"1min":0.59599999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":84.739000000000004,"5min":84.739000000000004,"15min":84.739000000000004},"last":4.6260000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.4820000000000002,"5min":2.6190000000000002,"15min":1.3720000000000001},"min":{"1min":0.30599999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.6589999999999998,"5min":52.795999999999999,"15min":52.795999999999999},"last":4.1619999999999999},{"interface":"front","average":{"1min":2.7389999999999999,"5min":3.6280000000000001,"15min":1.7090000000000001},"min":{"1min":0.434,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.478999999999999,"5min":68.894999999999996,"15min":68.894999999999996},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.9500000000000002,"5min":2.7719999999999998,"15min":1.4790000000000001},"min":{"1min":0.377,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.161000000000001,"5min":41.723999999999997,"15min":41.723999999999997},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.7200000000000002,"5min":2.7469999999999999,"15min":1.4410000000000001},"min":{"1min":0.30399999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":6.694,"5min":39.822000000000003,"15min":39.822000000000003},"last":4.1260000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":2.73,"5min":2.9239999999999999,"15min":1.5329999999999999},"min":{"1min":0.35999999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":6.5800000000000001,"5min":37.834000000000003,"15min":37.834000000000003},"last":4.0960000000000001},{"interface":"front","average":{"1min":2.4910000000000001,"5min":2.3959999999999999,"15min":1.387},"min":{"1min":0.372,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":13.019,"5min":37.749000000000002,"15min":37.749000000000002},"last":0.66500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:14:02 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.383,"15min":1.3260000000000001},"min":{"1min":0.42399999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":18.207999999999998,"5min":18.207999999999998,"15min":18.207999999999998},"last":0.53200000000000003},{"interface":"front","average":{"1min":2.7629999999999999,"5min":2.2669999999999999,"15min":1.3149999999999999},"min":{"1min":0.41299999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.6299999999999999,"5min":9.2560000000000002,"15min":9.2560000000000002},"last":4.1909999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402665,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2546624,"kb_used_data":2259320,"kb_used_omap":3,"kb_used_meta":287292,"kb_avail":91198528,"statfs":{"total":95995035648,"available":93387292672,"internally_reserved":0,"allocated":2313543680,"data_stored":2312698393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":294187177},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":4.0629999999999997,"5min":3.0760000000000001,"15min":1.764},"min":{"1min":0.36499999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":25.111999999999998,"5min":61.744,"15min":79.611000000000004},"last":13.064},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.0870000000000002,"15min":1.788},"min":{"1min":0.29599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.6270000000000007,"5min":61.686,"15min":79.656000000000006},"last":10.249000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":6.2789999999999999,"5min":4.9240000000000004,"15min":4.6980000000000004},"min":{"1min":0.34200000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.873999999999999,"5min":28.315000000000001,"15min":28.315000000000001},"last":10.268000000000001},{"interface":"front","average":{"1min":4.7350000000000003,"5min":4.8230000000000004,"15min":4.8380000000000001},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.395,"5min":14.427,"15min":14.427},"last":13.164}]},{"osd":2,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":5.4690000000000003,"5min":5.5890000000000004,"15min":3.2749999999999999},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":25.187000000000001,"5min":109.077,"15min":109.077},"last":13.196},{"interface":"front","average":{"1min":4.4580000000000002,"5min":6.0449999999999999,"15min":3.4279999999999999},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.873000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":7.4500000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.9910000000000001,"15min":1.1299999999999999},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4169999999999998,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":2.3799999999999999},{"interface":"front","average":{"1min":2.802,"5min":2.1509999999999998,"15min":1.3320000000000001},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.632999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":0.83299999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.8170000000000002,"15min":1.46},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":6.3869999999999996,"5min":56.960999999999999,"15min":56.960999999999999},"last":2.4249999999999998},{"interface":"front","average":{"1min":2.839,"5min":3.0630000000000002,"15min":1.675},"min":{"1min":0.36399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":10.738,"5min":56.981999999999999,"15min":56.981999999999999},"last":0.80200000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.355,"15min":1.3260000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.3419999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":2.4550000000000001},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.6219999999999999,"15min":1.5780000000000001},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":10.561999999999999,"5min":19.355,"15min":19.355},"last":0.85599999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.726,"15min":1.585},"min":{"1min":0.42599999999999999,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":10.603,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.876},{"interface":"front","average":{"1min":2.702,"5min":2.113,"15min":1.2050000000000001},"min":{"1min":0.437,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.351000000000001,"5min":14.351000000000001,"15min":14.351000000000001},"last":13.109}]}]},{"osd":4,"up_from":31,"seq":133143986430,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1214152,"kb_used_data":1034240,"kb_used_omap":0,"kb_used_meta":179904,"kb_avail":92531000,"statfs":{"total":95995035648,"available":94751744000,"internally_reserved":0,"allocated":1059061760,"data_stored":1058381034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":184221696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.2949999999999999,"5min":4.0780000000000003,"15min":2.2869999999999999},"min":{"1min":0.25,"5min":0.219,"15min":0.219},"max":{"1min":6.6799999999999997,"5min":108.148,"15min":122.69199999999999},"last":2.9910000000000001},{"interface":"front","average":{"1min":1.1679999999999999,"5min":3.903,"15min":2.306},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.6719999999999997,"5min":108.124,"15min":122.745},"last":0.27300000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":4.2539999999999996,"15min":4.4500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.783999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":2.6709999999999998},{"interface":"front","average":{"1min":3.5699999999999998,"5min":4.7750000000000004,"15min":4.976},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":28.815000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":1.4139999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:13:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6850000000000001,"5min":6.2439999999999998,"15min":2.7719999999999998},"min":{"1min":0.41099999999999998,"5min":0.255,"15min":0.255},"max":{"1min":6.96,"5min":148.08000000000001,"15min":148.08000000000001},"last":1.264},{"interface":"front","average":{"1min":4.1719999999999997,"5min":6.6150000000000002,"15min":3.0299999999999998},"min":{"1min":0.32600000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":34.798999999999999,"5min":147.88900000000001,"15min":147.88900000000001},"last":3.2570000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":4.8849999999999998,"5min":8.2460000000000004,"15min":3.3100000000000001},"min":{"1min":0.32400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":38.685000000000002,"5min":163.97499999999999,"15min":163.97499999999999},"last":3.1720000000000002},{"interface":"front","average":{"1min":5.5129999999999999,"5min":8.0139999999999993,"15min":3.149},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":38.767000000000003,"5min":164.447,"15min":164.447},"last":1.2030000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:14:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.452,"15min":1.1950000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.1479999999999999,"5min":14.073,"15min":28.891999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.208,"15min":1.0600000000000001},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.577,"5min":8.9920000000000009,"15min":32.100999999999999},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:14:22 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":3.6800000000000002,"15min":1.7350000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3319999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.69099999999999995,"5min":3.323,"15min":1.6040000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.5179999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.502,"15min":1.1319999999999999},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":1.6419999999999999,"5min":18.23,"15min":18.23},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.48,"15min":1.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":2.214,"5min":14.044,"15min":14.653},"last":0.47399999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724793,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1212800,"kb_used_data":1018936,"kb_used_omap":0,"kb_used_meta":193856,"kb_avail":92532352,"statfs":{"total":95995035648,"available":94753128448,"internally_reserved":0,"allocated":1043390464,"data_stored":1042713717,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":198508544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":2.0859999999999999,"15min":1.2190000000000001},"min":{"1min":0.24299999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.968,"5min":17.044,"15min":24.962},"last":0.621},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.823,"15min":1.1100000000000001},"min":{"1min":0.27200000000000002,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":1.95,"5min":18.814,"15min":25.317},"last":0.46800000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5030000000000001,"5min":2.2389999999999999,"15min":1.861},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.082999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":2.9510000000000001,"5min":2.4049999999999998,"15min":2.3140000000000001},"min":{"1min":0.39200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":17.963999999999999,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.497}]},{"osd":2,"last update":"Sat Sep 23 15:13:58 2023","interfaces":[{"interface":"back","average":{"1min":3.5830000000000002,"5min":4.7000000000000002,"15min":2.097},"min":{"1min":0.31,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":9.7859999999999996,"5min":97.087999999999994,"15min":97.087999999999994},"last":33.68},{"interface":"front","average":{"1min":4.5129999999999999,"5min":4.4930000000000003,"15min":2.0619999999999998},"min":{"1min":0.33400000000000002,"5min":0.254,"15min":0.254},"max":{"1min":13.332000000000001,"5min":97.305999999999997,"15min":97.305999999999997},"last":33.707999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":5.4550000000000001,"5min":3.7829999999999999,"15min":1.8360000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":32.305999999999997,"5min":53.088999999999999,"15min":53.088999999999999},"last":80.570999999999998},{"interface":"front","average":{"1min":5.0970000000000004,"5min":3.5249999999999999,"15min":1.734},"min":{"1min":0.46400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":24.052,"5min":53.764000000000003,"15min":53.764000000000003},"last":80.551000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":2.1480000000000001,"15min":1.1639999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.369,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.2349999999999999,"15min":1.238},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.88200000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.35899999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:14:03 2023","interfaces":[{"interface":"back","average":{"1min":0.94999999999999996,"5min":2.0539999999999998,"15min":1.107},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.8760000000000003,"5min":15.196,"15min":15.196},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":1.752,"15min":1.008},"min":{"1min":0.40000000000000002,"5min":0.32600000000000001,"15min":0.30299999999999999},"max":{"1min":1.6419999999999999,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.8580000000000001,"15min":1.228},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":1.331,"5min":11.353999999999999,"15min":15.301},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.9119999999999999,"15min":1.212},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.8460000000000001,"5min":15.762,"15min":15.762},"last":0.54000000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":835915776,"data_stored":835911704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":656936960,"data_stored":656932888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1918435328,"data_stored":1918435328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2305097728,"data_stored":2305097728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1050345472,"data_stored":1050345472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1038684160,"data_stored":1038680088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1679425536,"data_stored":1679425536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:14:56.144 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:14:56.179 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:56.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:56.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:55 smithi027 ceph-mon[195233]: from='client.45293 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:14:56.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:55 smithi027 ceph-mon[195233]: pgmap v552: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 24009/150531 objects degraded (15.950%); 1.5 MiB/s, 24 objects/s recovering 2023-09-23T15:14:56.456 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:56.607 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:56.629 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:56.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:56.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:56.698 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:14:56.698 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:14:56.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:56.797 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:56.870 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:14:56.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:56.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:14:56.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:57.039 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:56 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24009/150531 objects degraded (15.950%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:57.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:56 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24009/150531 objects degraded (15.950%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:57.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:56 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24009/150531 objects degraded (15.950%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:14:57.228 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:14:57.268 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:57.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:57.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:57.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:57.424 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:57.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:14:57.511 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:57.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:57.518 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:57.619 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:14:57.713 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:57.975 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:14:58.100 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:14:58.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:58.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:14:58.220 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:14:58.244 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:57 smithi027 ceph-mon[195233]: pgmap v553: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:58.263 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:58.336 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:57 smithi017 ceph-mon[222878]: pgmap v553: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:58.336 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:57 smithi017 ceph-mon[215646]: pgmap v553: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:58.336 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:14:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:14:58] "GET /metrics HTTP/1.1" 200 36322 "" "Prometheus/2.43.0" 2023-09-23T15:14:58.424 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:14:58.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:14:58.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:58.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:14:58.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:14:58.614 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:58.829 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:14:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:58.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:14:58.943 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:14:58.978 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:59.106 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:14:59.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:59.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:14:59.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:14:59.245 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:59.245 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:59.245 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:14:59 smithi027 ceph-mon[195233]: pgmap v554: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:59.249 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:59.343 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:14:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:59.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:14:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[215646]: pgmap v554: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:14:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:14:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:14:59 smithi017 ceph-mon[222878]: pgmap v554: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 23872/145779 objects degraded (16.375%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:14:59.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:14:59.521 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:14:59.672 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:14:59.672 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:14:59.823 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:14:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:14:59.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:14:59.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:14:59.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:00.031 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:00.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:00.052 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:00.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:00.165 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:00.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3558400874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.183 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:00 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.265 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:15:00.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3558400874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:00 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.423 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:00.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:00.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:00.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3558400874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:00 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:00.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:15:00.527 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:00.595 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:00.667 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:00.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:00.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:00.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:00.828 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:00.900 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:00.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:00.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:01.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:01.013 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:01.058 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[215646]: pgmap v556: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 23724/137763 objects degraded (17.221%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:01 smithi017 ceph-mon[222878]: pgmap v556: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 23724/137763 objects degraded (17.221%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: osdmap e208: 8 total, 8 up, 7 in 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: failed to encode map e208 with expected crc 2023-09-23T15:15:01.563 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:01 smithi027 ceph-mon[195233]: pgmap v556: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 614 op/s; 23724/137763 objects degraded (17.221%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:15:01.574 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:01.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:01.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:01.661 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:01.661 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23724/137763 objects degraded (17.221%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":709,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":208,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":45921,"data_bytes":3012401372,"bytes_used":10210934784,"bytes_avail":661754314752,"bytes_total":671965249536,"degraded_objects":23724,"degraded_total":137763,"degraded_ratio":0.17220879336251391,"recovering_objects_per_sec":32,"recovering_bytes_per_sec":2107557,"recovering_keys_per_sec":0,"num_objects_recovered":285,"num_bytes_recovered":18677760,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":614},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 24s)","progress":0.93577980995178223,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:15:01.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:01.718 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:01.763 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:01.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:01.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:01.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:01.975 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:02.431 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4129555784' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:02.431 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23724/137763 objects degraded (17.221%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:02.677 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:15:02.677 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:15:02.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4129555784' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:02.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23724/137763 objects degraded (17.221%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:02.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4129555784' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:02.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23724/137763 objects degraded (17.221%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:03.339 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:15:03.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:15:03.371 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:03.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:03.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:03.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:03.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:15:03.523 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:03 smithi027 ceph-mon[195233]: pgmap v557: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 23724/137763 objects degraded (17.221%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:03.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:03 smithi017 ceph-mon[215646]: pgmap v557: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 23724/137763 objects degraded (17.221%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:03.546 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:03 smithi017 ceph-mon[222878]: pgmap v557: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 544 op/s; 23724/137763 objects degraded (17.221%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:03.550 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:03.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:15:03.734 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:03.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:03.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:03.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:03.938 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:03.940 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:04.086 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:04.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:04.095 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:04.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:04.142 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:04.186 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:04.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:04.226 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:04.230 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:04.277 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:04.316 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:04.324 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:04.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:04.414 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:04.445 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.445 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:15:04.461 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/189475629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:04.504 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:04.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:04.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:04.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/189475629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:04.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/189475629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:04.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:04.668 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:04.951 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:05.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:05.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:05.179 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:05.253 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:05.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:05.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:05.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:05.438 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:05 smithi027 ceph-mon[195233]: pgmap v558: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 23724/134298 objects degraded (17.665%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:05.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:05 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:15:05.618 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:05.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:05 smithi017 ceph-mon[222878]: pgmap v558: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 23724/134298 objects degraded (17.665%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:05.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:05 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:15:05.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:05.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:05 smithi017 ceph-mon[215646]: pgmap v558: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 23724/134298 objects degraded (17.665%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:05.627 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:05 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:15:05.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:05.651 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:15:05.651 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:15:05.651 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:15:05.651 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:15:05.651 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:15:05.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:05.802 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:05.973 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:06.168 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:06.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:06.180 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:06.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:06.351 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:06.704 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:06.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:15:06.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:06.708 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:06.809 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:06.891 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:06.969 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:06.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:06.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:07.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:07.170 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:07.410 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:07.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:07.422 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:07.455 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:07.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:07.571 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:07.689 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:07 smithi027 ceph-mon[195233]: pgmap v559: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:07.689 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23584/129753 objects degraded (18.176%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:07.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:15:07.704 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:07.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:07.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:07.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:07 smithi017 ceph-mon[215646]: pgmap v559: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:07.788 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23584/129753 objects degraded (18.176%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:07.788 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:07 smithi017 ceph-mon[222878]: pgmap v559: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:07.788 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23584/129753 objects degraded (18.176%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:07.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:07.852 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:07.966 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:07.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:07.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:08.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:08.129 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:08] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T15:15:08.139 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:08.613 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:08.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:08.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:08.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:08.759 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:08.977 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:08.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:08.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:09.013 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:09.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:09.108 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:09.125 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:09.130 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:15:09.188 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:09.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:09.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:09.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:09.384 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:41922 5.2 5:47943cc1:::benchmark_data_smithi017_288048_object41921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 430.74510949699999, 2023-09-23T15:15:09.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.246440672, 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 41922 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696425+0000", 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696423+0000", 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696532+0000", 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108953 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696546+0000", 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3548e-05 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696553+0000", 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7490000000000001e-06 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696709+0000", 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015584999999999999 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.696849+0000", 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014056499999999999 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.697002+0000", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015253599999999999 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.698626+0000", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00162414 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.699083+0000", 2023-09-23T15:15:09.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045717099999999998 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942745+0000", 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24366205099999999 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942810+0000", 2023-09-23T15:15:09.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4618999999999999e-05 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:58.942866+0000", 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6088999999999998e-05 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166949 5.2 5:4bbd8647:::benchmark_data_smithi017_288048_object55180:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:15:09.451 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 291.35378319099999, 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.2839622, 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166949 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087751+0000", 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087750+0000", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087759+0000", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9779999999999994e-06 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087779+0000", 2023-09-23T15:15:09.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0055999999999999e-05 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087787+0000", 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0169999999999996e-06 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087834+0000", 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7286000000000001e-05 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087944+0000", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010972 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.088049+0000", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010541 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091850+0000", 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038003780000000001 2023-09-23T15:15:09.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099251+0000", 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007400968 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371615+0000", 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27236480099999999 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371672+0000", 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6801e-05 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371714+0000", 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1390000000000002e-05 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:8698 6.3 6:cf7ea04c:::benchmark_data_smithi017_350392_object8697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:52.452890+0000", 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 196.98864410499999, 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21246282499999999, 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 8698 2023-09-23T15:15:09.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.452890+0000", 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.452890+0000", 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.452889+0000", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453002+0000", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112543 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453017+0000", 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5551000000000001e-05 2023-09-23T15:15:09.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453024+0000", 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5860000000000001e-06 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453064+0000", 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0160000000000002e-05 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453180+0000", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011637900000000001 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.453307+0000", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012651700000000001 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.454199+0000", 2023-09-23T15:15:09.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00089222499999999996 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665214+0000", 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.2110148 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665275+0000", 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1187999999999998e-05 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665353+0000", 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8220000000000007e-05 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:8708 6.1 6:847fcbfc:::benchmark_data_smithi017_350392_object8707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:52.455675+0000", 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 196.98585957899999, 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20972206600000001, 2023-09-23T15:15:09.466 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 8708 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455675+0000", 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455675+0000", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455674+0000", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455729+0000", 2023-09-23T15:15:09.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4932000000000002e-05 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455743+0000", 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4411e-05 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455750+0000", 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8059999999999997e-06 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455800+0000", 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9793999999999998e-05 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.455925+0000", 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012548400000000001 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:15:09.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.456040+0000", 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115015 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.456685+0000", 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064474400000000005 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665249+0000", 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.208563522 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665302+0000", 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2902000000000003e-05 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:52.665397+0000", 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5550999999999994e-05 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.472 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51892 6.a e178/171 6:5225cf16:::benchmark_data_smithi017_350392_object51891:head v 178'3239, mlcod=178'3237)", 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.18465246900001, 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31651939499999998, 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:15:09.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256881+0000", 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:15:09.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256908+0000", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6789999999999999e-05 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256919+0000", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0925e-05 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256925+0000", 2023-09-23T15:15:09.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9009999999999999e-06 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256977+0000", 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1675999999999998e-05 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256998+0000", 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0988999999999998e-05 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573352+0000", 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31635466800000001 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573401+0000", 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9128999999999997e-05 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51896 6.5 6:ae721bbc:::benchmark_data_smithi017_350392_object51895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.18382640799999, 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32171442, 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.478 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51896 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257707+0000", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257767+0000", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0673e-05 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257781+0000", 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.343e-05 2023-09-23T15:15:09.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257786+0000", 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.395e-06 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257836+0000", 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9305000000000003e-05 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257953+0000", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117194 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258083+0000", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013044599999999999 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575282+0000", 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31719867800000001 2023-09-23T15:15:09.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575314+0000", 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2376000000000003e-05 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579321+0000", 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040069040000000004 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579377+0000", 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6243999999999999e-05 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579423+0000", 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5207999999999998e-05 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51897 6.3 6:cc65482f:::benchmark_data_smithi017_350392_object51896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:15:09.484 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.183734499, 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32175430300000002, 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51897 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257799+0000", 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:15:09.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257840+0000", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0457000000000002e-05 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257850+0000", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0448000000000001e-05 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257856+0000", 2023-09-23T15:15:09.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2780000000000001e-06 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257902+0000", 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6032999999999998e-05 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258014+0000", 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112636 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258155+0000", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141037 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575305+0000", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31715001799999998 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579465+0000", 2023-09-23T15:15:09.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041599280000000002 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579507+0000", 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1842999999999999e-05 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579554+0000", 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7205e-05 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.490 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51891 6.8 e178/171 6:1e19a070:::benchmark_data_smithi017_350392_object51890:head v 178'3166, mlcod=178'3163)", 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.18320117600001, 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32032912000000002, 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:15:09.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258417+0000", 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4161999999999998e-05 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573347+0000", 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.314930302 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573358+0000", 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0491e-05 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573497+0000", 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013895499999999999 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573523+0000", 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6495999999999998e-05 2023-09-23T15:15:09.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.578611+0000", 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050881320000000004 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.578663+0000", 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1143000000000001e-05 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51902 6.3 6:c005208d:::benchmark_data_smithi017_350392_object51901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.17796386800001, 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31611320199999998, 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51902 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.496 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263570+0000", 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263619+0000", 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9474000000000001e-05 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573336+0000", 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30971668099999999 2023-09-23T15:15:09.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573355+0000", 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9483999999999998e-05 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575330+0000", 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001974422 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575458+0000", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128398 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575588+0000", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012964199999999999 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579564+0000", 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039762349999999998 2023-09-23T15:15:09.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579593+0000", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.923e-05 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579625+0000", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2081e-05 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579684+0000", 2023-09-23T15:15:09.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8517000000000002e-05 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51912 6.a e178/171 6:5916b166:::benchmark_data_smithi017_350392_object51911:head v 178'3240, mlcod=178'3237)", 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 173.17296835600001, 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30900894899999998, 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.502 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268565+0000", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268593+0000", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7339999999999999e-05 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573405+0000", 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30481260799999998 2023-09-23T15:15:09.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573412+0000", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6950000000000003e-06 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573697+0000", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028508500000000003 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573716+0000", 2023-09-23T15:15:09.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9199999999999999e-05 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.577538+0000", 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038215749999999998 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.577575+0000", 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7370999999999998e-05 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78614 6.3 6:c41e9216:::benchmark_data_smithi017_350392_object78613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.86812254700001, 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.723727365, 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.507 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78614 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573410+0000", 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573445+0000", 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5234000000000002e-05 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573463+0000", 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7955e-05 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573472+0000", 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2269999999999993e-06 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574191+0000", 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071970900000000002 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574206+0000", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4419999999999999e-05 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.858632+0000", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28442586600000003 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.858645+0000", 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3179e-05 2023-09-23T15:15:09.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174166+0000", 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.315521355 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174350+0000", 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018351899999999999 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174561+0000", 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021169400000000001 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182906+0000", 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083446349999999996 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182925+0000", 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9171999999999999e-05 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.294629+0000", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11170437599999999 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.294669+0000", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9439e-05 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297139+0000", 2023-09-23T15:15:09.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024704890000000002 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78628 6.3 6:cf9578ef:::benchmark_data_smithi017_350392_object78627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.838663193, 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6971857319999999, 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.515 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78628 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:15:09.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602870+0000", 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602947+0000", 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7049e-05 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602962+0000", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.507e-05 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602968+0000", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3640000000000004e-06 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603027+0000", 2023-09-23T15:15:09.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9113000000000001e-05 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603035+0000", 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3730000000000004e-06 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863310+0000", 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26027560100000002 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863321+0000", 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0592e-05 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.179778+0000", 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316457467 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.179963+0000", 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184599 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.180154+0000", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019075400000000001 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186399+0000", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062453960000000003 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189483+0000", 2023-09-23T15:15:09.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030839719999999999 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297211+0000", 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10772744400000001 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297266+0000", 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5265e-05 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300057+0000", 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027911820000000001 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78631 6.3 6:cd655684:::benchmark_data_smithi017_350392_object78630:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.836964212, 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.697940681, 2023-09-23T15:15:09.523 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78631 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604569+0000", 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604681+0000", 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111625 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604696+0000", 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5099999999999999e-05 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604703+0000", 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2189999999999998e-06 2023-09-23T15:15:09.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604758+0000", 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4718e-05 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604770+0000", 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2207e-05 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863347+0000", 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.258576906 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863355+0000", 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0190000000000006e-06 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182958+0000", 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3196027080000001 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.183100+0000", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014283899999999999 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.183276+0000", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017582399999999999 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.192597+0000", 2023-09-23T15:15:09.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093209339999999995 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193414+0000", 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00081709899999999995 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300122+0000", 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106707381 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300186+0000", 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4689000000000004e-05 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.302511+0000", 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023246389999999999 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78639 6.3 6:ce866323:::benchmark_data_smithi017_350392_object78638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.828461804, 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6955158290000001, 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78639 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613071+0000", 2023-09-23T15:15:09.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613125+0000", 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3279999999999998e-05 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613143+0000", 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8697000000000001e-05 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613149+0000", 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1699999999999996e-06 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613197+0000", 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8810999999999997e-05 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613206+0000", 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4700000000000002e-06 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863411+0000", 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25020520600000001 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863419+0000", 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186425+0000", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3230066009999999 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186588+0000", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016276600000000001 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186770+0000", 2023-09-23T15:15:09.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000182265 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193425+0000", 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066544129999999996 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.195385+0000", 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001960074 2023-09-23T15:15:09.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.305864+0000", 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.110479492 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.305937+0000", 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2986999999999998e-05 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308589+0000", 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026511519999999999 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78642 6.3 6:cf556c25:::benchmark_data_smithi017_350392_object78641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 147.82398391999999, 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.693634818, 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.540 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78642 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617550+0000", 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617601+0000", 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1498000000000001e-05 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617612+0000", 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1119e-05 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617619+0000", 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.613e-06 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617661+0000", 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1548000000000003e-05 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617667+0000", 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1550000000000004e-06 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863446+0000", 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24577917099999999 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863456+0000", 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0068000000000001e-05 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189512+0000", 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3260563809999999 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189670+0000", 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015796199999999999 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189860+0000", 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190189 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193432+0000", 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035713189999999999 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.195427+0000", 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019955120000000001 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308655+0000", 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113228176 2023-09-23T15:15:09.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308719+0000", 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3638000000000003e-05 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.311185+0000", 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024663950000000001 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:111713 6.5 6:ab56db82:::benchmark_data_smithi017_350392_object11322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:13:39.815357+0000", 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 89.626177435000002, 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.214308627, 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.549 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 111713 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815357+0000", 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815357+0000", 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815356+0000", 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815361+0000", 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7099999999999998e-06 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815371+0000", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0402e-05 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815376+0000", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6e-06 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815423+0000", 2023-09-23T15:15:09.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6857999999999998e-05 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815545+0000", 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012222700000000001 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.815660+0000", 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115151 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.818422+0000", 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002762142 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.832777+0000", 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01435463 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029595+0000", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19681812800000001 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029640+0000", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5255e-05 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029666+0000", 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5374000000000001e-05 2023-09-23T15:15:09.555 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:111716 6.5 6:a3342a91:::benchmark_data_smithi017_350392_object11325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e198)", 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:13:39.822574+0000", 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 89.618960798000003, 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20723602399999999, 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.556 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 111716 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822574+0000", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822574+0000", 2023-09-23T15:15:09.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822573+0000", 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822579+0000", 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5719999999999999e-06 2023-09-23T15:15:09.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822592+0000", 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3118e-05 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822597+0000", 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1510000000000003e-06 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822645+0000", 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7647e-05 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822746+0000", 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010146500000000001 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.822831+0000", 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4525999999999997e-05 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.825401+0000", 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025697060000000002 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:39.835088+0000", 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096869360000000002 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029730+0000", 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.194642281 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029775+0000", 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5213000000000002e-05 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:40.029810+0000", 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4672000000000003e-05 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.562 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:112964 6.3 6:c3fdcbe4:::benchmark_data_smithi017_350392_object12573:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:13:41.973692+0000", 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 87.467842998999998, 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19680713, 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.563 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 112964 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973692+0000", 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973692+0000", 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973690+0000", 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973696+0000", 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0190000000000003e-06 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973709+0000", 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2993000000000001e-05 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973715+0000", 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3380000000000004e-06 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973762+0000", 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7129999999999998e-05 2023-09-23T15:15:09.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.973951+0000", 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018892 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.974166+0000", 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021511999999999999 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.977223+0000", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030568349999999999 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.994868+0000", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017645251000000001 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.168599+0000", 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17373107900000001 2023-09-23T15:15:09.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.168643+0000", 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4038000000000001e-05 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.170499+0000", 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018555150000000001 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:112971 6.5 6:a10ddc58:::benchmark_data_smithi017_350392_object12580:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e199)", 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:13:41.984572+0000", 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 87.456962720000007, 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19974852200000001, 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 112971 2023-09-23T15:15:09.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984572+0000", 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984572+0000", 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984571+0000", 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984577+0000", 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7360000000000004e-06 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984591+0000", 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3692e-05 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984596+0000", 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.152e-06 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984637+0000", 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0669e-05 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984732+0000", 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5484999999999998e-05 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.984817+0000", 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4884999999999998e-05 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.987014+0000", 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021969609999999999 2023-09-23T15:15:09.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:41.987711+0000", 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000696344 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.184253+0000", 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19654222299999999 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.184290+0000", 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6964e-05 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:13:42.184320+0000", 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0658999999999999e-05 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:123734 6.5 6:adb3b195:::benchmark_data_smithi017_350392_object23343:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e201)", 2023-09-23T15:15:09.576 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:01.495610+0000", 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 67.945924484000003, 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22381182399999999, 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 123734 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:15:09.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495610+0000", 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495610+0000", 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495610+0000", 2023-09-23T15:15:09.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495612+0000", 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7539999999999998e-06 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495619+0000", 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2949999999999999e-06 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495623+0000", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1049999999999997e-06 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495664+0000", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1165999999999999e-05 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495770+0000", 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106147 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.495889+0000", 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119089 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.498461+0000", 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025716480000000002 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.501883+0000", 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034217549999999998 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.719311+0000", 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.217428759 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.719375+0000", 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3627000000000006e-05 2023-09-23T15:15:09.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:01.719422+0000", 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6887e-05 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:15:09.583 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:15:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:09 smithi027 ceph-mon[195233]: pgmap v560: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:09 smithi027 ceph-mon[195233]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:09.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:09 smithi017 ceph-mon[215646]: pgmap v560: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:09.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:09 smithi017 ceph-mon[215646]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:09.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:09 smithi017 ceph-mon[222878]: pgmap v560: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 533 op/s; 23584/129753 objects degraded (18.176%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:09.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:09 smithi017 ceph-mon[222878]: from='client.55207 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:10.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:15:10.457 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":560,"stamp":"2023-09-23T15:15:08.388021+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2837420252,"num_objects":43251,"num_object_clones":0,"num_object_copies":129753,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23584,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43251,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":158582,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6178,"num_bytes_recovered":406362279,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123867,"ondisk_log_size":123867,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9488512,"kb_used_data":7920584,"kb_used_omap":30,"kb_used_meta":1567329,"kb_avail":646727552,"statfs":{"total":671965249536,"available":662249013248,"internally_reserved":0,"allocated":8110678016,"data_stored":8104970552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1604945496},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-350093312,"num_objects":-5342,"num_object_clones":0,"num_object_copies":-16026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-288,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5342,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":288,"num_bytes_recovered":18874368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003753"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":202,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535826+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:14:51.535826+0000","last_peered":"2023-09-23T15:14:51.535826+0000","last_clean":"2023-09-23T15:14:51.535826+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:14:51.535826+0000","last_undegraded":"2023-09-23T15:14:51.535826+0000","last_fullsized":"2023-09-23T15:14:51.535826+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550882+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:14:51.550882+0000","last_peered":"2023-09-23T15:14:51.550882+0000","last_clean":"2023-09-23T15:14:51.550882+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:14:51.550882+0000","last_undegraded":"2023-09-23T15:14:51.550882+0000","last_fullsized":"2023-09-23T15:14:51.550882+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":84,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559946+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:14:51.559946+0000","last_peered":"2023-09-23T15:14:51.559946+0000","last_clean":"2023-09-23T15:14:51.559946+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:14:51.559946+0000","last_undegraded":"2023-09-23T15:14:51.559946+0000","last_fullsized":"2023-09-23T15:14:51.559946+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":173,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546174+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:14:51.546174+0000","last_peered":"2023-09-23T15:14:51.546174+0000","last_clean":"2023-09-23T15:14:51.546174+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:14:51.546174+0000","last_undegraded":"2023-09-23T15:14:51.546174+0000","last_fullsized":"2023-09-23T15:14:51.546174+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":205,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.551641+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:14:51.551641+0000","last_peered":"2023-09-23T15:14:51.551641+0000","last_clean":"2023-09-23T15:14:51.551641+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:14:51.551641+0000","last_undegraded":"2023-09-23T15:14:51.551641+0000","last_fullsized":"2023-09-23T15:14:51.551641+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559920+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:14:51.559920+0000","last_peered":"2023-09-23T15:14:51.559920+0000","last_clean":"2023-09-23T15:14:51.559920+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:14:51.559920+0000","last_undegraded":"2023-09-23T15:14:51.559920+0000","last_fullsized":"2023-09-23T15:14:51.559920+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.548711+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:14:51.548711+0000","last_peered":"2023-09-23T15:14:51.548711+0000","last_clean":"2023-09-23T15:14:51.548711+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:14:51.548711+0000","last_undegraded":"2023-09-23T15:14:51.548711+0000","last_fullsized":"2023-09-23T15:14:51.548711+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":187,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.551716+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:14:51.551716+0000","last_peered":"2023-09-23T15:14:51.551716+0000","last_clean":"2023-09-23T15:14:51.551716+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:14:51.551716+0000","last_undegraded":"2023-09-23T15:14:51.551716+0000","last_fullsized":"2023-09-23T15:14:51.551716+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545768+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:14:51.545768+0000","last_peered":"2023-09-23T15:14:51.545768+0000","last_clean":"2023-09-23T15:14:51.545768+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:14:51.545768+0000","last_undegraded":"2023-09-23T15:14:51.545768+0000","last_fullsized":"2023-09-23T15:14:51.545768+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":198,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539467+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:14:51.539467+0000","last_peered":"2023-09-23T15:14:51.539467+0000","last_clean":"2023-09-23T15:14:51.539467+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:14:51.539467+0000","last_undegraded":"2023-09-23T15:14:51.539467+0000","last_fullsized":"2023-09-23T15:14:51.539467+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":130,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549763+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:14:51.549763+0000","last_peered":"2023-09-23T15:14:51.549763+0000","last_clean":"2023-09-23T15:14:51.549763+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:14:51.549763+0000","last_undegraded":"2023-09-23T15:14:51.549763+0000","last_fullsized":"2023-09-23T15:14:51.549763+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550805+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:14:51.550805+0000","last_peered":"2023-09-23T15:14:51.550805+0000","last_clean":"2023-09-23T15:14:51.550805+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:14:51.550805+0000","last_undegraded":"2023-09-23T15:14:51.550805+0000","last_fullsized":"2023-09-23T15:14:51.550805+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":80,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.221901+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:00.221901+0000","last_peered":"2023-09-23T15:15:00.221901+0000","last_clean":"2023-09-23T15:15:00.221901+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:00.221901+0000","last_undegraded":"2023-09-23T15:15:00.221901+0000","last_fullsized":"2023-09-23T15:15:00.221901+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":192,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536488+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:14:51.536488+0000","last_peered":"2023-09-23T15:14:51.536488+0000","last_clean":"2023-09-23T15:14:51.536488+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:14:51.536488+0000","last_undegraded":"2023-09-23T15:14:51.536488+0000","last_fullsized":"2023-09-23T15:14:51.536488+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536082+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:14:51.536082+0000","last_peered":"2023-09-23T15:14:51.536082+0000","last_clean":"2023-09-23T15:14:51.536082+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:14:51.536082+0000","last_undegraded":"2023-09-23T15:14:51.536082+0000","last_fullsized":"2023-09-23T15:14:51.536082+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550107+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:14:51.550107+0000","last_peered":"2023-09-23T15:14:51.550107+0000","last_clean":"2023-09-23T15:14:51.550107+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:14:51.550107+0000","last_undegraded":"2023-09-23T15:14:51.550107+0000","last_fullsized":"2023-09-23T15:14:51.550107+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":70,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536183+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:14:51.536183+0000","last_peered":"2023-09-23T15:14:51.536183+0000","last_clean":"2023-09-23T15:14:51.536183+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:14:51.536183+0000","last_undegraded":"2023-09-23T15:14:51.536183+0000","last_fullsized":"2023-09-23T15:14:51.536183+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":198,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.557400+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:14:51.557400+0000","last_peered":"2023-09-23T15:14:51.557400+0000","last_clean":"2023-09-23T15:14:51.557400+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:14:51.557400+0000","last_undegraded":"2023-09-23T15:14:51.557400+0000","last_fullsized":"2023-09-23T15:14:51.557400+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546232+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:14:51.546232+0000","last_peered":"2023-09-23T15:14:51.546232+0000","last_clean":"2023-09-23T15:14:51.546232+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:14:51.546232+0000","last_undegraded":"2023-09-23T15:14:51.546232+0000","last_fullsized":"2023-09-23T15:14:51.546232+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":172,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.548876+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:14:51.548876+0000","last_peered":"2023-09-23T15:14:51.548876+0000","last_clean":"2023-09-23T15:14:51.548876+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:14:51.548876+0000","last_undegraded":"2023-09-23T15:14:51.548876+0000","last_fullsized":"2023-09-23T15:14:51.548876+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539402+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:14:51.539402+0000","last_peered":"2023-09-23T15:14:51.539402+0000","last_clean":"2023-09-23T15:14:51.539402+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:14:51.539402+0000","last_undegraded":"2023-09-23T15:14:51.539402+0000","last_fullsized":"2023-09-23T15:14:51.539402+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.544848+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:14:51.544848+0000","last_peered":"2023-09-23T15:14:51.544848+0000","last_clean":"2023-09-23T15:14:51.544848+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:14:51.544848+0000","last_undegraded":"2023-09-23T15:14:51.544848+0000","last_fullsized":"2023-09-23T15:14:51.544848+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536023+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:14:51.536023+0000","last_peered":"2023-09-23T15:14:51.536023+0000","last_clean":"2023-09-23T15:14:51.536023+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:14:51.536023+0000","last_undegraded":"2023-09-23T15:14:51.536023+0000","last_fullsized":"2023-09-23T15:14:51.536023+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":161,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546247+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:14:51.546247+0000","last_peered":"2023-09-23T15:14:51.546247+0000","last_clean":"2023-09-23T15:14:51.546247+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:14:51.546247+0000","last_undegraded":"2023-09-23T15:14:51.546247+0000","last_fullsized":"2023-09-23T15:14:51.546247+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549175+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:14:51.549175+0000","last_peered":"2023-09-23T15:14:51.549175+0000","last_clean":"2023-09-23T15:14:51.549175+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:14:51.549175+0000","last_undegraded":"2023-09-23T15:14:51.549175+0000","last_fullsized":"2023-09-23T15:14:51.549175+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":174,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559156+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:14:51.559156+0000","last_peered":"2023-09-23T15:14:51.559156+0000","last_clean":"2023-09-23T15:14:51.559156+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:14:51.559156+0000","last_undegraded":"2023-09-23T15:14:51.559156+0000","last_fullsized":"2023-09-23T15:14:51.559156+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":198,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.558315+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:14:51.558315+0000","last_peered":"2023-09-23T15:14:51.558315+0000","last_clean":"2023-09-23T15:14:51.558315+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:14:51.558315+0000","last_undegraded":"2023-09-23T15:14:51.558315+0000","last_fullsized":"2023-09-23T15:14:51.558315+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"208'9767","reported_seq":9812,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.668415+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:15:04.668415+0000","last_peered":"2023-09-23T15:15:04.668415+0000","last_clean":"2023-09-23T15:15:04.668415+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:15:04.668415+0000","last_undegraded":"2023-09-23T15:15:04.668415+0000","last_fullsized":"2023-09-23T15:15:04.668415+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9767,"log_dups_size":0,"ondisk_log_size":9767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172163072,"num_objects":2627,"num_object_clones":0,"num_object_copies":7881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9767,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549244+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:14:51.549244+0000","last_peered":"2023-09-23T15:14:51.549244+0000","last_clean":"2023-09-23T15:14:51.549244+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:14:51.549244+0000","last_undegraded":"2023-09-23T15:14:51.549244+0000","last_fullsized":"2023-09-23T15:14:51.549244+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":82,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.537949+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:14:51.537949+0000","last_peered":"2023-09-23T15:14:51.537949+0000","last_clean":"2023-09-23T15:14:51.537949+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:14:51.537949+0000","last_undegraded":"2023-09-23T15:14:51.537949+0000","last_fullsized":"2023-09-23T15:14:51.537949+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.538983+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:14:51.538983+0000","last_peered":"2023-09-23T15:14:51.538983+0000","last_clean":"2023-09-23T15:14:51.538983+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:14:51.538983+0000","last_undegraded":"2023-09-23T15:14:51.538983+0000","last_fullsized":"2023-09-23T15:14:51.538983+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"208'9982","reported_seq":12474,"reported_epoch":208,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:04.613634+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:15:04.613634+0000","last_peered":"2023-09-23T15:15:04.613634+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:15:04.613634+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"178'3500","ondisk_log_start":"178'3500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6482,"log_dups_size":0,"ondisk_log_size":6482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175898624,"num_objects":2684,"num_object_clones":0,"num_object_copies":8052,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3779,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2684,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9982,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1219,"num_bytes_recovered":79888384,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1095}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":475,"reported_epoch":207,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:51.545536+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:14:51.545536+0000","last_peered":"2023-09-23T15:14:51.545536+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:14:51.545536+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":82,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539226+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:14:51.539226+0000","last_peered":"2023-09-23T15:14:51.539226+0000","last_clean":"2023-09-23T15:14:51.539226+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:14:51.539226+0000","last_undegraded":"2023-09-23T15:14:51.539226+0000","last_fullsized":"2023-09-23T15:14:51.539226+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":200,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535882+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:14:51.535882+0000","last_peered":"2023-09-23T15:14:51.535882+0000","last_clean":"2023-09-23T15:14:51.535882+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:14:51.535882+0000","last_undegraded":"2023-09-23T15:14:51.535882+0000","last_fullsized":"2023-09-23T15:14:51.535882+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"208'9695","reported_seq":9742,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:03.547302+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:15:03.547302+0000","last_peered":"2023-09-23T15:15:03.547302+0000","last_clean":"2023-09-23T15:15:03.547302+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:15:03.547302+0000","last_undegraded":"2023-09-23T15:15:03.547302+0000","last_fullsized":"2023-09-23T15:15:03.547302+0000","mapping_epoch":171,"log_start":"175'600","ondisk_log_start":"175'600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9095,"log_dups_size":0,"ondisk_log_size":9095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180027392,"num_objects":2747,"num_object_clones":0,"num_object_copies":8241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9695,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546425+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:14:51.546425+0000","last_peered":"2023-09-23T15:14:51.546425+0000","last_clean":"2023-09-23T15:14:51.546425+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:14:51.546425+0000","last_undegraded":"2023-09-23T15:14:51.546425+0000","last_fullsized":"2023-09-23T15:14:51.546425+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":68,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539373+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:14:51.539373+0000","last_peered":"2023-09-23T15:14:51.539373+0000","last_clean":"2023-09-23T15:14:51.539373+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:14:51.539373+0000","last_undegraded":"2023-09-23T15:14:51.539373+0000","last_fullsized":"2023-09-23T15:14:51.539373+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":196,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559814+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:14:51.559814+0000","last_peered":"2023-09-23T15:14:51.559814+0000","last_clean":"2023-09-23T15:14:51.559814+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:14:51.559814+0000","last_undegraded":"2023-09-23T15:14:51.559814+0000","last_fullsized":"2023-09-23T15:14:51.559814+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"208'9877","reported_seq":9924,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.419002+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:15:04.419002+0000","last_peered":"2023-09-23T15:15:04.419002+0000","last_clean":"2023-09-23T15:15:04.419002+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:15:04.419002+0000","last_undegraded":"2023-09-23T15:15:04.419002+0000","last_fullsized":"2023-09-23T15:15:04.419002+0000","mapping_epoch":171,"log_start":"180'4300","ondisk_log_start":"180'4300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5577,"log_dups_size":0,"ondisk_log_size":5577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182648832,"num_objects":2787,"num_object_clones":0,"num_object_copies":8361,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2787,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9877,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545947+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:14:51.545947+0000","last_peered":"2023-09-23T15:14:51.545947+0000","last_clean":"2023-09-23T15:14:51.545947+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:14:51.545947+0000","last_undegraded":"2023-09-23T15:14:51.545947+0000","last_fullsized":"2023-09-23T15:14:51.545947+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539260+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:51.539260+0000","last_peered":"2023-09-23T15:14:51.539260+0000","last_clean":"2023-09-23T15:14:51.539260+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:14:51.539260+0000","last_undegraded":"2023-09-23T15:14:51.539260+0000","last_fullsized":"2023-09-23T15:14:51.539260+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545084+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:14:51.545084+0000","last_peered":"2023-09-23T15:14:51.545084+0000","last_clean":"2023-09-23T15:14:51.545084+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:14:51.545084+0000","last_undegraded":"2023-09-23T15:14:51.545084+0000","last_fullsized":"2023-09-23T15:14:51.545084+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"208'10071","reported_seq":10122,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:04.270638+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:15:04.270638+0000","last_peered":"2023-09-23T15:15:04.270638+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:15:04.270638+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"177'2500","ondisk_log_start":"177'2500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7571,"log_dups_size":0,"ondisk_log_size":7571,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10071,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-2169}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550221+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:14:51.550221+0000","last_peered":"2023-09-23T15:14:51.550221+0000","last_clean":"2023-09-23T15:14:51.550221+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:14:51.550221+0000","last_undegraded":"2023-09-23T15:14:51.550221+0000","last_fullsized":"2023-09-23T15:14:51.550221+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":170,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.555624+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:14:51.555624+0000","last_peered":"2023-09-23T15:14:51.555624+0000","last_clean":"2023-09-23T15:14:51.555624+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:14:51.555624+0000","last_undegraded":"2023-09-23T15:14:51.555624+0000","last_fullsized":"2023-09-23T15:14:51.555624+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":166,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536561+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:14:51.536561+0000","last_peered":"2023-09-23T15:14:51.536561+0000","last_clean":"2023-09-23T15:14:51.536561+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:14:51.536561+0000","last_undegraded":"2023-09-23T15:14:51.536561+0000","last_fullsized":"2023-09-23T15:14:51.536561+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"208'9866","reported_seq":9913,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:03.441101+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:15:03.441101+0000","last_peered":"2023-09-23T15:15:03.441101+0000","last_clean":"2023-09-23T15:15:03.441101+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:15:03.441101+0000","last_undegraded":"2023-09-23T15:15:03.441101+0000","last_fullsized":"2023-09-23T15:15:03.441101+0000","mapping_epoch":171,"log_start":"175'700","ondisk_log_start":"175'700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9166,"log_dups_size":0,"ondisk_log_size":9166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177733632,"num_objects":2712,"num_object_clones":0,"num_object_copies":8136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2712,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9866,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":501,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546316+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:14:51.546316+0000","last_peered":"2023-09-23T15:14:51.546316+0000","last_clean":"2023-09-23T15:14:51.546316+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:14:51.546316+0000","last_undegraded":"2023-09-23T15:14:51.546316+0000","last_fullsized":"2023-09-23T15:14:51.546316+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559675+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:14:51.559675+0000","last_peered":"2023-09-23T15:14:51.559675+0000","last_clean":"2023-09-23T15:14:51.559675+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:14:51.559675+0000","last_undegraded":"2023-09-23T15:14:51.559675+0000","last_fullsized":"2023-09-23T15:14:51.559675+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":200,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536718+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:14:51.536718+0000","last_peered":"2023-09-23T15:14:51.536718+0000","last_clean":"2023-09-23T15:14:51.536718+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:14:51.536718+0000","last_undegraded":"2023-09-23T15:14:51.536718+0000","last_fullsized":"2023-09-23T15:14:51.536718+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"208'9902","reported_seq":9949,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.544486+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:15:04.544486+0000","last_peered":"2023-09-23T15:15:04.544486+0000","last_clean":"2023-09-23T15:15:04.544486+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:15:04.544486+0000","last_undegraded":"2023-09-23T15:15:04.544486+0000","last_fullsized":"2023-09-23T15:15:04.544486+0000","mapping_epoch":171,"log_start":"178'3500","ondisk_log_start":"178'3500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6402,"log_dups_size":0,"ondisk_log_size":6402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175374336,"num_objects":2676,"num_object_clones":0,"num_object_copies":8028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9902,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.544937+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:14:51.544937+0000","last_peered":"2023-09-23T15:14:51.544937+0000","last_clean":"2023-09-23T15:14:51.544937+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:14:51.544937+0000","last_undegraded":"2023-09-23T15:14:51.544937+0000","last_fullsized":"2023-09-23T15:14:51.544937+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":191,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535640+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:14:51.535640+0000","last_peered":"2023-09-23T15:14:51.535640+0000","last_clean":"2023-09-23T15:14:51.535640+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:14:51.535640+0000","last_undegraded":"2023-09-23T15:14:51.535640+0000","last_fullsized":"2023-09-23T15:14:51.535640+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539308+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:14:51.539308+0000","last_peered":"2023-09-23T15:14:51.539308+0000","last_clean":"2023-09-23T15:14:51.539308+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:14:51.539308+0000","last_undegraded":"2023-09-23T15:14:51.539308+0000","last_fullsized":"2023-09-23T15:14:51.539308+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"208'9853","reported_seq":19719,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.655647+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:15:04.655647+0000","last_peered":"2023-09-23T15:15:04.655647+0000","last_clean":"2023-09-23T15:15:04.655647+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:15:04.655647+0000","last_undegraded":"2023-09-23T15:15:04.655647+0000","last_fullsized":"2023-09-23T15:15:04.655647+0000","mapping_epoch":203,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9853,"log_dups_size":0,"ondisk_log_size":9853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175505432,"num_objects":2679,"num_object_clones":0,"num_object_copies":8037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2679,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9853,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549094+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:14:51.549094+0000","last_peered":"2023-09-23T15:14:51.549094+0000","last_clean":"2023-09-23T15:14:51.549094+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:14:51.549094+0000","last_undegraded":"2023-09-23T15:14:51.549094+0000","last_fullsized":"2023-09-23T15:14:51.549094+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":200,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:54.586511+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:14:54.586511+0000","last_peered":"2023-09-23T15:14:54.586511+0000","last_clean":"2023-09-23T15:14:54.586511+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:14:54.586511+0000","last_undegraded":"2023-09-23T15:14:54.586511+0000","last_fullsized":"2023-09-23T15:14:54.586511+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":85,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222073+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:00.222073+0000","last_peered":"2023-09-23T15:15:00.222073+0000","last_clean":"2023-09-23T15:15:00.222073+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:00.222073+0000","last_undegraded":"2023-09-23T15:15:00.222073+0000","last_fullsized":"2023-09-23T15:15:00.222073+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"208'9928","reported_seq":9975,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.389160+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:15:04.389160+0000","last_peered":"2023-09-23T15:15:04.389160+0000","last_clean":"2023-09-23T15:15:04.389160+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:15:04.389160+0000","last_undegraded":"2023-09-23T15:15:04.389160+0000","last_fullsized":"2023-09-23T15:15:04.389160+0000","mapping_epoch":171,"log_start":"180'4300","ondisk_log_start":"180'4300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5628,"log_dups_size":0,"ondisk_log_size":5628,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173015040,"num_objects":2640,"num_object_clones":0,"num_object_copies":7920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9928,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546513+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:14:51.546513+0000","last_peered":"2023-09-23T15:14:51.546513+0000","last_clean":"2023-09-23T15:14:51.546513+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:14:51.546513+0000","last_undegraded":"2023-09-23T15:14:51.546513+0000","last_fullsized":"2023-09-23T15:14:51.546513+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":384,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.566075+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:14:51.566075+0000","last_peered":"2023-09-23T15:14:51.566075+0000","last_clean":"2023-09-23T15:14:51.566075+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:14:51.566075+0000","last_undegraded":"2023-09-23T15:14:51.566075+0000","last_fullsized":"2023-09-23T15:14:51.566075+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":184,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539290+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:14:51.539290+0000","last_peered":"2023-09-23T15:14:51.539290+0000","last_clean":"2023-09-23T15:14:51.539290+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:14:51.539290+0000","last_undegraded":"2023-09-23T15:14:51.539290+0000","last_fullsized":"2023-09-23T15:14:51.539290+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":129,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.553723+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:14:51.553723+0000","last_peered":"2023-09-23T15:14:51.553723+0000","last_clean":"2023-09-23T15:14:51.553723+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:14:51.553723+0000","last_undegraded":"2023-09-23T15:14:51.553723+0000","last_fullsized":"2023-09-23T15:14:51.553723+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"208'9677","reported_seq":9724,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.569014+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:15:04.569014+0000","last_peered":"2023-09-23T15:15:04.569014+0000","last_clean":"2023-09-23T15:15:04.569014+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:15:04.569014+0000","last_undegraded":"2023-09-23T15:15:04.569014+0000","last_fullsized":"2023-09-23T15:15:04.569014+0000","mapping_epoch":171,"log_start":"178'3200","ondisk_log_start":"178'3200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6477,"log_dups_size":0,"ondisk_log_size":6477,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175439872,"num_objects":2677,"num_object_clones":0,"num_object_copies":8031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9677,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550186+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:14:51.550186+0000","last_peered":"2023-09-23T15:14:51.550186+0000","last_clean":"2023-09-23T15:14:51.550186+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:14:51.550186+0000","last_undegraded":"2023-09-23T15:14:51.550186+0000","last_fullsized":"2023-09-23T15:14:51.550186+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":726,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539267+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:14:51.539267+0000","last_peered":"2023-09-23T15:14:51.539267+0000","last_clean":"2023-09-23T15:14:51.539267+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:14:51.539267+0000","last_undegraded":"2023-09-23T15:14:51.539267+0000","last_fullsized":"2023-09-23T15:14:51.539267+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":167,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536005+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:14:51.536005+0000","last_peered":"2023-09-23T15:14:51.536005+0000","last_clean":"2023-09-23T15:14:51.536005+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:14:51.536005+0000","last_undegraded":"2023-09-23T15:14:51.536005+0000","last_fullsized":"2023-09-23T15:14:51.536005+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.565761+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:14:51.565761+0000","last_peered":"2023-09-23T15:14:51.565761+0000","last_clean":"2023-09-23T15:14:51.565761+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:14:51.565761+0000","last_undegraded":"2023-09-23T15:14:51.565761+0000","last_fullsized":"2023-09-23T15:14:51.565761+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"208'9897","reported_seq":9942,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.198307+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:15:04.198307+0000","last_peered":"2023-09-23T15:15:04.198307+0000","last_clean":"2023-09-23T15:15:04.198307+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:15:04.198307+0000","last_undegraded":"2023-09-23T15:15:04.198307+0000","last_fullsized":"2023-09-23T15:15:04.198307+0000","mapping_epoch":171,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5297,"log_dups_size":0,"ondisk_log_size":5297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178847744,"num_objects":2729,"num_object_clones":0,"num_object_copies":8187,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2729,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9897,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.548961+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:14:51.548961+0000","last_peered":"2023-09-23T15:14:51.548961+0000","last_clean":"2023-09-23T15:14:51.548961+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:14:51.548961+0000","last_undegraded":"2023-09-23T15:14:51.548961+0000","last_fullsized":"2023-09-23T15:14:51.548961+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":820,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559002+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:14:51.559002+0000","last_peered":"2023-09-23T15:14:51.559002+0000","last_clean":"2023-09-23T15:14:51.559002+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:14:51.559002+0000","last_undegraded":"2023-09-23T15:14:51.559002+0000","last_fullsized":"2023-09-23T15:14:51.559002+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":80,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.557567+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:14:51.557567+0000","last_peered":"2023-09-23T15:14:51.557567+0000","last_clean":"2023-09-23T15:14:51.557567+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:14:51.557567+0000","last_undegraded":"2023-09-23T15:14:51.557567+0000","last_fullsized":"2023-09-23T15:14:51.557567+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":104,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:56.182560+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:14:56.182560+0000","last_peered":"2023-09-23T15:14:56.182560+0000","last_clean":"2023-09-23T15:14:56.182560+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:14:56.182560+0000","last_undegraded":"2023-09-23T15:14:56.182560+0000","last_fullsized":"2023-09-23T15:14:56.182560+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"208'9733","reported_seq":9778,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.670463+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:15:04.670463+0000","last_peered":"2023-09-23T15:15:04.670463+0000","last_clean":"2023-09-23T15:15:04.670463+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:15:04.670463+0000","last_undegraded":"2023-09-23T15:15:04.670463+0000","last_fullsized":"2023-09-23T15:15:04.670463+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9733,"log_dups_size":0,"ondisk_log_size":9733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173473792,"num_objects":2647,"num_object_clones":0,"num_object_copies":7941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9733,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":495,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539088+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:14:51.539088+0000","last_peered":"2023-09-23T15:14:51.539088+0000","last_clean":"2023-09-23T15:14:51.539088+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:14:51.539088+0000","last_undegraded":"2023-09-23T15:14:51.539088+0000","last_fullsized":"2023-09-23T15:14:51.539088+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":496,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546461+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:14:51.546461+0000","last_peered":"2023-09-23T15:14:51.546461+0000","last_clean":"2023-09-23T15:14:51.546461+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:14:51.546461+0000","last_undegraded":"2023-09-23T15:14:51.546461+0000","last_fullsized":"2023-09-23T15:14:51.546461+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":122,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550466+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:14:51.550466+0000","last_peered":"2023-09-23T15:14:51.550466+0000","last_clean":"2023-09-23T15:14:51.550466+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:14:51.550466+0000","last_undegraded":"2023-09-23T15:14:51.550466+0000","last_fullsized":"2023-09-23T15:14:51.550466+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":158,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.560257+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:14:51.560257+0000","last_peered":"2023-09-23T15:14:51.560257+0000","last_clean":"2023-09-23T15:14:51.560257+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:14:51.560257+0000","last_undegraded":"2023-09-23T15:14:51.560257+0000","last_fullsized":"2023-09-23T15:14:51.560257+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"208'9951","reported_seq":9999,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:04.535996+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:15:04.535996+0000","last_peered":"2023-09-23T15:15:04.535996+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:15:04.535996+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"178'3500","ondisk_log_start":"178'3500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176488448,"num_objects":2693,"num_object_clones":0,"num_object_copies":8079,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2693,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9951,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-2259}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545013+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:14:51.545013+0000","last_peered":"2023-09-23T15:14:51.545013+0000","last_clean":"2023-09-23T15:14:51.545013+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:14:51.545013+0000","last_undegraded":"2023-09-23T15:14:51.545013+0000","last_fullsized":"2023-09-23T15:14:51.545013+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":291,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.557311+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:14:51.557311+0000","last_peered":"2023-09-23T15:14:51.557311+0000","last_clean":"2023-09-23T15:14:51.557311+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:14:51.557311+0000","last_undegraded":"2023-09-23T15:14:51.557311+0000","last_fullsized":"2023-09-23T15:14:51.557311+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":153,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246495+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:00.246495+0000","last_peered":"2023-09-23T15:15:00.246495+0000","last_clean":"2023-09-23T15:15:00.246495+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:00.246495+0000","last_undegraded":"2023-09-23T15:15:00.246495+0000","last_fullsized":"2023-09-23T15:15:00.246495+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":182,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546015+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:14:51.546015+0000","last_peered":"2023-09-23T15:14:51.546015+0000","last_clean":"2023-09-23T15:14:51.546015+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:14:51.546015+0000","last_undegraded":"2023-09-23T15:14:51.546015+0000","last_fullsized":"2023-09-23T15:14:51.546015+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"208'9795","reported_seq":9842,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:03.546145+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:15:03.546145+0000","last_peered":"2023-09-23T15:15:03.546145+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:15:03.546145+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"175'700","ondisk_log_start":"175'700","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9095,"log_dups_size":0,"ondisk_log_size":9095,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177799168,"num_objects":2713,"num_object_clones":0,"num_object_copies":8139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9795,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-2161}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546324+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:14:51.546324+0000","last_peered":"2023-09-23T15:14:51.546324+0000","last_clean":"2023-09-23T15:14:51.546324+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:14:51.546324+0000","last_undegraded":"2023-09-23T15:14:51.546324+0000","last_fullsized":"2023-09-23T15:14:51.546324+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":813,"reported_epoch":207,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:14:51.549012+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:14:51.549012+0000","last_peered":"2023-09-23T15:14:51.549012+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:14:51.549012+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":80,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.565950+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:14:51.565950+0000","last_peered":"2023-09-23T15:14:51.565950+0000","last_clean":"2023-09-23T15:14:51.565950+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:14:51.565950+0000","last_undegraded":"2023-09-23T15:14:51.565950+0000","last_fullsized":"2023-09-23T15:14:51.565950+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":158,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244115+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:00.244115+0000","last_peered":"2023-09-23T15:15:00.244115+0000","last_clean":"2023-09-23T15:15:00.244115+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:00.244115+0000","last_undegraded":"2023-09-23T15:15:00.244115+0000","last_fullsized":"2023-09-23T15:15:00.244115+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"208'9617","reported_seq":9662,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.242715+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:15:04.242715+0000","last_peered":"2023-09-23T15:15:04.242715+0000","last_clean":"2023-09-23T15:15:04.242715+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:15:04.242715+0000","last_undegraded":"2023-09-23T15:15:04.242715+0000","last_fullsized":"2023-09-23T15:15:04.242715+0000","mapping_epoch":171,"log_start":"176'2100","ondisk_log_start":"176'2100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7517,"log_dups_size":0,"ondisk_log_size":7517,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175964160,"num_objects":2685,"num_object_clones":0,"num_object_copies":8055,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9617,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549161+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:14:51.549161+0000","last_peered":"2023-09-23T15:14:51.549161+0000","last_clean":"2023-09-23T15:14:51.549161+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:14:51.549161+0000","last_undegraded":"2023-09-23T15:14:51.549161+0000","last_fullsized":"2023-09-23T15:14:51.549161+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":184,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:54.596341+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:14:54.596341+0000","last_peered":"2023-09-23T15:14:54.596341+0000","last_clean":"2023-09-23T15:14:54.596341+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:14:54.596341+0000","last_undegraded":"2023-09-23T15:14:54.596341+0000","last_fullsized":"2023-09-23T15:14:54.596341+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":668,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:56.182540+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:14:56.182540+0000","last_peered":"2023-09-23T15:14:56.182540+0000","last_clean":"2023-09-23T15:14:56.182540+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:14:56.182540+0000","last_undegraded":"2023-09-23T15:14:56.182540+0000","last_fullsized":"2023-09-23T15:14:56.182540+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":188,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.551773+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:14:51.551773+0000","last_peered":"2023-09-23T15:14:51.551773+0000","last_clean":"2023-09-23T15:14:51.551773+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:14:51.551773+0000","last_undegraded":"2023-09-23T15:14:51.551773+0000","last_fullsized":"2023-09-23T15:14:51.551773+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"208'9943","reported_seq":9995,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:04.102950+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:15:04.102950+0000","last_peered":"2023-09-23T15:15:04.102950+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:15:04.102950+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5343,"log_dups_size":0,"ondisk_log_size":5343,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176095232,"num_objects":2687,"num_object_clones":0,"num_object_copies":8061,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2687,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9943,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-2279}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":484,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545691+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:14:51.545691+0000","last_peered":"2023-09-23T15:14:51.545691+0000","last_clean":"2023-09-23T15:14:51.545691+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:14:51.545691+0000","last_undegraded":"2023-09-23T15:14:51.545691+0000","last_fullsized":"2023-09-23T15:14:51.545691+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":684,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.559777+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:14:51.559777+0000","last_peered":"2023-09-23T15:14:51.559777+0000","last_clean":"2023-09-23T15:14:51.559777+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:14:51.559777+0000","last_undegraded":"2023-09-23T15:14:51.559777+0000","last_fullsized":"2023-09-23T15:14:51.559777+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":170,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.546095+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:14:51.546095+0000","last_peered":"2023-09-23T15:14:51.546095+0000","last_clean":"2023-09-23T15:14:51.546095+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:14:51.546095+0000","last_undegraded":"2023-09-23T15:14:51.546095+0000","last_fullsized":"2023-09-23T15:14:51.546095+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":149,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535802+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:14:51.535802+0000","last_peered":"2023-09-23T15:14:51.535802+0000","last_clean":"2023-09-23T15:14:51.535802+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:14:51.535802+0000","last_undegraded":"2023-09-23T15:14:51.535802+0000","last_fullsized":"2023-09-23T15:14:51.535802+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":160,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.545228+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:14:51.545228+0000","last_peered":"2023-09-23T15:14:51.545228+0000","last_clean":"2023-09-23T15:14:51.545228+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:14:51.545228+0000","last_undegraded":"2023-09-23T15:14:51.545228+0000","last_fullsized":"2023-09-23T15:14:51.545228+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":207,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.556922+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:14:51.556922+0000","last_peered":"2023-09-23T15:14:51.556922+0000","last_clean":"2023-09-23T15:14:51.556922+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:14:51.556922+0000","last_undegraded":"2023-09-23T15:14:51.556922+0000","last_fullsized":"2023-09-23T15:14:51.556922+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":186,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539426+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:14:51.539426+0000","last_peered":"2023-09-23T15:14:51.539426+0000","last_clean":"2023-09-23T15:14:51.539426+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:14:51.539426+0000","last_undegraded":"2023-09-23T15:14:51.539426+0000","last_fullsized":"2023-09-23T15:14:51.539426+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":200,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.536259+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:14:51.536259+0000","last_peered":"2023-09-23T15:14:51.536259+0000","last_clean":"2023-09-23T15:14:51.536259+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:14:51.536259+0000","last_undegraded":"2023-09-23T15:14:51.536259+0000","last_fullsized":"2023-09-23T15:14:51.536259+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":152,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.539204+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:14:51.539204+0000","last_peered":"2023-09-23T15:14:51.539204+0000","last_clean":"2023-09-23T15:14:51.539204+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:14:51.539204+0000","last_undegraded":"2023-09-23T15:14:51.539204+0000","last_fullsized":"2023-09-23T15:14:51.539204+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":183,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535740+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:14:51.535740+0000","last_peered":"2023-09-23T15:14:51.535740+0000","last_clean":"2023-09-23T15:14:51.535740+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:14:51.535740+0000","last_undegraded":"2023-09-23T15:14:51.535740+0000","last_fullsized":"2023-09-23T15:14:51.535740+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":179,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.550962+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:14:51.550962+0000","last_peered":"2023-09-23T15:14:51.550962+0000","last_clean":"2023-09-23T15:14:51.550962+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:14:51.550962+0000","last_undegraded":"2023-09-23T15:14:51.550962+0000","last_fullsized":"2023-09-23T15:14:51.550962+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":193,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.549722+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:14:51.549722+0000","last_peered":"2023-09-23T15:14:51.549722+0000","last_clean":"2023-09-23T15:14:51.549722+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:14:51.549722+0000","last_undegraded":"2023-09-23T15:14:51.549722+0000","last_fullsized":"2023-09-23T15:14:51.549722+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":156,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245739+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:00.245739+0000","last_peered":"2023-09-23T15:15:00.245739+0000","last_clean":"2023-09-23T15:15:00.245739+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:00.245739+0000","last_undegraded":"2023-09-23T15:15:00.245739+0000","last_fullsized":"2023-09-23T15:15:00.245739+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":176,"reported_epoch":207,"state":"active+clean","last_fresh":"2023-09-23T15:14:51.535490+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:14:51.535490+0000","last_peered":"2023-09-23T15:14:51.535490+0000","last_clean":"2023-09-23T15:14:51.535490+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:14:51.535490+0000","last_undegraded":"2023-09-23T15:14:51.535490+0000","last_fullsized":"2023-09-23T15:14:51.535490+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2832662552,"num_objects":43224,"num_object_clones":0,"num_object_copies":129672,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23581,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43224,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157554,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6119,"num_bytes_recovered":401014784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8055238656,"data_stored":8055226440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119454,"ondisk_log_size":119454,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201524,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463160,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1531620,"kb_used_data":1384156,"kb_used_omap":0,"kb_used_meta":147456,"kb_avail":92213532,"statfs":{"total":95995035648,"available":94426656768,"internally_reserved":0,"allocated":1417375744,"data_stored":1416683413,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":150994944},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.617,"15min":2.6779999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":6.367,"5min":41.491,"15min":108.39400000000001},"last":1.675},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.8130000000000002,"15min":2.8180000000000001},"min":{"1min":0.29299999999999998,"5min":0.223,"15min":0.223},"max":{"1min":12.634,"5min":43.151000000000003,"15min":108.452},"last":3.9569999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":4.5350000000000001,"5min":3.6030000000000002,"15min":3.448},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":43.377000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.95299999999999996},{"interface":"front","average":{"1min":7.2919999999999998,"5min":4.3440000000000003,"15min":3.8519999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":42.704000000000001,"5min":42.704000000000001,"15min":42.704000000000001},"last":1.1599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":13.1,"5min":6.4619999999999997,"15min":5.702},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":112.352,"5min":112.352,"15min":180.374},"last":1.2989999999999999},{"interface":"front","average":{"1min":13.564,"5min":6.4489999999999998,"15min":5.7329999999999997},"min":{"1min":0.45200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":117.667,"5min":117.667,"15min":181.685},"last":4.1600000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":12.955,"5min":4.3739999999999997,"15min":1.992},"min":{"1min":0.441,"5min":0.249,"15min":0.249},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":3.1539999999999999},{"interface":"front","average":{"1min":12.819000000000001,"5min":4.4420000000000002,"15min":1.982},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":80.977999999999994,"5min":80.977999999999994,"15min":80.977999999999994},"last":23.373999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":1.8999999999999999,"15min":1.0329999999999999},"min":{"1min":0.26300000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5119999999999996,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.998,"15min":1.0980000000000001},"min":{"1min":0.28699999999999998,"5min":0.26300000000000001,"15min":0.25600000000000001},"max":{"1min":3.5760000000000001,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.41199999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:14:08 2023","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":2.8399999999999999,"15min":1.389},"min":{"1min":0.32000000000000001,"5min":0.307,"15min":0.28100000000000003},"max":{"1min":5.5880000000000001,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.434},{"interface":"front","average":{"1min":0.61399999999999999,"5min":2.9470000000000001,"15min":1.431},"min":{"1min":0.33200000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":3.4550000000000001,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.53800000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.381,"15min":1},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":7.5910000000000002,"15min":17},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3939999999999999,"15min":0.97499999999999998},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":3.754,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.59399999999999997}]}]},{"osd":1,"up_from":183,"seq":785979015198,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761460,"kb_used_data":616876,"kb_used_omap":6,"kb_used_meta":144505,"kb_avail":92983692,"statfs":{"total":95995035648,"available":95215300608,"internally_reserved":0,"allocated":631681024,"data_stored":630688039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":147973195},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":3.6080000000000001,"15min":3.738},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.0839999999999996,"5min":16.626000000000001,"15min":16.626000000000001},"last":1.4390000000000001},{"interface":"front","average":{"1min":3.5609999999999999,"5min":4.4000000000000004,"15min":4.54},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.558,"5min":16.131,"15min":16.131},"last":14.957000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0229999999999997,"5min":4.3070000000000004,"15min":4.1870000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.757999999999999,"5min":13.843,"15min":13.843},"last":2.3519999999999999},{"interface":"front","average":{"1min":4.3380000000000001,"5min":4.2460000000000004,"15min":4.2300000000000004},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9539999999999997,"5min":9.6270000000000007,"15min":10.406000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.27,"5min":123.426,"15min":123.426},"last":0.89700000000000002},{"interface":"front","average":{"1min":3.5499999999999998,"5min":10.762,"15min":11.964},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1010000000000009,"5min":123.32899999999999,"15min":123.32899999999999},"last":14.82}]},{"osd":4,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.7090000000000001,"15min":2.7000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.7199999999999998,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":2.226},{"interface":"front","average":{"1min":3.129,"5min":3.472,"15min":3.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.779,"5min":16.779,"15min":16.779},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":3.419,"15min":3.6589999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8259999999999996,"5min":16.166,"15min":16.166},"last":14.898999999999999},{"interface":"front","average":{"1min":3.0190000000000001,"5min":2.9209999999999998,"15min":2.9049999999999998},"min":{"1min":0.35399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7640000000000002,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":2.27}]},{"osd":6,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":3.488,"15min":3.702},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.8010000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":14.867000000000001},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.8050000000000002,"15min":2.7669999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.7930000000000001,"5min":7.7930000000000001,"15min":7.7930000000000001},"last":2.2949999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.1749999999999998,"15min":3.1629999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.8179999999999996,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":2.3170000000000002},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2909999999999999,"15min":3.3330000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.47399999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893247,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":974344,"kb_used_data":698816,"kb_used_omap":8,"kb_used_meta":275511,"kb_avail":92770808,"statfs":{"total":95995035648,"available":94997307392,"internally_reserved":0,"allocated":715587584,"data_stored":714691837,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":282123860},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.875,"15min":4.1529999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.5099999999999998,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.249},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.274,"15min":4.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4249999999999998,"5min":31.378,"15min":31.378},"last":6.2720000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":15.058999999999999,"5min":7.9790000000000001,"15min":3.3519999999999999},"min":{"1min":0.311,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":93.873999999999995,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":13.999000000000001,"5min":6.9569999999999999,"15min":2.891},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":94.652000000000001,"5min":122.071,"15min":122.071},"last":4.4690000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:14:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8159999999999998,"5min":5.5419999999999998,"15min":2.3420000000000001},"min":{"1min":0.35299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.643000000000001,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":4.9729999999999999,"5min":4.6059999999999999,"15min":2.3490000000000002},"min":{"1min":0.40699999999999997,"5min":0.252,"15min":0.252},"max":{"1min":25.632000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":0.32200000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.9319999999999999,"15min":1.595},"min":{"1min":0.46200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0499999999999998,"5min":31.341999999999999,"15min":31.341999999999999},"last":3.0379999999999998},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.0150000000000001,"15min":1.5980000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9809999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.9420000000000002,"15min":1.6479999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2370000000000001,"5min":109.369,"15min":109.369},"last":0.627},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.532,"15min":1.8049999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.1260000000000003,"5min":109.387,"15min":109.387},"last":2.9100000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.9590000000000001,"15min":1.6319999999999999},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.032,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.171,"5min":2.8359999999999999,"15min":1.5580000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2910000000000004,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.0870000000000002,"15min":1.6759999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2640000000000002,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.4199999999999999,"15min":1.4890000000000001},"min":{"1min":0.497,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.093,"5min":31.225999999999999,"15min":31.225999999999999},"last":2.996}]}]},{"osd":2,"up_from":105,"seq":450971566194,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1901324,"kb_used_data":1586308,"kb_used_omap":11,"kb_used_meta":314676,"kb_avail":91843828,"statfs":{"total":95995035648,"available":94048079872,"internally_reserved":0,"allocated":1624379392,"data_stored":1623483606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":322229008},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":10.598000000000001,"5min":4.6280000000000001,"15min":1.9299999999999999},"min":{"1min":0.215,"5min":0.184,"15min":0.184},"max":{"1min":87.132999999999996,"5min":87.132999999999996,"15min":87.132999999999996},"last":12.840999999999999},{"interface":"front","average":{"1min":7.6420000000000003,"5min":3.6579999999999999,"15min":1.595},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":84.820999999999998,"5min":84.820999999999998,"15min":84.820999999999998},"last":1.2470000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:14:47 2023","interfaces":[{"interface":"back","average":{"1min":9.2129999999999992,"5min":5.8739999999999997,"15min":5.3179999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":84.641000000000005,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.51300000000000001},{"interface":"front","average":{"1min":9.2040000000000006,"5min":3.8799999999999999,"15min":2.992},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":84.751999999999995,"5min":84.751999999999995,"15min":84.751999999999995},"last":1.3819999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:14:27 2023","interfaces":[{"interface":"back","average":{"1min":10.016,"5min":3.4990000000000001,"15min":1.748},"min":{"1min":0.44700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":87.108000000000004,"5min":87.108000000000004,"15min":87.108000000000004},"last":6.1360000000000001},{"interface":"front","average":{"1min":9.4209999999999994,"5min":3.3980000000000001,"15min":1.754},"min":{"1min":0.59599999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":84.739000000000004,"5min":84.739000000000004,"15min":84.739000000000004},"last":4.4749999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.54,"5min":3.6659999999999999,"15min":1.7649999999999999},"min":{"1min":0.42299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":84.608000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.56499999999999995},{"interface":"front","average":{"1min":8.8379999999999992,"5min":4.8460000000000001,"15min":2.254},"min":{"1min":0.41099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":6.4640000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6260000000000003,"5min":3.6459999999999999,"15min":1.8779999999999999},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":84.498000000000005,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.55000000000000004},{"interface":"front","average":{"1min":6.4429999999999996,"5min":3.5600000000000001,"15min":1.825},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":84.664000000000001,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9589999999999996,"5min":3.9089999999999998,"15min":1.9490000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":84.715999999999994,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.64900000000000002},{"interface":"front","average":{"1min":9.0410000000000004,"5min":3.847,"15min":1.9430000000000001},"min":{"1min":0.44,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":87.055000000000007,"5min":87.055000000000007,"15min":87.055000000000007},"last":6.4119999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8410000000000002,"5min":3.2010000000000001,"15min":1.7350000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":84.596000000000004,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.57799999999999996},{"interface":"front","average":{"1min":6.7770000000000001,"5min":3.274,"15min":1.7210000000000001},"min":{"1min":0.46999999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":84.783000000000001,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.59999999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402668,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2189980,"kb_used_data":1902548,"kb_used_omap":3,"kb_used_meta":287292,"kb_avail":91555172,"statfs":{"total":95995035648,"available":93752496128,"internally_reserved":0,"allocated":1948209152,"data_stored":1947352070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":294187177},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":4.0629999999999997,"5min":3.0760000000000001,"15min":1.764},"min":{"1min":0.36499999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":25.111999999999998,"5min":61.744,"15min":79.611000000000004},"last":0.72199999999999998},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.0870000000000002,"15min":1.788},"min":{"1min":0.29599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.6270000000000007,"5min":61.686,"15min":79.656000000000006},"last":0.84199999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":6.2789999999999999,"5min":4.9240000000000004,"15min":4.6980000000000004},"min":{"1min":0.34200000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.873999999999999,"5min":28.315000000000001,"15min":28.315000000000001},"last":3.7909999999999999},{"interface":"front","average":{"1min":4.7350000000000003,"5min":4.8230000000000004,"15min":4.8380000000000001},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.395,"5min":14.427,"15min":14.427},"last":0.78000000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":5.4690000000000003,"5min":5.5890000000000004,"15min":3.2749999999999999},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":25.187000000000001,"5min":109.077,"15min":109.077},"last":0.751},{"interface":"front","average":{"1min":4.4580000000000002,"5min":6.0449999999999999,"15min":3.4279999999999999},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.873000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":1.7170000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.9910000000000001,"15min":1.1299999999999999},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4169999999999998,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":0.73199999999999998},{"interface":"front","average":{"1min":2.802,"5min":2.1509999999999998,"15min":1.3320000000000001},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.632999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":1.2629999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.8170000000000002,"15min":1.46},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":6.3869999999999996,"5min":56.960999999999999,"15min":56.960999999999999},"last":0.76300000000000001},{"interface":"front","average":{"1min":2.839,"5min":3.0630000000000002,"15min":1.675},"min":{"1min":0.36399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":10.738,"5min":56.981999999999999,"15min":56.981999999999999},"last":1.206}]},{"osd":6,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.355,"15min":1.3260000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.3419999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":0.79900000000000004},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.6219999999999999,"15min":1.5780000000000001},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":10.561999999999999,"5min":19.355,"15min":19.355},"last":1.2390000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.726,"15min":1.585},"min":{"1min":0.42599999999999999,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":10.603,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.284},{"interface":"front","average":{"1min":2.702,"5min":2.113,"15min":1.2050000000000001},"min":{"1min":0.437,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.351000000000001,"5min":14.351000000000001,"15min":14.351000000000001},"last":0.68700000000000006}]}]},{"osd":4,"up_from":31,"seq":133143986433,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064032,"kb_used_data":874072,"kb_used_omap":0,"kb_used_meta":189952,"kb_avail":92681120,"statfs":{"total":95995035648,"available":94905466880,"internally_reserved":0,"allocated":895049728,"data_stored":894360972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":194510848},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":3.375,"15min":2.5289999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":65.988,"5min":66.605999999999995,"15min":122.69199999999999},"last":2.694},{"interface":"front","average":{"1min":5.4029999999999996,"5min":3.3639999999999999,"15min":2.6309999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":65.396000000000001,"5min":66.384,"15min":122.745},"last":0.437}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":4.2539999999999996,"15min":4.4500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.783999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.5699999999999998,"5min":4.7750000000000004,"15min":4.976},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":28.815000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":4.0640000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.5140000000000002,"15min":2.883},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.670999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":4.8230000000000004},{"interface":"front","average":{"1min":3.0369999999999999,"5min":4.984,"15min":3.1829999999999998},"min":{"1min":0.41399999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":17.756,"5min":147.88900000000001,"15min":147.88900000000001},"last":3.5720000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":4.8849999999999998,"5min":8.2460000000000004,"15min":3.3100000000000001},"min":{"1min":0.32400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":38.685000000000002,"5min":163.97499999999999,"15min":163.97499999999999},"last":2.9630000000000001},{"interface":"front","average":{"1min":5.5129999999999999,"5min":8.0139999999999993,"15min":3.149},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":38.767000000000003,"5min":164.447,"15min":164.447},"last":1.6579999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.452,"15min":1.1950000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.1479999999999999,"5min":14.073,"15min":28.891999999999999},"last":3.5840000000000001},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.208,"15min":1.0600000000000001},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.577,"5min":8.9920000000000009,"15min":32.100999999999999},"last":1.696}]},{"osd":6,"last update":"Sat Sep 23 15:14:22 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":3.6800000000000002,"15min":1.7350000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3319999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":2.734},{"interface":"front","average":{"1min":0.69099999999999995,"5min":3.323,"15min":1.6040000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.5179999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":1.8}]},{"osd":7,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.502,"15min":1.1319999999999999},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":1.6419999999999999,"5min":18.23,"15min":18.23},"last":0.505},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.48,"15min":1.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":2.214,"5min":14.044,"15min":14.653},"last":6.1399999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724796,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1065752,"kb_used_data":857808,"kb_used_omap":0,"kb_used_meta":207936,"kb_avail":92679400,"statfs":{"total":95995035648,"available":94903705600,"internally_reserved":0,"allocated":878395392,"data_stored":877710615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":212926464},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1829999999999998,"15min":1.292},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.9530000000000003,"5min":17.044,"15min":24.962},"last":0.27000000000000002},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.9039999999999999,"15min":1.226},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":6.2549999999999999,"5min":18.814,"15min":25.317},"last":11.343999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5030000000000001,"5min":2.2389999999999999,"15min":1.861},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.082999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.9510000000000001,"5min":2.4049999999999998,"15min":2.3140000000000001},"min":{"1min":0.39200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":17.963999999999999,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":4.7460000000000004,"15min":2.2989999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":33.68,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.46400000000000002},{"interface":"front","average":{"1min":4.0510000000000002,"5min":4.8639999999999999,"15min":2.2930000000000001},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":33.707999999999998,"5min":97.305999999999997,"15min":97.305999999999997},"last":0.52100000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":5.4550000000000001,"5min":3.7829999999999999,"15min":1.8360000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":32.305999999999997,"5min":53.088999999999999,"15min":53.088999999999999},"last":1.944},{"interface":"front","average":{"1min":5.0970000000000004,"5min":3.5249999999999999,"15min":1.734},"min":{"1min":0.46400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":24.052,"5min":53.764000000000003,"15min":53.764000000000003},"last":5.7300000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":2.1480000000000001,"15min":1.1639999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.369,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.2349999999999999,"15min":1.238},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.88200000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.494,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":11.076000000000001,"15min":15.196},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.3859999999999999,"15min":1.012},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":1.367,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.8580000000000001,"15min":1.228},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":1.331,"5min":11.353999999999999,"15min":15.301},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.9119999999999999,"15min":1.212},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.8460000000000001,"5min":15.762,"15min":15.762},"last":0.66000000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":704516096,"data_stored":704512024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":622727168,"data_stored":622723096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1616838656,"data_stored":1616838656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1938423808,"data_stored":1938423808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":888274944,"data_stored":888274944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":872615936,"data_stored":872611864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1411842048,"data_stored":1411842048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:15:10.483 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:10.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:10.492 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:10.527 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:15:10.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:10.648 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:10 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:10.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:10 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:10.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:10 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:10.787 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:10.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:10.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:10.891 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:10.936 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:11.073 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:11.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:11.078 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:11.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:11.226 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:11.260 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:11.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: pgmap v561: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:11.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:11.684 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: pgmap v561: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:11.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: pgmap v561: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1638381347' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: osdmap e209: 8 total, 8 up, 7 in 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: failed to encode map e209 with expected crc 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:11.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:12.246 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:12.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:12.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:12.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:12.392 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:12.559 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:12.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:12.564 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:12.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:12.753 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:12.790 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23444/121782 objects degraded (19.251%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:12.790 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23444/121782 objects degraded (19.251%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:12.822 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23444/121782 objects degraded (19.251%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:12.907 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:12.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:12.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:13.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:13.065 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:13.070 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:15:13.070 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:15:13.070 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:15:13.070 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:15:13.074 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:13.123 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:13.203 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:13.243 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:13.285 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:13.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:13.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:13.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:13.377 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:13.410 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:13.436 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.436 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:15:13.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:13.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:15:13.550 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:13.709 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:13.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:13.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:13.860 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:15:13.860 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:13.935 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:13.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:13.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:14.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:13 smithi027 ceph-mon[195233]: pgmap v563: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2460622996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:14.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[215646]: pgmap v563: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:14.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2460622996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:14.119 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:14.121 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[222878]: pgmap v563: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 23444/121782 objects degraded (19.251%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:14.122 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2460622996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:14.122 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:14.123 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:14.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:15:14.337 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:15.074 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:15.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:15.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:15.182 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:15.313 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:15.313 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:15.420 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:15.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:15.472 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:15.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:15.586 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:15.708 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:15.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:15.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:15.847 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:15.958 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:15.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:15.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:16.046 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:15:16.046 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:15:16.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:15 smithi017 ceph-mon[222878]: pgmap v564: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 23444/118266 objects degraded (19.823%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:16.048 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:15 smithi017 ceph-mon[215646]: pgmap v564: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 23444/118266 objects degraded (19.823%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:16.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:16.071 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:15 smithi027 ceph-mon[195233]: pgmap v564: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 534 op/s; 23444/118266 objects degraded (19.823%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:16.093 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:16.256 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:16.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:16.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:16.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:16.452 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:16.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:15:16.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:15:17.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:17 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23444/118266 objects degraded (19.823%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23444/118266 objects degraded (19.823%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23444/118266 objects degraded (19.823%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:17.302 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:15:17.665 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:17.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:17.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:17.815 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:17.884 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:17.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:17.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:17.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:18.015 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:18.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:18.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:18.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:18.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:18.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[215646]: pgmap v565: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:18.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:15:18.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:15:18.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:18.257 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:18] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T15:15:18.259 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[222878]: pgmap v565: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:18.259 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:15:18.259 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:15:18.259 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:18.267 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:18 smithi027 ceph-mon[195233]: pgmap v565: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:15:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:15:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:18.471 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:18.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:18.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:18.595 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:18.625 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:18.683 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:18.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:18.748 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:18.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:18.897 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:18.989 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:15:18.989 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:15:19.129 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:19.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:19.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:19.175 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:19.233 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:15:19.240 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:19.305 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:19.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:15:20.101 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:15:20.282 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:20.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:20.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:20.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:20 smithi017 ceph-mon[215646]: pgmap v566: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:20.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:20 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:20.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:20 smithi017 ceph-mon[222878]: pgmap v566: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:20.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:20 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:20.473 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:20.589 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:20 smithi027 ceph-mon[195233]: pgmap v566: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 536 op/s; 23303/113652 objects degraded (20.504%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:20 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:20.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:20.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:20.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:20.731 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:20.777 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:20.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:20.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:20.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:20.978 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:21.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: from='client.45317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[215646]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: from='client.45317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:21 smithi017 ceph-mon[222878]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T15:15:21.461 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:21.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:21.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:21.586 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: from='client.45317 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: failed to encode map e210 with expected crc 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3875826877' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:21 smithi027 ceph-mon[195233]: osdmap e210: 8 total, 8 up, 7 in 2023-09-23T15:15:21.623 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:21.681 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:21.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:21.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:21.791 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":566,"stamp":"2023-09-23T15:15:18.395407+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2485688540,"num_objects":37884,"num_object_clones":0,"num_object_copies":113652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23303,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37884,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":163949,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6459,"num_bytes_recovered":424777895,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124634,"ondisk_log_size":124634,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":8561752,"kb_used_data":6997792,"kb_used_omap":30,"kb_used_meta":1563809,"kb_avail":647654312,"statfs":{"total":671965249536,"available":663198015488,"internally_reserved":0,"allocated":7165739008,"data_stored":7159999674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1601341016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-351731712,"num_objects":-5367,"num_object_clones":0,"num_object_copies":-16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-281,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5367,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":281,"num_bytes_recovered":18415616,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007385"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":203,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.227589+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:15:00.227589+0000","last_peered":"2023-09-23T15:15:00.227589+0000","last_clean":"2023-09-23T15:15:00.227589+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:15:00.227589+0000","last_undegraded":"2023-09-23T15:15:00.227589+0000","last_fullsized":"2023-09-23T15:15:00.227589+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":137,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.237897+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:15:00.237897+0000","last_peered":"2023-09-23T15:15:00.237897+0000","last_clean":"2023-09-23T15:15:00.237897+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:15:00.237897+0000","last_undegraded":"2023-09-23T15:15:00.237897+0000","last_fullsized":"2023-09-23T15:15:00.237897+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":86,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:10.536709+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:15:10.536709+0000","last_peered":"2023-09-23T15:15:10.536709+0000","last_clean":"2023-09-23T15:15:10.536709+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:15:10.536709+0000","last_undegraded":"2023-09-23T15:15:10.536709+0000","last_fullsized":"2023-09-23T15:15:10.536709+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":174,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245045+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:15:00.245045+0000","last_peered":"2023-09-23T15:15:00.245045+0000","last_clean":"2023-09-23T15:15:00.245045+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:15:00.245045+0000","last_undegraded":"2023-09-23T15:15:00.245045+0000","last_fullsized":"2023-09-23T15:15:00.245045+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":206,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239117+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:15:00.239117+0000","last_peered":"2023-09-23T15:15:00.239117+0000","last_clean":"2023-09-23T15:15:00.239117+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:15:00.239117+0000","last_undegraded":"2023-09-23T15:15:00.239117+0000","last_fullsized":"2023-09-23T15:15:00.239117+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243602+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:15:00.243602+0000","last_peered":"2023-09-23T15:15:00.243602+0000","last_clean":"2023-09-23T15:15:00.243602+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:15:00.243602+0000","last_undegraded":"2023-09-23T15:15:00.243602+0000","last_fullsized":"2023-09-23T15:15:00.243602+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":153,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245647+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:15:00.245647+0000","last_peered":"2023-09-23T15:15:00.245647+0000","last_clean":"2023-09-23T15:15:00.245647+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:15:00.245647+0000","last_undegraded":"2023-09-23T15:15:00.245647+0000","last_fullsized":"2023-09-23T15:15:00.245647+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":188,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239189+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:15:00.239189+0000","last_peered":"2023-09-23T15:15:00.239189+0000","last_clean":"2023-09-23T15:15:00.239189+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:15:00.239189+0000","last_undegraded":"2023-09-23T15:15:00.239189+0000","last_fullsized":"2023-09-23T15:15:00.239189+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244621+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:15:00.244621+0000","last_peered":"2023-09-23T15:15:00.244621+0000","last_clean":"2023-09-23T15:15:00.244621+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:15:00.244621+0000","last_undegraded":"2023-09-23T15:15:00.244621+0000","last_fullsized":"2023-09-23T15:15:00.244621+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":199,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247382+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:15:00.247382+0000","last_peered":"2023-09-23T15:15:00.247382+0000","last_clean":"2023-09-23T15:15:00.247382+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:15:00.247382+0000","last_undegraded":"2023-09-23T15:15:00.247382+0000","last_fullsized":"2023-09-23T15:15:00.247382+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":131,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.237558+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:15:00.237558+0000","last_peered":"2023-09-23T15:15:00.237558+0000","last_clean":"2023-09-23T15:15:00.237558+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:15:00.237558+0000","last_undegraded":"2023-09-23T15:15:00.237558+0000","last_fullsized":"2023-09-23T15:15:00.237558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":71,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.237803+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:15:00.237803+0000","last_peered":"2023-09-23T15:15:00.237803+0000","last_clean":"2023-09-23T15:15:00.237803+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:15:00.237803+0000","last_undegraded":"2023-09-23T15:15:00.237803+0000","last_fullsized":"2023-09-23T15:15:00.237803+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":81,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:06.220170+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:06.220170+0000","last_peered":"2023-09-23T15:15:06.220170+0000","last_clean":"2023-09-23T15:15:06.220170+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:06.220170+0000","last_undegraded":"2023-09-23T15:15:06.220170+0000","last_fullsized":"2023-09-23T15:15:06.220170+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":193,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225675+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:15:00.225675+0000","last_peered":"2023-09-23T15:15:00.225675+0000","last_clean":"2023-09-23T15:15:00.225675+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:15:00.225675+0000","last_undegraded":"2023-09-23T15:15:00.225675+0000","last_fullsized":"2023-09-23T15:15:00.225675+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225365+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:15:00.225365+0000","last_peered":"2023-09-23T15:15:00.225365+0000","last_clean":"2023-09-23T15:15:00.225365+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:15:00.225365+0000","last_undegraded":"2023-09-23T15:15:00.225365+0000","last_fullsized":"2023-09-23T15:15:00.225365+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246572+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:15:00.246572+0000","last_peered":"2023-09-23T15:15:00.246572+0000","last_clean":"2023-09-23T15:15:00.246572+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:15:00.246572+0000","last_undegraded":"2023-09-23T15:15:00.246572+0000","last_fullsized":"2023-09-23T15:15:00.246572+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225411+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:15:00.225411+0000","last_peered":"2023-09-23T15:15:00.225411+0000","last_clean":"2023-09-23T15:15:00.225411+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:15:00.225411+0000","last_undegraded":"2023-09-23T15:15:00.225411+0000","last_fullsized":"2023-09-23T15:15:00.225411+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":199,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.240801+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:15:00.240801+0000","last_peered":"2023-09-23T15:15:00.240801+0000","last_clean":"2023-09-23T15:15:00.240801+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:15:00.240801+0000","last_undegraded":"2023-09-23T15:15:00.240801+0000","last_fullsized":"2023-09-23T15:15:00.240801+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246876+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:15:00.246876+0000","last_peered":"2023-09-23T15:15:00.246876+0000","last_clean":"2023-09-23T15:15:00.246876+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:15:00.246876+0000","last_undegraded":"2023-09-23T15:15:00.246876+0000","last_fullsized":"2023-09-23T15:15:00.246876+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":173,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246692+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:15:00.246692+0000","last_peered":"2023-09-23T15:15:00.246692+0000","last_clean":"2023-09-23T15:15:00.246692+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:15:00.246692+0000","last_undegraded":"2023-09-23T15:15:00.246692+0000","last_fullsized":"2023-09-23T15:15:00.246692+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247629+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:15:00.247629+0000","last_peered":"2023-09-23T15:15:00.247629+0000","last_clean":"2023-09-23T15:15:00.247629+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:15:00.247629+0000","last_undegraded":"2023-09-23T15:15:00.247629+0000","last_fullsized":"2023-09-23T15:15:00.247629+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243646+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:15:00.243646+0000","last_peered":"2023-09-23T15:15:00.243646+0000","last_clean":"2023-09-23T15:15:00.243646+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:15:00.243646+0000","last_undegraded":"2023-09-23T15:15:00.243646+0000","last_fullsized":"2023-09-23T15:15:00.243646+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225281+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:15:00.225281+0000","last_peered":"2023-09-23T15:15:00.225281+0000","last_clean":"2023-09-23T15:15:00.225281+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:15:00.225281+0000","last_undegraded":"2023-09-23T15:15:00.225281+0000","last_fullsized":"2023-09-23T15:15:00.225281+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":162,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245115+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:15:00.245115+0000","last_peered":"2023-09-23T15:15:00.245115+0000","last_clean":"2023-09-23T15:15:00.245115+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:15:00.245115+0000","last_undegraded":"2023-09-23T15:15:00.245115+0000","last_fullsized":"2023-09-23T15:15:00.245115+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246925+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:15:00.246925+0000","last_peered":"2023-09-23T15:15:00.246925+0000","last_clean":"2023-09-23T15:15:00.246925+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:15:00.246925+0000","last_undegraded":"2023-09-23T15:15:00.246925+0000","last_fullsized":"2023-09-23T15:15:00.246925+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":175,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243104+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:15:00.243104+0000","last_peered":"2023-09-23T15:15:00.243104+0000","last_clean":"2023-09-23T15:15:00.243104+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:15:00.243104+0000","last_undegraded":"2023-09-23T15:15:00.243104+0000","last_fullsized":"2023-09-23T15:15:00.243104+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":199,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239394+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:15:00.239394+0000","last_peered":"2023-09-23T15:15:00.239394+0000","last_clean":"2023-09-23T15:15:00.239394+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:15:00.239394+0000","last_undegraded":"2023-09-23T15:15:00.239394+0000","last_fullsized":"2023-09-23T15:15:00.239394+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"209'10127","reported_seq":10173,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.696396+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:15:14.696396+0000","last_peered":"2023-09-23T15:15:14.696396+0000","last_clean":"2023-09-23T15:15:14.696396+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:15:14.696396+0000","last_undegraded":"2023-09-23T15:15:14.696396+0000","last_fullsized":"2023-09-23T15:15:14.696396+0000","mapping_epoch":171,"log_start":"174'100","ondisk_log_start":"174'100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10027,"log_dups_size":0,"ondisk_log_size":10027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148570112,"num_objects":2267,"num_object_clones":0,"num_object_copies":6801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10127,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246402+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:15:00.246402+0000","last_peered":"2023-09-23T15:15:00.246402+0000","last_clean":"2023-09-23T15:15:00.246402+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:15:00.246402+0000","last_undegraded":"2023-09-23T15:15:00.246402+0000","last_fullsized":"2023-09-23T15:15:00.246402+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":83,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.226147+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:15:00.226147+0000","last_peered":"2023-09-23T15:15:00.226147+0000","last_clean":"2023-09-23T15:15:00.226147+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:15:00.226147+0000","last_undegraded":"2023-09-23T15:15:00.226147+0000","last_fullsized":"2023-09-23T15:15:00.226147+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":160,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247116+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:15:00.247116+0000","last_peered":"2023-09-23T15:15:00.247116+0000","last_clean":"2023-09-23T15:15:00.247116+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:15:00.247116+0000","last_undegraded":"2023-09-23T15:15:00.247116+0000","last_fullsized":"2023-09-23T15:15:00.247116+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"209'10337","reported_seq":13392,"reported_epoch":209,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:14.606233+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:15:14.606233+0000","last_peered":"2023-09-23T15:15:14.606233+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:15:14.606233+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"179'3900","ondisk_log_start":"179'3900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152633344,"num_objects":2329,"num_object_clones":0,"num_object_copies":6987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3498,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10337,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1500,"num_bytes_recovered":98304000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1169}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":476,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:00.244365+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:15:00.244365+0000","last_peered":"2023-09-23T15:15:00.244365+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:15:00.244365+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":83,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.226199+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:15:00.226199+0000","last_peered":"2023-09-23T15:15:00.226199+0000","last_clean":"2023-09-23T15:15:00.226199+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:15:00.226199+0000","last_undegraded":"2023-09-23T15:15:00.226199+0000","last_fullsized":"2023-09-23T15:15:00.226199+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":201,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225170+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:15:00.225170+0000","last_peered":"2023-09-23T15:15:00.225170+0000","last_clean":"2023-09-23T15:15:00.225170+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:15:00.225170+0000","last_undegraded":"2023-09-23T15:15:00.225170+0000","last_fullsized":"2023-09-23T15:15:00.225170+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"209'10023","reported_seq":10071,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:13.502629+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:15:13.502629+0000","last_peered":"2023-09-23T15:15:13.502629+0000","last_clean":"2023-09-23T15:15:13.502629+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:15:13.502629+0000","last_undegraded":"2023-09-23T15:15:13.502629+0000","last_fullsized":"2023-09-23T15:15:13.502629+0000","mapping_epoch":171,"log_start":"175'900","ondisk_log_start":"175'900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9123,"log_dups_size":0,"ondisk_log_size":9123,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158531584,"num_objects":2419,"num_object_clones":0,"num_object_copies":7257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10023,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245361+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:15:00.245361+0000","last_peered":"2023-09-23T15:15:00.245361+0000","last_clean":"2023-09-23T15:15:00.245361+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:15:00.245361+0000","last_undegraded":"2023-09-23T15:15:00.245361+0000","last_fullsized":"2023-09-23T15:15:00.245361+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":69,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247621+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:15:00.247621+0000","last_peered":"2023-09-23T15:15:00.247621+0000","last_clean":"2023-09-23T15:15:00.247621+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:15:00.247621+0000","last_undegraded":"2023-09-23T15:15:00.247621+0000","last_fullsized":"2023-09-23T15:15:00.247621+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245493+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:15:00.245493+0000","last_peered":"2023-09-23T15:15:00.245493+0000","last_clean":"2023-09-23T15:15:00.245493+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:15:00.245493+0000","last_undegraded":"2023-09-23T15:15:00.245493+0000","last_fullsized":"2023-09-23T15:15:00.245493+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"209'10201","reported_seq":10249,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.373493+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:15:14.373493+0000","last_peered":"2023-09-23T15:15:14.373493+0000","last_clean":"2023-09-23T15:15:14.373493+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:15:14.373493+0000","last_undegraded":"2023-09-23T15:15:14.373493+0000","last_fullsized":"2023-09-23T15:15:14.373493+0000","mapping_epoch":171,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5601,"log_dups_size":0,"ondisk_log_size":5601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161415168,"num_objects":2463,"num_object_clones":0,"num_object_copies":7389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10201,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244818+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:15:00.244818+0000","last_peered":"2023-09-23T15:15:00.244818+0000","last_clean":"2023-09-23T15:15:00.244818+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:15:00.244818+0000","last_undegraded":"2023-09-23T15:15:00.244818+0000","last_fullsized":"2023-09-23T15:15:00.244818+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247443+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:00.247443+0000","last_peered":"2023-09-23T15:15:00.247443+0000","last_clean":"2023-09-23T15:15:00.247443+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:15:00.247443+0000","last_undegraded":"2023-09-23T15:15:00.247443+0000","last_fullsized":"2023-09-23T15:15:00.247443+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243891+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:15:00.243891+0000","last_peered":"2023-09-23T15:15:00.243891+0000","last_clean":"2023-09-23T15:15:00.243891+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:15:00.243891+0000","last_undegraded":"2023-09-23T15:15:00.243891+0000","last_fullsized":"2023-09-23T15:15:00.243891+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"209'10396","reported_seq":10447,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:14.270978+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:15:14.270978+0000","last_peered":"2023-09-23T15:15:14.270978+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:15:14.270978+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"177'2800","ondisk_log_start":"177'2800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7596,"log_dups_size":0,"ondisk_log_size":7596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164888576,"num_objects":2516,"num_object_clones":0,"num_object_copies":7548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10396,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-2494}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246413+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:15:00.246413+0000","last_peered":"2023-09-23T15:15:00.246413+0000","last_clean":"2023-09-23T15:15:00.246413+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:15:00.246413+0000","last_undegraded":"2023-09-23T15:15:00.246413+0000","last_fullsized":"2023-09-23T15:15:00.246413+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":172,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:10.541285+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:15:10.541285+0000","last_peered":"2023-09-23T15:15:10.541285+0000","last_clean":"2023-09-23T15:15:10.541285+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:15:10.541285+0000","last_undegraded":"2023-09-23T15:15:10.541285+0000","last_fullsized":"2023-09-23T15:15:10.541285+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":167,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225773+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:15:00.225773+0000","last_peered":"2023-09-23T15:15:00.225773+0000","last_clean":"2023-09-23T15:15:00.225773+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:15:00.225773+0000","last_undegraded":"2023-09-23T15:15:00.225773+0000","last_fullsized":"2023-09-23T15:15:00.225773+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"209'10210","reported_seq":10258,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:13.502470+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:15:13.502470+0000","last_peered":"2023-09-23T15:15:13.502470+0000","last_clean":"2023-09-23T15:15:13.502470+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:15:13.502470+0000","last_undegraded":"2023-09-23T15:15:13.502470+0000","last_fullsized":"2023-09-23T15:15:13.502470+0000","mapping_epoch":171,"log_start":"175'1100","ondisk_log_start":"175'1100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9110,"log_dups_size":0,"ondisk_log_size":9110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155189248,"num_objects":2368,"num_object_clones":0,"num_object_copies":7104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10210,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":502,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246962+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:15:00.246962+0000","last_peered":"2023-09-23T15:15:00.246962+0000","last_clean":"2023-09-23T15:15:00.246962+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:15:00.246962+0000","last_undegraded":"2023-09-23T15:15:00.246962+0000","last_fullsized":"2023-09-23T15:15:00.246962+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245420+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:15:00.245420+0000","last_peered":"2023-09-23T15:15:00.245420+0000","last_clean":"2023-09-23T15:15:00.245420+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:15:00.245420+0000","last_undegraded":"2023-09-23T15:15:00.245420+0000","last_fullsized":"2023-09-23T15:15:00.245420+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":201,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225910+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:15:00.225910+0000","last_peered":"2023-09-23T15:15:00.225910+0000","last_clean":"2023-09-23T15:15:00.225910+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:15:00.225910+0000","last_undegraded":"2023-09-23T15:15:00.225910+0000","last_fullsized":"2023-09-23T15:15:00.225910+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"209'10223","reported_seq":10271,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.599874+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:15:14.599874+0000","last_peered":"2023-09-23T15:15:14.599874+0000","last_clean":"2023-09-23T15:15:14.599874+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:15:14.599874+0000","last_undegraded":"2023-09-23T15:15:14.599874+0000","last_fullsized":"2023-09-23T15:15:14.599874+0000","mapping_epoch":171,"log_start":"179'3800","ondisk_log_start":"179'3800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6423,"log_dups_size":0,"ondisk_log_size":6423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154337280,"num_objects":2355,"num_object_clones":0,"num_object_copies":7065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10223,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243736+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:15:00.243736+0000","last_peered":"2023-09-23T15:15:00.243736+0000","last_clean":"2023-09-23T15:15:00.243736+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:15:00.243736+0000","last_undegraded":"2023-09-23T15:15:00.243736+0000","last_fullsized":"2023-09-23T15:15:00.243736+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":192,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225064+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:15:00.225064+0000","last_peered":"2023-09-23T15:15:00.225064+0000","last_clean":"2023-09-23T15:15:00.225064+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:15:00.225064+0000","last_undegraded":"2023-09-23T15:15:00.225064+0000","last_fullsized":"2023-09-23T15:15:00.225064+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.226268+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:15:00.226268+0000","last_peered":"2023-09-23T15:15:00.226268+0000","last_clean":"2023-09-23T15:15:00.226268+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:15:00.226268+0000","last_undegraded":"2023-09-23T15:15:00.226268+0000","last_fullsized":"2023-09-23T15:15:00.226268+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"209'10215","reported_seq":20082,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.694363+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:15:14.694363+0000","last_peered":"2023-09-23T15:15:14.694363+0000","last_clean":"2023-09-23T15:15:14.694363+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:15:14.694363+0000","last_undegraded":"2023-09-23T15:15:14.694363+0000","last_fullsized":"2023-09-23T15:15:14.694363+0000","mapping_epoch":203,"log_start":"175'200","ondisk_log_start":"175'200","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10015,"log_dups_size":0,"ondisk_log_size":10015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151781400,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10215,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246876+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:15:00.246876+0000","last_peered":"2023-09-23T15:15:00.246876+0000","last_clean":"2023-09-23T15:15:00.246876+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:15:00.246876+0000","last_undegraded":"2023-09-23T15:15:00.246876+0000","last_fullsized":"2023-09-23T15:15:00.246876+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":201,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239043+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:15:00.239043+0000","last_peered":"2023-09-23T15:15:00.239043+0000","last_clean":"2023-09-23T15:15:00.239043+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:15:00.239043+0000","last_undegraded":"2023-09-23T15:15:00.239043+0000","last_fullsized":"2023-09-23T15:15:00.239043+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":86,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:01.215531+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:01.215531+0000","last_peered":"2023-09-23T15:15:01.215531+0000","last_clean":"2023-09-23T15:15:01.215531+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:01.215531+0000","last_undegraded":"2023-09-23T15:15:01.215531+0000","last_fullsized":"2023-09-23T15:15:01.215531+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"209'10243","reported_seq":10291,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.346505+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:15:14.346505+0000","last_peered":"2023-09-23T15:15:14.346505+0000","last_clean":"2023-09-23T15:15:14.346505+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:15:14.346505+0000","last_undegraded":"2023-09-23T15:15:14.346505+0000","last_fullsized":"2023-09-23T15:15:14.346505+0000","mapping_epoch":171,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5643,"log_dups_size":0,"ondisk_log_size":5643,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152371200,"num_objects":2325,"num_object_clones":0,"num_object_copies":6975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10243,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245452+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:15:00.245452+0000","last_peered":"2023-09-23T15:15:00.245452+0000","last_clean":"2023-09-23T15:15:00.245452+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:15:00.245452+0000","last_undegraded":"2023-09-23T15:15:00.245452+0000","last_fullsized":"2023-09-23T15:15:00.245452+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":385,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.223092+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:15:00.223092+0000","last_peered":"2023-09-23T15:15:00.223092+0000","last_clean":"2023-09-23T15:15:00.223092+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:15:00.223092+0000","last_undegraded":"2023-09-23T15:15:00.223092+0000","last_fullsized":"2023-09-23T15:15:00.223092+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":185,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247580+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:00.247580+0000","last_peered":"2023-09-23T15:15:00.247580+0000","last_clean":"2023-09-23T15:15:00.247580+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:15:00.247580+0000","last_undegraded":"2023-09-23T15:15:00.247580+0000","last_fullsized":"2023-09-23T15:15:00.247580+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.232045+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:15:00.232045+0000","last_peered":"2023-09-23T15:15:00.232045+0000","last_clean":"2023-09-23T15:15:00.232045+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:15:00.232045+0000","last_undegraded":"2023-09-23T15:15:00.232045+0000","last_fullsized":"2023-09-23T15:15:00.232045+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"209'10013","reported_seq":10060,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.605520+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:15:14.605520+0000","last_peered":"2023-09-23T15:15:14.605520+0000","last_clean":"2023-09-23T15:15:14.605520+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:15:14.605520+0000","last_undegraded":"2023-09-23T15:15:14.605520+0000","last_fullsized":"2023-09-23T15:15:14.605520+0000","mapping_epoch":171,"log_start":"178'3600","ondisk_log_start":"178'3600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6413,"log_dups_size":0,"ondisk_log_size":6413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153419776,"num_objects":2341,"num_object_clones":0,"num_object_copies":7023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10013,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246651+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:15:00.246651+0000","last_peered":"2023-09-23T15:15:00.246651+0000","last_clean":"2023-09-23T15:15:00.246651+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:15:00.246651+0000","last_undegraded":"2023-09-23T15:15:00.246651+0000","last_fullsized":"2023-09-23T15:15:00.246651+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":727,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247338+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:15:00.247338+0000","last_peered":"2023-09-23T15:15:00.247338+0000","last_clean":"2023-09-23T15:15:00.247338+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:15:00.247338+0000","last_undegraded":"2023-09-23T15:15:00.247338+0000","last_fullsized":"2023-09-23T15:15:00.247338+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":168,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225323+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:15:00.225323+0000","last_peered":"2023-09-23T15:15:00.225323+0000","last_clean":"2023-09-23T15:15:00.225323+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:15:00.225323+0000","last_undegraded":"2023-09-23T15:15:00.225323+0000","last_fullsized":"2023-09-23T15:15:00.225323+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":101,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222804+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:15:00.222804+0000","last_peered":"2023-09-23T15:15:00.222804+0000","last_clean":"2023-09-23T15:15:00.222804+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:15:00.222804+0000","last_undegraded":"2023-09-23T15:15:00.222804+0000","last_fullsized":"2023-09-23T15:15:00.222804+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"209'10245","reported_seq":10291,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.041586+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:15:14.041586+0000","last_peered":"2023-09-23T15:15:14.041586+0000","last_clean":"2023-09-23T15:15:14.041586+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:15:14.041586+0000","last_undegraded":"2023-09-23T15:15:14.041586+0000","last_fullsized":"2023-09-23T15:15:14.041586+0000","mapping_epoch":171,"log_start":"182'4900","ondisk_log_start":"182'4900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5345,"log_dups_size":0,"ondisk_log_size":5345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10245,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246786+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:15:00.246786+0000","last_peered":"2023-09-23T15:15:00.246786+0000","last_clean":"2023-09-23T15:15:00.246786+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:15:00.246786+0000","last_undegraded":"2023-09-23T15:15:00.246786+0000","last_fullsized":"2023-09-23T15:15:00.246786+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":821,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.235699+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:15:00.235699+0000","last_peered":"2023-09-23T15:15:00.235699+0000","last_clean":"2023-09-23T15:15:00.235699+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:15:00.235699+0000","last_undegraded":"2023-09-23T15:15:00.235699+0000","last_fullsized":"2023-09-23T15:15:00.235699+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":81,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222412+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:15:00.222412+0000","last_peered":"2023-09-23T15:15:00.222412+0000","last_clean":"2023-09-23T15:15:00.222412+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:15:00.222412+0000","last_undegraded":"2023-09-23T15:15:00.222412+0000","last_fullsized":"2023-09-23T15:15:00.222412+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":105,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222005+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:15:00.222005+0000","last_peered":"2023-09-23T15:15:00.222005+0000","last_clean":"2023-09-23T15:15:00.222005+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:15:00.222005+0000","last_undegraded":"2023-09-23T15:15:00.222005+0000","last_fullsized":"2023-09-23T15:15:00.222005+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"209'10080","reported_seq":10126,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.690605+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:15:14.690605+0000","last_peered":"2023-09-23T15:15:14.690605+0000","last_clean":"2023-09-23T15:15:14.690605+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:15:14.690605+0000","last_undegraded":"2023-09-23T15:15:14.690605+0000","last_fullsized":"2023-09-23T15:15:14.690605+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150732800,"num_objects":2300,"num_object_clones":0,"num_object_copies":6900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10080,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":496,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247368+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:15:00.247368+0000","last_peered":"2023-09-23T15:15:00.247368+0000","last_clean":"2023-09-23T15:15:00.247368+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:15:00.247368+0000","last_undegraded":"2023-09-23T15:15:00.247368+0000","last_fullsized":"2023-09-23T15:15:00.247368+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":497,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247085+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:15:00.247085+0000","last_peered":"2023-09-23T15:15:00.247085+0000","last_clean":"2023-09-23T15:15:00.247085+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:15:00.247085+0000","last_undegraded":"2023-09-23T15:15:00.247085+0000","last_fullsized":"2023-09-23T15:15:00.247085+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":123,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.237480+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:15:00.237480+0000","last_peered":"2023-09-23T15:15:00.237480+0000","last_clean":"2023-09-23T15:15:00.237480+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:15:00.237480+0000","last_undegraded":"2023-09-23T15:15:00.237480+0000","last_fullsized":"2023-09-23T15:15:00.237480+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":159,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243443+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:15:00.243443+0000","last_peered":"2023-09-23T15:15:00.243443+0000","last_clean":"2023-09-23T15:15:00.243443+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:15:00.243443+0000","last_undegraded":"2023-09-23T15:15:00.243443+0000","last_fullsized":"2023-09-23T15:15:00.243443+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"209'10274","reported_seq":10323,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:14.605545+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:15:14.605545+0000","last_peered":"2023-09-23T15:15:14.605545+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:15:14.605545+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"179'3800","ondisk_log_start":"179'3800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155320320,"num_objects":2370,"num_object_clones":0,"num_object_copies":7110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10274,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-2582}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243818+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:15:00.243818+0000","last_peered":"2023-09-23T15:15:00.243818+0000","last_clean":"2023-09-23T15:15:00.243818+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:15:00.243818+0000","last_undegraded":"2023-09-23T15:15:00.243818+0000","last_fullsized":"2023-09-23T15:15:00.243818+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":292,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222170+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:15:00.222170+0000","last_peered":"2023-09-23T15:15:00.222170+0000","last_clean":"2023-09-23T15:15:00.222170+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:15:00.222170+0000","last_undegraded":"2023-09-23T15:15:00.222170+0000","last_fullsized":"2023-09-23T15:15:00.222170+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":154,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.440242+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:04.440242+0000","last_peered":"2023-09-23T15:15:04.440242+0000","last_clean":"2023-09-23T15:15:04.440242+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:04.440242+0000","last_undegraded":"2023-09-23T15:15:04.440242+0000","last_fullsized":"2023-09-23T15:15:04.440242+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":183,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244891+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:15:00.244891+0000","last_peered":"2023-09-23T15:15:00.244891+0000","last_clean":"2023-09-23T15:15:00.244891+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:15:00.244891+0000","last_undegraded":"2023-09-23T15:15:00.244891+0000","last_fullsized":"2023-09-23T15:15:00.244891+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"209'10119","reported_seq":10166,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:13.505359+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:15:13.505359+0000","last_peered":"2023-09-23T15:15:13.505359+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:15:13.505359+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"175'1000","ondisk_log_start":"175'1000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9119,"log_dups_size":0,"ondisk_log_size":9119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156565504,"num_objects":2389,"num_object_clones":0,"num_object_copies":7167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10119,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-2485}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.245259+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:15:00.245259+0000","last_peered":"2023-09-23T15:15:00.245259+0000","last_clean":"2023-09-23T15:15:00.245259+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:15:00.245259+0000","last_undegraded":"2023-09-23T15:15:00.245259+0000","last_fullsized":"2023-09-23T15:15:00.245259+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":814,"reported_epoch":208,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:00.246152+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:15:00.246152+0000","last_peered":"2023-09-23T15:15:00.246152+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:15:00.246152+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":81,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.222975+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:15:00.222975+0000","last_peered":"2023-09-23T15:15:00.222975+0000","last_clean":"2023-09-23T15:15:00.222975+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:15:00.222975+0000","last_undegraded":"2023-09-23T15:15:00.222975+0000","last_fullsized":"2023-09-23T15:15:00.222975+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":159,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.440447+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:04.440447+0000","last_peered":"2023-09-23T15:15:04.440447+0000","last_clean":"2023-09-23T15:15:04.440447+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:04.440447+0000","last_undegraded":"2023-09-23T15:15:04.440447+0000","last_fullsized":"2023-09-23T15:15:04.440447+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"209'9925","reported_seq":9970,"reported_epoch":209,"state":"active+clean","last_fresh":"2023-09-23T15:15:14.155954+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:15:14.155954+0000","last_peered":"2023-09-23T15:15:14.155954+0000","last_clean":"2023-09-23T15:15:14.155954+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:15:14.155954+0000","last_undegraded":"2023-09-23T15:15:14.155954+0000","last_fullsized":"2023-09-23T15:15:14.155954+0000","mapping_epoch":171,"log_start":"177'2400","ondisk_log_start":"177'2400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7525,"log_dups_size":0,"ondisk_log_size":7525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155779072,"num_objects":2377,"num_object_clones":0,"num_object_copies":7131,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2377,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9925,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.246311+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:15:00.246311+0000","last_peered":"2023-09-23T15:15:00.246311+0000","last_clean":"2023-09-23T15:15:00.246311+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:15:00.246311+0000","last_undegraded":"2023-09-23T15:15:00.246311+0000","last_fullsized":"2023-09-23T15:15:00.246311+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":185,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239378+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:15:00.239378+0000","last_peered":"2023-09-23T15:15:00.239378+0000","last_clean":"2023-09-23T15:15:00.239378+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:15:00.239378+0000","last_undegraded":"2023-09-23T15:15:00.239378+0000","last_fullsized":"2023-09-23T15:15:00.239378+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":669,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.223690+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:15:00.223690+0000","last_peered":"2023-09-23T15:15:00.223690+0000","last_clean":"2023-09-23T15:15:00.223690+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:15:00.223690+0000","last_undegraded":"2023-09-23T15:15:00.223690+0000","last_fullsized":"2023-09-23T15:15:00.223690+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":189,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.239284+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:15:00.239284+0000","last_peered":"2023-09-23T15:15:00.239284+0000","last_clean":"2023-09-23T15:15:00.239284+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:15:00.239284+0000","last_undegraded":"2023-09-23T15:15:00.239284+0000","last_fullsized":"2023-09-23T15:15:00.239284+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"209'10290","reported_seq":10343,"reported_epoch":209,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:14.123536+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:15:14.123536+0000","last_peered":"2023-09-23T15:15:14.123536+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:15:14.123536+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"186'5000","ondisk_log_start":"186'5000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5290,"log_dups_size":0,"ondisk_log_size":5290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153354240,"num_objects":2340,"num_object_clones":0,"num_object_copies":7020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10290,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-2626}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":485,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244537+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:15:00.244537+0000","last_peered":"2023-09-23T15:15:00.244537+0000","last_clean":"2023-09-23T15:15:00.244537+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:15:00.244537+0000","last_undegraded":"2023-09-23T15:15:00.244537+0000","last_fullsized":"2023-09-23T15:15:00.244537+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":685,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.243452+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:15:00.243452+0000","last_peered":"2023-09-23T15:15:00.243452+0000","last_clean":"2023-09-23T15:15:00.243452+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:15:00.243452+0000","last_undegraded":"2023-09-23T15:15:00.243452+0000","last_fullsized":"2023-09-23T15:15:00.243452+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":171,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244972+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:15:00.244972+0000","last_peered":"2023-09-23T15:15:00.244972+0000","last_clean":"2023-09-23T15:15:00.244972+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:15:00.244972+0000","last_undegraded":"2023-09-23T15:15:00.244972+0000","last_fullsized":"2023-09-23T15:15:00.244972+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":150,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225162+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:15:00.225162+0000","last_peered":"2023-09-23T15:15:00.225162+0000","last_clean":"2023-09-23T15:15:00.225162+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:15:00.225162+0000","last_undegraded":"2023-09-23T15:15:00.225162+0000","last_fullsized":"2023-09-23T15:15:00.225162+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":161,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.244043+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:15:00.244043+0000","last_peered":"2023-09-23T15:15:00.244043+0000","last_clean":"2023-09-23T15:15:00.244043+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:15:00.244043+0000","last_undegraded":"2023-09-23T15:15:00.244043+0000","last_fullsized":"2023-09-23T15:15:00.244043+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":208,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.229371+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:15:00.229371+0000","last_peered":"2023-09-23T15:15:00.229371+0000","last_clean":"2023-09-23T15:15:00.229371+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:15:00.229371+0000","last_undegraded":"2023-09-23T15:15:00.229371+0000","last_fullsized":"2023-09-23T15:15:00.229371+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":187,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247524+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:15:00.247524+0000","last_peered":"2023-09-23T15:15:00.247524+0000","last_clean":"2023-09-23T15:15:00.247524+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:15:00.247524+0000","last_undegraded":"2023-09-23T15:15:00.247524+0000","last_fullsized":"2023-09-23T15:15:00.247524+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":201,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225477+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:15:00.225477+0000","last_peered":"2023-09-23T15:15:00.225477+0000","last_clean":"2023-09-23T15:15:00.225477+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:15:00.225477+0000","last_undegraded":"2023-09-23T15:15:00.225477+0000","last_fullsized":"2023-09-23T15:15:00.225477+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":153,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.247520+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:15:00.247520+0000","last_peered":"2023-09-23T15:15:00.247520+0000","last_clean":"2023-09-23T15:15:00.247520+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:15:00.247520+0000","last_undegraded":"2023-09-23T15:15:00.247520+0000","last_fullsized":"2023-09-23T15:15:00.247520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":184,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225266+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:15:00.225266+0000","last_peered":"2023-09-23T15:15:00.225266+0000","last_clean":"2023-09-23T15:15:00.225266+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:15:00.225266+0000","last_undegraded":"2023-09-23T15:15:00.225266+0000","last_fullsized":"2023-09-23T15:15:00.225266+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":180,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.238228+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:15:00.238228+0000","last_peered":"2023-09-23T15:15:00.238228+0000","last_clean":"2023-09-23T15:15:00.238228+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:15:00.238228+0000","last_undegraded":"2023-09-23T15:15:00.238228+0000","last_fullsized":"2023-09-23T15:15:00.238228+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":194,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.237523+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:15:00.237523+0000","last_peered":"2023-09-23T15:15:00.237523+0000","last_clean":"2023-09-23T15:15:00.237523+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:15:00.237523+0000","last_undegraded":"2023-09-23T15:15:00.237523+0000","last_fullsized":"2023-09-23T15:15:00.237523+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":157,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:04.440741+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:04.440741+0000","last_peered":"2023-09-23T15:15:04.440741+0000","last_clean":"2023-09-23T15:15:04.440741+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:04.440741+0000","last_undegraded":"2023-09-23T15:15:04.440741+0000","last_fullsized":"2023-09-23T15:15:04.440741+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":177,"reported_epoch":208,"state":"active+clean","last_fresh":"2023-09-23T15:15:00.225022+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:15:00.225022+0000","last_peered":"2023-09-23T15:15:00.225022+0000","last_clean":"2023-09-23T15:15:00.225022+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:15:00.225022+0000","last_undegraded":"2023-09-23T15:15:00.225022+0000","last_fullsized":"2023-09-23T15:15:00.225022+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2480930840,"num_objects":37857,"num_object_clones":0,"num_object_copies":113571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":37857,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162921,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6400,"num_bytes_recovered":419430400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7111913472,"data_stored":7111901256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120221,"ondisk_log_size":120221,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201526,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463162,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1374064,"kb_used_data":1212072,"kb_used_omap":0,"kb_used_meta":161984,"kb_avail":92371088,"statfs":{"total":95995035648,"available":94587994112,"internally_reserved":0,"allocated":1241161728,"data_stored":1240465382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":165871616},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.617,"15min":2.6779999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":6.367,"5min":41.491,"15min":108.39400000000001},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.8130000000000002,"15min":2.8180000000000001},"min":{"1min":0.29299999999999998,"5min":0.223,"15min":0.223},"max":{"1min":12.634,"5min":43.151000000000003,"15min":108.452},"last":0.78600000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":4.5350000000000001,"5min":3.6030000000000002,"15min":3.448},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":43.377000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":2.9089999999999998},{"interface":"front","average":{"1min":7.2919999999999998,"5min":4.3440000000000003,"15min":3.8519999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":42.704000000000001,"5min":42.704000000000001,"15min":42.704000000000001},"last":3.395}]},{"osd":2,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":13.1,"5min":6.4619999999999997,"15min":5.702},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":112.352,"5min":112.352,"15min":180.374},"last":2.173},{"interface":"front","average":{"1min":13.564,"5min":6.4489999999999998,"15min":5.7329999999999997},"min":{"1min":0.45200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":117.667,"5min":117.667,"15min":181.685},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":12.955,"5min":4.3739999999999997,"15min":1.992},"min":{"1min":0.441,"5min":0.249,"15min":0.249},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":0.94399999999999995},{"interface":"front","average":{"1min":12.819000000000001,"5min":4.4420000000000002,"15min":1.982},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":80.977999999999994,"5min":80.977999999999994,"15min":80.977999999999994},"last":3.0169999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4870000000000001,"15min":1.0309999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.75800000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.6040000000000001,"15min":1.121},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7590000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":2.5230000000000001,"15min":1.4059999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":2.976,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.80600000000000005,"5min":2.5310000000000001,"15min":1.452},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7859999999999996,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.443}]},{"osd":7,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.381,"15min":1},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":7.5910000000000002,"15min":17},"last":1.528},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3939999999999999,"15min":0.97499999999999998},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":3.754,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":1.5469999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015200,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":747136,"kb_used_data":594424,"kb_used_omap":6,"kb_used_meta":152697,"kb_avail":92998016,"statfs":{"total":95995035648,"available":95229968384,"internally_reserved":0,"allocated":608690176,"data_stored":607693176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":156361803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":3.6080000000000001,"15min":3.738},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.0839999999999996,"5min":16.626000000000001,"15min":16.626000000000001},"last":3.3279999999999998},{"interface":"front","average":{"1min":3.5609999999999999,"5min":4.4000000000000004,"15min":4.54},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.558,"5min":16.131,"15min":16.131},"last":11.772}]},{"osd":2,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0229999999999997,"5min":4.3070000000000004,"15min":4.1870000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.757999999999999,"5min":13.843,"15min":13.843},"last":2.4380000000000002},{"interface":"front","average":{"1min":4.3380000000000001,"5min":4.2460000000000004,"15min":4.2300000000000004},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":3.4900000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9539999999999997,"5min":9.6270000000000007,"15min":10.406000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.27,"5min":123.426,"15min":123.426},"last":19.617999999999999},{"interface":"front","average":{"1min":3.5499999999999998,"5min":10.762,"15min":11.964},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1010000000000009,"5min":123.32899999999999,"15min":123.32899999999999},"last":7.7530000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.7090000000000001,"15min":2.7000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.7199999999999998,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":2.0150000000000001},{"interface":"front","average":{"1min":3.129,"5min":3.472,"15min":3.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.779,"5min":16.779,"15min":16.779},"last":3.2970000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":3.419,"15min":3.6589999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8259999999999996,"5min":16.166,"15min":16.166},"last":2.899},{"interface":"front","average":{"1min":3.0190000000000001,"5min":2.9209999999999998,"15min":2.9049999999999998},"min":{"1min":0.35399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7640000000000002,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":1.99}]},{"osd":6,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":3.488,"15min":3.702},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.8010000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":2.9369999999999998},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.8050000000000002,"15min":2.7669999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.7930000000000001,"5min":7.7930000000000001,"15min":7.7930000000000001},"last":1.9510000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.1749999999999998,"15min":3.1629999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.8179999999999996,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":2.0369999999999999},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2909999999999999,"15min":3.3330000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":3.351}]}]},{"osd":0,"up_from":95,"seq":408021893249,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873944,"kb_used_data":610704,"kb_used_omap":8,"kb_used_meta":263223,"kb_avail":92871208,"statfs":{"total":95995035648,"available":95100116992,"internally_reserved":0,"allocated":625360896,"data_stored":624457369,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":269540948},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.875,"15min":4.1529999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.5099999999999998,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.19800000000000001},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.274,"15min":4.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4249999999999998,"5min":31.378,"15min":31.378},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":15.058999999999999,"5min":7.9790000000000001,"15min":3.3519999999999999},"min":{"1min":0.311,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":93.873999999999995,"5min":122.20099999999999,"15min":122.20099999999999},"last":6.7249999999999996},{"interface":"front","average":{"1min":13.999000000000001,"5min":6.9569999999999999,"15min":2.891},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":94.652000000000001,"5min":122.071,"15min":122.071},"last":2.0630000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:14:28 2023","interfaces":[{"interface":"back","average":{"1min":4.8159999999999998,"5min":5.5419999999999998,"15min":2.3420000000000001},"min":{"1min":0.35299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":25.643000000000001,"5min":99.204999999999998,"15min":99.204999999999998},"last":6.7510000000000003},{"interface":"front","average":{"1min":4.9729999999999999,"5min":4.6059999999999999,"15min":2.3490000000000002},"min":{"1min":0.40699999999999997,"5min":0.252,"15min":0.252},"max":{"1min":25.632000000000001,"5min":78.902000000000001,"15min":78.902000000000001},"last":2.0289999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.9319999999999999,"15min":1.595},"min":{"1min":0.46200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0499999999999998,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.0150000000000001,"15min":1.5980000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9809999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":6.6909999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.9420000000000002,"15min":1.6479999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2370000000000001,"5min":109.369,"15min":109.369},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.532,"15min":1.8049999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.1260000000000003,"5min":109.387,"15min":109.387},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.9590000000000001,"15min":1.6319999999999999},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.032,"5min":99.114999999999995,"15min":99.114999999999995},"last":6.641},{"interface":"front","average":{"1min":1.171,"5min":2.8359999999999999,"15min":1.5580000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2910000000000004,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.378}]},{"osd":7,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.0870000000000002,"15min":1.6759999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2640000000000002,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.4199999999999999,"15min":1.4890000000000001},"min":{"1min":0.497,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.093,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.46700000000000003}]}]},{"osd":2,"up_from":105,"seq":450971566196,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1719256,"kb_used_data":1396368,"kb_used_omap":11,"kb_used_meta":322868,"kb_avail":92025896,"statfs":{"total":95995035648,"available":94234517504,"internally_reserved":0,"allocated":1429880832,"data_stored":1428981031,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":330617616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":10.598000000000001,"5min":4.6280000000000001,"15min":1.9299999999999999},"min":{"1min":0.215,"5min":0.184,"15min":0.184},"max":{"1min":87.132999999999996,"5min":87.132999999999996,"15min":87.132999999999996},"last":4.4649999999999999},{"interface":"front","average":{"1min":7.6420000000000003,"5min":3.6579999999999999,"15min":1.595},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":84.820999999999998,"5min":84.820999999999998,"15min":84.820999999999998},"last":4.6299999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:14:47 2023","interfaces":[{"interface":"back","average":{"1min":9.2129999999999992,"5min":5.8739999999999997,"15min":5.3179999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":84.641000000000005,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":9.2040000000000006,"5min":3.8799999999999999,"15min":2.992},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":84.751999999999995,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 15:14:27 2023","interfaces":[{"interface":"back","average":{"1min":10.016,"5min":3.4990000000000001,"15min":1.748},"min":{"1min":0.44700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":87.108000000000004,"5min":87.108000000000004,"15min":87.108000000000004},"last":9.6600000000000001},{"interface":"front","average":{"1min":9.4209999999999994,"5min":3.3980000000000001,"15min":1.754},"min":{"1min":0.59599999999999997,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":84.739000000000004,"5min":84.739000000000004,"15min":84.739000000000004},"last":9.6859999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.54,"5min":3.6659999999999999,"15min":1.7649999999999999},"min":{"1min":0.42299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":84.608000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.59899999999999998},{"interface":"front","average":{"1min":8.8379999999999992,"5min":4.8460000000000001,"15min":2.254},"min":{"1min":0.41099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":4.4279999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6260000000000003,"5min":3.6459999999999999,"15min":1.8779999999999999},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":84.498000000000005,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.67900000000000005},{"interface":"front","average":{"1min":6.4429999999999996,"5min":3.5600000000000001,"15min":1.825},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":84.664000000000001,"5min":84.664000000000001,"15min":84.664000000000001},"last":4.5369999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9589999999999996,"5min":3.9089999999999998,"15min":1.9490000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":84.715999999999994,"5min":84.715999999999994,"15min":84.715999999999994},"last":4.601},{"interface":"front","average":{"1min":9.0410000000000004,"5min":3.847,"15min":1.9430000000000001},"min":{"1min":0.44,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":87.055000000000007,"5min":87.055000000000007,"15min":87.055000000000007},"last":4.3920000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8410000000000002,"5min":3.2010000000000001,"15min":1.7350000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":84.596000000000004,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.72999999999999998},{"interface":"front","average":{"1min":6.7770000000000001,"5min":3.274,"15min":1.7210000000000001},"min":{"1min":0.46999999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":84.783000000000001,"5min":84.783000000000001,"15min":84.783000000000001},"last":4.5739999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402670,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1950056,"kb_used_data":1666784,"kb_used_omap":3,"kb_used_meta":283196,"kb_avail":91795096,"statfs":{"total":95995035648,"available":93998178304,"internally_reserved":0,"allocated":1706786816,"data_stored":1705925719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":289992873},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":4.0629999999999997,"5min":3.0760000000000001,"15min":1.764},"min":{"1min":0.36499999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":25.111999999999998,"5min":61.744,"15min":79.611000000000004},"last":4.6710000000000003},{"interface":"front","average":{"1min":3.2970000000000002,"5min":3.0870000000000002,"15min":1.788},"min":{"1min":0.29599999999999999,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":9.6270000000000007,"5min":61.686,"15min":79.656000000000006},"last":3.133}]},{"osd":1,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":6.2789999999999999,"5min":4.9240000000000004,"15min":4.6980000000000004},"min":{"1min":0.34200000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.873999999999999,"5min":28.315000000000001,"15min":28.315000000000001},"last":1.3220000000000001},{"interface":"front","average":{"1min":4.7350000000000003,"5min":4.8230000000000004,"15min":4.8380000000000001},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.395,"5min":14.427,"15min":14.427},"last":4.6239999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":5.4690000000000003,"5min":5.5890000000000004,"15min":3.2749999999999999},"min":{"1min":0.309,"5min":0.22,"15min":0.22},"max":{"1min":25.187000000000001,"5min":109.077,"15min":109.077},"last":4.6959999999999997},{"interface":"front","average":{"1min":4.4580000000000002,"5min":6.0449999999999999,"15min":3.4279999999999999},"min":{"1min":0.31,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":19.873000000000001,"5min":119.72799999999999,"15min":119.72799999999999},"last":6.2480000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":1.9910000000000001,"15min":1.1299999999999999},"min":{"1min":0.34200000000000003,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4169999999999998,"5min":9.0399999999999991,"15min":9.0399999999999991},"last":0.63500000000000001},{"interface":"front","average":{"1min":2.802,"5min":2.1509999999999998,"15min":1.3320000000000001},"min":{"1min":0.32300000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.632999999999999,"5min":10.632999999999999,"15min":10.632999999999999},"last":1.2450000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.4060000000000001,"5min":2.8170000000000002,"15min":1.46},"min":{"1min":0.33000000000000002,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":6.3869999999999996,"5min":56.960999999999999,"15min":56.960999999999999},"last":0.75700000000000001},{"interface":"front","average":{"1min":2.839,"5min":3.0630000000000002,"15min":1.675},"min":{"1min":0.36399999999999999,"5min":0.315,"15min":0.315},"max":{"1min":10.738,"5min":56.981999999999999,"15min":56.981999999999999},"last":1.2050000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.1419999999999999,"5min":2.355,"15min":1.3260000000000001},"min":{"1min":0.32300000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.3419999999999996,"5min":24.318000000000001,"15min":24.318000000000001},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.7789999999999999,"5min":2.6219999999999999,"15min":1.5780000000000001},"min":{"1min":0.39400000000000002,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":10.561999999999999,"5min":19.355,"15min":19.355},"last":1.296}]},{"osd":7,"last update":"Sat Sep 23 15:14:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8380000000000001,"5min":2.726,"15min":1.585},"min":{"1min":0.42599999999999999,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":10.603,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.272},{"interface":"front","average":{"1min":2.702,"5min":2.113,"15min":1.2050000000000001},"min":{"1min":0.437,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":14.351000000000001,"5min":14.351000000000001,"15min":14.351000000000001},"last":4.5839999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986435,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952684,"kb_used_data":769700,"kb_used_omap":0,"kb_used_meta":182976,"kb_avail":92792468,"statfs":{"total":95995035648,"available":95019487232,"internally_reserved":0,"allocated":788172800,"data_stored":787480029,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":187367424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":3.375,"15min":2.5289999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":65.988,"5min":66.605999999999995,"15min":122.69199999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":5.4029999999999996,"5min":3.3639999999999999,"15min":2.6309999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":65.396000000000001,"5min":66.384,"15min":122.745},"last":0.67800000000000005}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":4.2539999999999996,"15min":4.4500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.783999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":4.6929999999999996},{"interface":"front","average":{"1min":3.5699999999999998,"5min":4.7750000000000004,"15min":4.976},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":28.815000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":6.4989999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.5140000000000002,"15min":2.883},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.670999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":6.8010000000000002},{"interface":"front","average":{"1min":3.0369999999999999,"5min":4.984,"15min":3.1829999999999998},"min":{"1min":0.41399999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":17.756,"5min":147.88900000000001,"15min":147.88900000000001},"last":4.1420000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":4.8849999999999998,"5min":8.2460000000000004,"15min":3.3100000000000001},"min":{"1min":0.32400000000000001,"5min":0.252,"15min":0.252},"max":{"1min":38.685000000000002,"5min":163.97499999999999,"15min":163.97499999999999},"last":3.077},{"interface":"front","average":{"1min":5.5129999999999999,"5min":8.0139999999999993,"15min":3.149},"min":{"1min":0.375,"5min":0.29499999999999998,"15min":0.27800000000000002},"max":{"1min":38.767000000000003,"5min":164.447,"15min":164.447},"last":3.0209999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.452,"15min":1.1950000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.1479999999999999,"5min":14.073,"15min":28.891999999999999},"last":2.5089999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.208,"15min":1.0600000000000001},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.577,"5min":8.9920000000000009,"15min":32.100999999999999},"last":2.98}]},{"osd":6,"last update":"Sat Sep 23 15:14:22 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":3.6800000000000002,"15min":1.7350000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.3319999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.69099999999999995,"5min":3.323,"15min":1.6040000000000001},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":2.5179999999999998,"5min":210.78899999999999,"15min":210.78899999999999},"last":3.0419999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:14:24 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.502,"15min":1.1319999999999999},"min":{"1min":0.34200000000000003,"5min":0.309,"15min":0.28799999999999998},"max":{"1min":1.6419999999999999,"5min":18.23,"15min":18.23},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.48,"15min":1.1459999999999999},"min":{"1min":0.40699999999999997,"5min":0.314,"15min":0.314},"max":{"1min":2.214,"5min":14.044,"15min":14.653},"last":0.76900000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724798,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":944612,"kb_used_data":747740,"kb_used_omap":0,"kb_used_meta":196864,"kb_avail":92800540,"statfs":{"total":95995035648,"available":95027752960,"internally_reserved":0,"allocated":765685760,"data_stored":764996968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":201588736},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1829999999999998,"15min":1.292},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.9530000000000003,"5min":17.044,"15min":24.962},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.9039999999999999,"15min":1.226},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":6.2549999999999999,"5min":18.814,"15min":25.317},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5030000000000001,"5min":2.2389999999999999,"15min":1.861},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.082999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.252},{"interface":"front","average":{"1min":2.9510000000000001,"5min":2.4049999999999998,"15min":2.3140000000000001},"min":{"1min":0.39200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":17.963999999999999,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.27800000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":4.7460000000000004,"15min":2.2989999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":33.68,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.315},{"interface":"front","average":{"1min":4.0510000000000002,"5min":4.8639999999999999,"15min":2.2930000000000001},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":33.707999999999998,"5min":97.305999999999997,"15min":97.305999999999997},"last":0.38400000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":5.4550000000000001,"5min":3.7829999999999999,"15min":1.8360000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":32.305999999999997,"5min":53.088999999999999,"15min":53.088999999999999},"last":26.338000000000001},{"interface":"front","average":{"1min":5.0970000000000004,"5min":3.5249999999999999,"15min":1.734},"min":{"1min":0.46400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":24.052,"5min":53.764000000000003,"15min":53.764000000000003},"last":20.699999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":2.1480000000000001,"15min":1.1639999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.369,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.2349999999999999,"15min":1.238},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.88200000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.494,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":11.076000000000001,"15min":15.196},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.3859999999999999,"15min":1.012},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":1.367,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.8580000000000001,"15min":1.228},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":1.331,"5min":11.353999999999999,"15min":15.301},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.9119999999999999,"15min":1.212},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.8460000000000001,"5min":15.762,"15min":15.762},"last":0.43099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":615780352,"data_stored":615776280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":600117248,"data_stored":600113176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1423507456,"data_stored":1423507456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1697972224,"data_stored":1697972224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":780337152,"data_stored":780337152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":759959552,"data_stored":759955480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1234239488,"data_stored":1234239488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:15:21.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:15:21.922 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:21.927 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:22.148 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:22.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:22.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:22.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:22.291 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:22.351 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:22.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:22.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:22 smithi017 ceph-mon[215646]: pgmap v568: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:22.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23145/105912 objects degraded (21.853%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:22.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:22 smithi017 ceph-mon[222878]: pgmap v568: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:22.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23145/105912 objects degraded (21.853%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:22.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:22.542 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:22.542 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:22 smithi027 ceph-mon[195233]: pgmap v568: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 535 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23145/105912 objects degraded (21.853%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:22.628 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:22.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:22.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:22.830 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:22.867 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:22.902 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:22.940 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:22.972 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.972 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.973 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:15:22.996 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:23.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:23.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:23.080 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:23.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:23.158 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:23.226 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:23.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:23.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:23.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:23.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:23 smithi017 ceph-mon[215646]: pgmap v569: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:23.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3888039233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:23.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:23 smithi017 ceph-mon[222878]: pgmap v569: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:23.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3888039233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:23.384 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:23.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:15:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:23 smithi027 ceph-mon[195233]: pgmap v569: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/105912 objects degraded (21.853%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3888039233' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:24.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:24.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:24.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:24.732 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:24.762 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:24.814 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:24.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:24.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:24.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:24.956 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:25.018 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:25.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:25.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:25.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:25.157 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:25.214 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:25.221 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:25.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:168635 6.d e210/203 6:bb1ae862:::benchmark_data_smithi017_350392_object68244:head v 210'10569, mlcod=210'10567)", 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:15:25.340045+0000", 2023-09-23T15:15:25.349 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0062885950000000001, 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063160389999999999, 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340045+0000", 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:15:25.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340045+0000", 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340045+0000", 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340053+0000", 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1499999999999999e-06 2023-09-23T15:15:25.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340071+0000", 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7917000000000001e-05 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340081+0000", 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0161e-05 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:15:25.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340140+0000", 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9147999999999997e-05 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:25.340157+0000", 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7523000000000001e-05 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 1 2023-09-23T15:15:25.353 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:15:25.428 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:25.523 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:25.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:25.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:25.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:25.665 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:25 smithi017 ceph-mon[215646]: pgmap v570: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/102417 objects degraded (22.599%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:25.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:25 smithi017 ceph-mon[222878]: pgmap v570: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/102417 objects degraded (22.599%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:25.735 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:25.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:25.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:25.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:25 smithi027 ceph-mon[195233]: pgmap v570: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 528 op/s; 23145/102417 objects degraded (22.599%); 1.9 MiB/s, 29 objects/s recovering 2023-09-23T15:15:25.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:25.872 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:25.917 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:25.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:25.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:26.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:26.119 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:26.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:26.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:26.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:26.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:26.464 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:26.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:15:26.896 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:26.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:27.001 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:27.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:27.036 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:27.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:15:27.092 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:27.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:27.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:27.227 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:27.282 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:27.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:27.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:27.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:27.420 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:27.469 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:27.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:27.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:27.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:27.666 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:27.925 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:27.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:28.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:28.070 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:27 smithi017 ceph-mon[215646]: pgmap v571: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:28.071 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23015/98118 objects degraded (23.456%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:28.071 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:28] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T15:15:28.071 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:27 smithi017 ceph-mon[222878]: pgmap v571: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:28.072 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23015/98118 objects degraded (23.456%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:28.083 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:28.083 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 23015/98118 objects degraded (23.456%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":735,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":210,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":32706,"data_bytes":2146343132,"bytes_used":7888416768,"bytes_avail":664076832768,"bytes_total":671965249536,"degraded_objects":23015,"degraded_total":98118,"degraded_ratio":0.23456450396461404,"recovering_objects_per_sec":28,"recovering_bytes_per_sec":1886753,"recovering_keys_per_sec":0,"num_objects_recovered":288,"num_bytes_recovered":18874368,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":517},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 26s)","progress":0.93577980995178223,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:15:28.162 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:28.291 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:28.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:28.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:27 smithi027 ceph-mon[195233]: pgmap v571: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23015/98118 objects degraded (23.456%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:28.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:28.467 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:28.599 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:28.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:28.605 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:28.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:28.781 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:29.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/141124961' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:29.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:29.029 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/141124961' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:29.030 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:29.313 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:29.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:29.319 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/141124961' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:29.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:29.517 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:29.688 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:29.689 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:29.721 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:15:29.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:15:29.728 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:29.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:29.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:30.013 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:30.249 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:29 smithi017 ceph-mon[215646]: pgmap v572: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:30.249 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:29 smithi017 ceph-mon[222878]: pgmap v572: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:30.281 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:30.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:30.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:29 smithi027 ceph-mon[195233]: pgmap v572: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 23015/98118 objects degraded (23.456%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:30.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:30.509 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:31.016 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:15:31.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1943843148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:31.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:31.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1943843148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:31.197 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1943843148' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: pgmap v573: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: failed to encode map e211 with expected crc 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T15:15:32.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22887/90381 objects degraded (25.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: pgmap v573: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: failed to encode map e211 with expected crc 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T15:15:32.202 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:31 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22887/90381 objects degraded (25.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: pgmap v573: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 502 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: failed to encode map e211 with expected crc 2023-09-23T15:15:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:15:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: osdmap e211: 8 total, 8 up, 7 in 2023-09-23T15:15:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:31 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22887/90381 objects degraded (25.323%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:32.497 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:32.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:32.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:32.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:32.664 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:32.968 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:32.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:32.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:33.080 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:33.113 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:33.180 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:33.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:33.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:33.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:33.453 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:33.614 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:33.636 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:33.659 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:33.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:33.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:33.686 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:15:33.686 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:15:33.686 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:15:33.686 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:15:33.686 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:15:33.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:33.733 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:33.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:33.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:33.809 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:33.832 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:33.870 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:33.872 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:33.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:33.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:33.905 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:33.942 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:33.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:33.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.993 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.994 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.994 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:15:33.994 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:15:34.013 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:34.067 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:34.073 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:34 smithi027 ceph-mon[195233]: pgmap v575: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:34.073 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3782939148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:34.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:34.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:34.174 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:34.231 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:34 smithi017 ceph-mon[222878]: pgmap v575: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3782939148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:34.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:34 smithi017 ceph-mon[215646]: pgmap v575: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 517 op/s; 22887/90381 objects degraded (25.323%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:34.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3782939148' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:35.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:15:35.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:15:35.950 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:35.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:35.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:36.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:36.087 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:36.145 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:36.152 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:36.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:36.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:36 smithi027 ceph-mon[195233]: pgmap v576: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 22887/86898 objects degraded (26.338%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:36.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:36 smithi017 ceph-mon[215646]: pgmap v576: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 22887/86898 objects degraded (26.338%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:36.375 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:36 smithi017 ceph-mon[222878]: pgmap v576: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.8 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 516 op/s; 22887/86898 objects degraded (26.338%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T15:15:36.382 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:36.712 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:36.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:36.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:36.903 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:36.959 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:37.026 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:37.190 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:37.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:37.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:37.226 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22887/86898 objects degraded (26.338%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:37.226 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22887/86898 objects degraded (26.338%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:37.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:37.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22887/86898 objects degraded (26.338%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:37.328 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:37.382 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:37.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:37.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:37.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:37.590 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:37.929 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:37.960 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:15:38.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:38 smithi017 ceph-mon[215646]: pgmap v577: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:38.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:38] "GET /metrics HTTP/1.1" 200 36318 "" "Prometheus/2.43.0" 2023-09-23T15:15:38.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:38 smithi017 ceph-mon[222878]: pgmap v577: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:38.216 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:38.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:38.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:38.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:38.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:38 smithi027 ceph-mon[195233]: pgmap v577: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:38.429 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:38.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:15:39.215 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:39.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:39.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:39.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:39.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:39 smithi027 ceph-mon[195233]: from='client.45341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:39.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:39 smithi017 ceph-mon[215646]: from='client.45341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:39.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:39 smithi017 ceph-mon[222878]: from='client.45341 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:39.369 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:40.240 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:40.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:40.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:40.258 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":577,"stamp":"2023-09-23T15:15:36.405085+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1802082524,"num_objects":27453,"num_object_clones":0,"num_object_copies":82359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22741,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27453,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":174380,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7021,"num_bytes_recovered":461609127,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124765,"ondisk_log_size":124765,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6870404,"kb_used_data":5222860,"kb_used_omap":30,"kb_used_meta":1647393,"kb_avail":649345660,"statfs":{"total":671965249536,"available":664929955840,"internally_reserved":0,"allocated":5348208640,"data_stored":5342409331,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1686931032},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,2,3,9,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-344260608,"num_objects":-5253,"num_object_clones":0,"num_object_copies":-15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-274,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5253,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":274,"num_bytes_recovered":17956864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006882"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":205,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101794+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:15:20.101794+0000","last_peered":"2023-09-23T15:15:20.101794+0000","last_clean":"2023-09-23T15:15:20.101794+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:15:20.101794+0000","last_undegraded":"2023-09-23T15:15:20.101794+0000","last_fullsized":"2023-09-23T15:15:20.101794+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":139,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.173617+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:15:20.173617+0000","last_peered":"2023-09-23T15:15:20.173617+0000","last_clean":"2023-09-23T15:15:20.173617+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:15:20.173617+0000","last_undegraded":"2023-09-23T15:15:20.173617+0000","last_fullsized":"2023-09-23T15:15:20.173617+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":88,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.140174+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:15:20.140174+0000","last_peered":"2023-09-23T15:15:20.140174+0000","last_clean":"2023-09-23T15:15:20.140174+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:15:20.140174+0000","last_undegraded":"2023-09-23T15:15:20.140174+0000","last_fullsized":"2023-09-23T15:15:20.140174+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":176,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138498+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:15:20.138498+0000","last_peered":"2023-09-23T15:15:20.138498+0000","last_clean":"2023-09-23T15:15:20.138498+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:15:20.138498+0000","last_undegraded":"2023-09-23T15:15:20.138498+0000","last_fullsized":"2023-09-23T15:15:20.138498+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":208,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.171811+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:15:20.171811+0000","last_peered":"2023-09-23T15:15:20.171811+0000","last_clean":"2023-09-23T15:15:20.171811+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:15:20.171811+0000","last_undegraded":"2023-09-23T15:15:20.171811+0000","last_fullsized":"2023-09-23T15:15:20.171811+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":185,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.099786+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:15:20.099786+0000","last_peered":"2023-09-23T15:15:20.099786+0000","last_clean":"2023-09-23T15:15:20.099786+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:15:20.099786+0000","last_undegraded":"2023-09-23T15:15:20.099786+0000","last_fullsized":"2023-09-23T15:15:20.099786+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":155,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139063+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:15:20.139063+0000","last_peered":"2023-09-23T15:15:20.139063+0000","last_clean":"2023-09-23T15:15:20.139063+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:15:20.139063+0000","last_undegraded":"2023-09-23T15:15:20.139063+0000","last_fullsized":"2023-09-23T15:15:20.139063+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984624+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:15:30.984624+0000","last_peered":"2023-09-23T15:15:30.984624+0000","last_clean":"2023-09-23T15:15:30.984624+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:15:30.984624+0000","last_undegraded":"2023-09-23T15:15:30.984624+0000","last_fullsized":"2023-09-23T15:15:30.984624+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138083+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:15:20.138083+0000","last_peered":"2023-09-23T15:15:20.138083+0000","last_clean":"2023-09-23T15:15:20.138083+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:15:20.138083+0000","last_undegraded":"2023-09-23T15:15:20.138083+0000","last_fullsized":"2023-09-23T15:15:20.138083+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":201,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.131866+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:15:20.131866+0000","last_peered":"2023-09-23T15:15:20.131866+0000","last_clean":"2023-09-23T15:15:20.131866+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:15:20.131866+0000","last_undegraded":"2023-09-23T15:15:20.131866+0000","last_fullsized":"2023-09-23T15:15:20.131866+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":134,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985211+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:15:30.985211+0000","last_peered":"2023-09-23T15:15:30.985211+0000","last_clean":"2023-09-23T15:15:30.985211+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:15:30.985211+0000","last_undegraded":"2023-09-23T15:15:30.985211+0000","last_fullsized":"2023-09-23T15:15:30.985211+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.173517+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:15:20.173517+0000","last_peered":"2023-09-23T15:15:20.173517+0000","last_clean":"2023-09-23T15:15:20.173517+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:15:20.173517+0000","last_undegraded":"2023-09-23T15:15:20.173517+0000","last_fullsized":"2023-09-23T15:15:20.173517+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":83,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137594+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:20.137594+0000","last_peered":"2023-09-23T15:15:20.137594+0000","last_clean":"2023-09-23T15:15:20.137594+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:20.137594+0000","last_undegraded":"2023-09-23T15:15:20.137594+0000","last_fullsized":"2023-09-23T15:15:20.137594+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":195,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101985+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:15:20.101985+0000","last_peered":"2023-09-23T15:15:20.101985+0000","last_clean":"2023-09-23T15:15:20.101985+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:15:20.101985+0000","last_undegraded":"2023-09-23T15:15:20.101985+0000","last_fullsized":"2023-09-23T15:15:20.101985+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101707+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:15:20.101707+0000","last_peered":"2023-09-23T15:15:20.101707+0000","last_clean":"2023-09-23T15:15:20.101707+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:15:20.101707+0000","last_undegraded":"2023-09-23T15:15:20.101707+0000","last_fullsized":"2023-09-23T15:15:20.101707+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139021+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:15:20.139021+0000","last_peered":"2023-09-23T15:15:20.139021+0000","last_clean":"2023-09-23T15:15:20.139021+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:15:20.139021+0000","last_undegraded":"2023-09-23T15:15:20.139021+0000","last_fullsized":"2023-09-23T15:15:20.139021+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":73,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101793+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:15:20.101793+0000","last_peered":"2023-09-23T15:15:20.101793+0000","last_clean":"2023-09-23T15:15:20.101793+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:15:20.101793+0000","last_undegraded":"2023-09-23T15:15:20.101793+0000","last_fullsized":"2023-09-23T15:15:20.101793+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":202,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.988268+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:15:30.988268+0000","last_peered":"2023-09-23T15:15:30.988268+0000","last_clean":"2023-09-23T15:15:30.988268+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:15:30.988268+0000","last_undegraded":"2023-09-23T15:15:30.988268+0000","last_fullsized":"2023-09-23T15:15:30.988268+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138478+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:15:20.138478+0000","last_peered":"2023-09-23T15:15:20.138478+0000","last_clean":"2023-09-23T15:15:20.138478+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:15:20.138478+0000","last_undegraded":"2023-09-23T15:15:20.138478+0000","last_fullsized":"2023-09-23T15:15:20.138478+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":175,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.140846+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:15:20.140846+0000","last_peered":"2023-09-23T15:15:20.140846+0000","last_clean":"2023-09-23T15:15:20.140846+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:15:20.140846+0000","last_undegraded":"2023-09-23T15:15:20.140846+0000","last_fullsized":"2023-09-23T15:15:20.140846+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":179,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.135371+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:15:20.135371+0000","last_peered":"2023-09-23T15:15:20.135371+0000","last_clean":"2023-09-23T15:15:20.135371+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:15:20.135371+0000","last_undegraded":"2023-09-23T15:15:20.135371+0000","last_fullsized":"2023-09-23T15:15:20.135371+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137170+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:15:20.137170+0000","last_peered":"2023-09-23T15:15:20.137170+0000","last_clean":"2023-09-23T15:15:20.137170+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:15:20.137170+0000","last_undegraded":"2023-09-23T15:15:20.137170+0000","last_fullsized":"2023-09-23T15:15:20.137170+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":57,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101741+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:15:20.101741+0000","last_peered":"2023-09-23T15:15:20.101741+0000","last_clean":"2023-09-23T15:15:20.101741+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:15:20.101741+0000","last_undegraded":"2023-09-23T15:15:20.101741+0000","last_fullsized":"2023-09-23T15:15:20.101741+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":164,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138571+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:15:20.138571+0000","last_peered":"2023-09-23T15:15:20.138571+0000","last_clean":"2023-09-23T15:15:20.138571+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:15:20.138571+0000","last_undegraded":"2023-09-23T15:15:20.138571+0000","last_fullsized":"2023-09-23T15:15:20.138571+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.141190+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:15:20.141190+0000","last_peered":"2023-09-23T15:15:20.141190+0000","last_clean":"2023-09-23T15:15:20.141190+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:15:20.141190+0000","last_undegraded":"2023-09-23T15:15:20.141190+0000","last_fullsized":"2023-09-23T15:15:20.141190+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":177,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.098818+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:15:20.098818+0000","last_peered":"2023-09-23T15:15:20.098818+0000","last_clean":"2023-09-23T15:15:20.098818+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:15:20.098818+0000","last_undegraded":"2023-09-23T15:15:20.098818+0000","last_fullsized":"2023-09-23T15:15:20.098818+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":201,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.173129+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:15:20.173129+0000","last_peered":"2023-09-23T15:15:20.173129+0000","last_clean":"2023-09-23T15:15:20.173129+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:15:20.173129+0000","last_undegraded":"2023-09-23T15:15:20.173129+0000","last_fullsized":"2023-09-23T15:15:20.173129+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"211'10769","reported_seq":10817,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.635609+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:15:34.635609+0000","last_peered":"2023-09-23T15:15:34.635609+0000","last_clean":"2023-09-23T15:15:34.635609+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:15:34.635609+0000","last_undegraded":"2023-09-23T15:15:34.635609+0000","last_fullsized":"2023-09-23T15:15:34.635609+0000","mapping_epoch":171,"log_start":"175'700","ondisk_log_start":"175'700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10069,"log_dups_size":0,"ondisk_log_size":10069,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106496000,"num_objects":1625,"num_object_clones":0,"num_object_copies":4875,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1625,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10769,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.140439+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:15:20.140439+0000","last_peered":"2023-09-23T15:15:20.140439+0000","last_clean":"2023-09-23T15:15:20.140439+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:15:20.140439+0000","last_undegraded":"2023-09-23T15:15:20.140439+0000","last_fullsized":"2023-09-23T15:15:20.140439+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":85,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.102107+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:15:20.102107+0000","last_peered":"2023-09-23T15:15:20.102107+0000","last_clean":"2023-09-23T15:15:20.102107+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:15:20.102107+0000","last_undegraded":"2023-09-23T15:15:20.102107+0000","last_fullsized":"2023-09-23T15:15:20.102107+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":162,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.131461+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:15:20.131461+0000","last_peered":"2023-09-23T15:15:20.131461+0000","last_clean":"2023-09-23T15:15:20.131461+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:15:20.131461+0000","last_undegraded":"2023-09-23T15:15:20.131461+0000","last_fullsized":"2023-09-23T15:15:20.131461+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"211'10985","reported_seq":15166,"reported_epoch":211,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:34.686478+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:15:34.686478+0000","last_peered":"2023-09-23T15:15:34.686478+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:15:34.686478+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6385,"log_dups_size":0,"ondisk_log_size":6385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2936,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10985,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2062,"num_bytes_recovered":135135232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1255}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":478,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:20.137849+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:15:20.137849+0000","last_peered":"2023-09-23T15:15:20.137849+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:15:20.137849+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":85,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.102082+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:15:20.102082+0000","last_peered":"2023-09-23T15:15:20.102082+0000","last_clean":"2023-09-23T15:15:20.102082+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:15:20.102082+0000","last_undegraded":"2023-09-23T15:15:20.102082+0000","last_fullsized":"2023-09-23T15:15:20.102082+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101582+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:15:20.101582+0000","last_peered":"2023-09-23T15:15:20.101582+0000","last_clean":"2023-09-23T15:15:20.101582+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:15:20.101582+0000","last_undegraded":"2023-09-23T15:15:20.101582+0000","last_fullsized":"2023-09-23T15:15:20.101582+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"211'10679","reported_seq":10729,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:33.488141+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:15:33.488141+0000","last_peered":"2023-09-23T15:15:33.488141+0000","last_clean":"2023-09-23T15:15:33.488141+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:15:33.488141+0000","last_undegraded":"2023-09-23T15:15:33.488141+0000","last_fullsized":"2023-09-23T15:15:33.488141+0000","mapping_epoch":171,"log_start":"175'1500","ondisk_log_start":"175'1500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9179,"log_dups_size":0,"ondisk_log_size":9179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115539968,"num_objects":1763,"num_object_clones":0,"num_object_copies":5289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10679,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138751+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:15:20.138751+0000","last_peered":"2023-09-23T15:15:20.138751+0000","last_clean":"2023-09-23T15:15:20.138751+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:15:20.138751+0000","last_undegraded":"2023-09-23T15:15:20.138751+0000","last_fullsized":"2023-09-23T15:15:20.138751+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":71,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.135401+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:15:20.135401+0000","last_peered":"2023-09-23T15:15:20.135401+0000","last_clean":"2023-09-23T15:15:20.135401+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:15:20.135401+0000","last_undegraded":"2023-09-23T15:15:20.135401+0000","last_fullsized":"2023-09-23T15:15:20.135401+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":199,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.099050+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:15:20.099050+0000","last_peered":"2023-09-23T15:15:20.099050+0000","last_clean":"2023-09-23T15:15:20.099050+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:15:20.099050+0000","last_undegraded":"2023-09-23T15:15:20.099050+0000","last_fullsized":"2023-09-23T15:15:20.099050+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"211'10918","reported_seq":10967,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.461547+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:15:34.461547+0000","last_peered":"2023-09-23T15:15:34.461547+0000","last_clean":"2023-09-23T15:15:34.461547+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:15:34.461547+0000","last_undegraded":"2023-09-23T15:15:34.461547+0000","last_fullsized":"2023-09-23T15:15:34.461547+0000","mapping_epoch":171,"log_start":"189'5300","ondisk_log_start":"189'5300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5618,"log_dups_size":0,"ondisk_log_size":5618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114425856,"num_objects":1746,"num_object_clones":0,"num_object_copies":5238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10918,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138268+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:15:20.138268+0000","last_peered":"2023-09-23T15:15:20.138268+0000","last_clean":"2023-09-23T15:15:20.138268+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:15:20.138268+0000","last_undegraded":"2023-09-23T15:15:20.138268+0000","last_fullsized":"2023-09-23T15:15:20.138268+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.132679+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:20.132679+0000","last_peered":"2023-09-23T15:15:20.132679+0000","last_clean":"2023-09-23T15:15:20.132679+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:15:20.132679+0000","last_undegraded":"2023-09-23T15:15:20.132679+0000","last_fullsized":"2023-09-23T15:15:20.132679+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137422+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:15:20.137422+0000","last_peered":"2023-09-23T15:15:20.137422+0000","last_clean":"2023-09-23T15:15:20.137422+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:15:20.137422+0000","last_undegraded":"2023-09-23T15:15:20.137422+0000","last_fullsized":"2023-09-23T15:15:20.137422+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"211'11109","reported_seq":11163,"reported_epoch":211,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:34.288669+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:15:34.288669+0000","last_peered":"2023-09-23T15:15:34.288669+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:15:34.288669+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"178'3600","ondisk_log_start":"178'3600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7509,"log_dups_size":0,"ondisk_log_size":7509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118161408,"num_objects":1803,"num_object_clones":0,"num_object_copies":5409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11109,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-3207}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138933+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:15:20.138933+0000","last_peered":"2023-09-23T15:15:20.138933+0000","last_clean":"2023-09-23T15:15:20.138933+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:15:20.138933+0000","last_undegraded":"2023-09-23T15:15:20.138933+0000","last_fullsized":"2023-09-23T15:15:20.138933+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":174,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.175725+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:15:20.175725+0000","last_peered":"2023-09-23T15:15:20.175725+0000","last_clean":"2023-09-23T15:15:20.175725+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:15:20.175725+0000","last_undegraded":"2023-09-23T15:15:20.175725+0000","last_fullsized":"2023-09-23T15:15:20.175725+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":169,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.102071+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:15:20.102071+0000","last_peered":"2023-09-23T15:15:20.102071+0000","last_clean":"2023-09-23T15:15:20.102071+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:15:20.102071+0000","last_undegraded":"2023-09-23T15:15:20.102071+0000","last_fullsized":"2023-09-23T15:15:20.102071+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"211'10851","reported_seq":10901,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:33.418528+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:15:33.418528+0000","last_peered":"2023-09-23T15:15:33.418528+0000","last_clean":"2023-09-23T15:15:33.418528+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:15:33.418528+0000","last_undegraded":"2023-09-23T15:15:33.418528+0000","last_fullsized":"2023-09-23T15:15:33.418528+0000","mapping_epoch":171,"log_start":"175'1700","ondisk_log_start":"175'1700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9151,"log_dups_size":0,"ondisk_log_size":9151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113180672,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10851,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":504,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138529+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:15:20.138529+0000","last_peered":"2023-09-23T15:15:20.138529+0000","last_clean":"2023-09-23T15:15:20.138529+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:15:20.138529+0000","last_undegraded":"2023-09-23T15:15:20.138529+0000","last_fullsized":"2023-09-23T15:15:20.138529+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.098950+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:15:20.098950+0000","last_peered":"2023-09-23T15:15:20.098950+0000","last_clean":"2023-09-23T15:15:20.098950+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:15:20.098950+0000","last_undegraded":"2023-09-23T15:15:20.098950+0000","last_fullsized":"2023-09-23T15:15:20.098950+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":203,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101932+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:15:20.101932+0000","last_peered":"2023-09-23T15:15:20.101932+0000","last_clean":"2023-09-23T15:15:20.101932+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:15:20.101932+0000","last_undegraded":"2023-09-23T15:15:20.101932+0000","last_fullsized":"2023-09-23T15:15:20.101932+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"211'10875","reported_seq":10924,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.535305+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:15:34.535305+0000","last_peered":"2023-09-23T15:15:34.535305+0000","last_clean":"2023-09-23T15:15:34.535305+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:15:34.535305+0000","last_undegraded":"2023-09-23T15:15:34.535305+0000","last_fullsized":"2023-09-23T15:15:34.535305+0000","mapping_epoch":171,"log_start":"180'4400","ondisk_log_start":"180'4400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6475,"log_dups_size":0,"ondisk_log_size":6475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111607808,"num_objects":1703,"num_object_clones":0,"num_object_copies":5109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10875,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137255+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:15:20.137255+0000","last_peered":"2023-09-23T15:15:20.137255+0000","last_clean":"2023-09-23T15:15:20.137255+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:15:20.137255+0000","last_undegraded":"2023-09-23T15:15:20.137255+0000","last_fullsized":"2023-09-23T15:15:20.137255+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101477+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:15:20.101477+0000","last_peered":"2023-09-23T15:15:20.101477+0000","last_clean":"2023-09-23T15:15:20.101477+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:15:20.101477+0000","last_undegraded":"2023-09-23T15:15:20.101477+0000","last_fullsized":"2023-09-23T15:15:20.101477+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.102159+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:15:20.102159+0000","last_peered":"2023-09-23T15:15:20.102159+0000","last_clean":"2023-09-23T15:15:20.102159+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:15:20.102159+0000","last_undegraded":"2023-09-23T15:15:20.102159+0000","last_fullsized":"2023-09-23T15:15:20.102159+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"211'10836","reported_seq":20705,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.640375+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:15:34.640375+0000","last_peered":"2023-09-23T15:15:34.640375+0000","last_clean":"2023-09-23T15:15:34.640375+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:15:34.640375+0000","last_undegraded":"2023-09-23T15:15:34.640375+0000","last_fullsized":"2023-09-23T15:15:34.640375+0000","mapping_epoch":203,"log_start":"175'800","ondisk_log_start":"175'800","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10036,"log_dups_size":0,"ondisk_log_size":10036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111083544,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10836,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.141097+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:15:20.141097+0000","last_peered":"2023-09-23T15:15:20.141097+0000","last_clean":"2023-09-23T15:15:20.141097+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:15:20.141097+0000","last_undegraded":"2023-09-23T15:15:20.141097+0000","last_fullsized":"2023-09-23T15:15:20.141097+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":203,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.171733+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:15:20.171733+0000","last_peered":"2023-09-23T15:15:20.171733+0000","last_clean":"2023-09-23T15:15:20.171733+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:15:20.171733+0000","last_undegraded":"2023-09-23T15:15:20.171733+0000","last_fullsized":"2023-09-23T15:15:20.171733+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":88,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138028+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:20.138028+0000","last_peered":"2023-09-23T15:15:20.138028+0000","last_clean":"2023-09-23T15:15:20.138028+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:20.138028+0000","last_undegraded":"2023-09-23T15:15:20.138028+0000","last_fullsized":"2023-09-23T15:15:20.138028+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"211'10878","reported_seq":10928,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.391933+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:15:34.391933+0000","last_peered":"2023-09-23T15:15:34.391933+0000","last_clean":"2023-09-23T15:15:34.391933+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:15:34.391933+0000","last_undegraded":"2023-09-23T15:15:34.391933+0000","last_fullsized":"2023-09-23T15:15:34.391933+0000","mapping_epoch":171,"log_start":"189'5300","ondisk_log_start":"189'5300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5578,"log_dups_size":0,"ondisk_log_size":5578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110755840,"num_objects":1690,"num_object_clones":0,"num_object_copies":5070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10878,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138835+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:15:20.138835+0000","last_peered":"2023-09-23T15:15:20.138835+0000","last_clean":"2023-09-23T15:15:20.138835+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:15:20.138835+0000","last_undegraded":"2023-09-23T15:15:20.138835+0000","last_fullsized":"2023-09-23T15:15:20.138835+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":387,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139646+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:15:20.139646+0000","last_peered":"2023-09-23T15:15:20.139646+0000","last_clean":"2023-09-23T15:15:20.139646+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:15:20.139646+0000","last_undegraded":"2023-09-23T15:15:20.139646+0000","last_fullsized":"2023-09-23T15:15:20.139646+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":187,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.135327+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:20.135327+0000","last_peered":"2023-09-23T15:15:20.135327+0000","last_clean":"2023-09-23T15:15:20.135327+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:15:20.135327+0000","last_undegraded":"2023-09-23T15:15:20.135327+0000","last_fullsized":"2023-09-23T15:15:20.135327+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":132,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.130714+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:15:20.130714+0000","last_peered":"2023-09-23T15:15:20.130714+0000","last_clean":"2023-09-23T15:15:20.130714+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:15:20.130714+0000","last_undegraded":"2023-09-23T15:15:20.130714+0000","last_fullsized":"2023-09-23T15:15:20.130714+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"211'10673","reported_seq":10723,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.563098+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:15:34.563098+0000","last_peered":"2023-09-23T15:15:34.563098+0000","last_clean":"2023-09-23T15:15:34.563098+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:15:34.563098+0000","last_undegraded":"2023-09-23T15:15:34.563098+0000","last_fullsized":"2023-09-23T15:15:34.563098+0000","mapping_epoch":171,"log_start":"180'4200","ondisk_log_start":"180'4200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10673,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139065+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:15:20.139065+0000","last_peered":"2023-09-23T15:15:20.139065+0000","last_clean":"2023-09-23T15:15:20.139065+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:15:20.139065+0000","last_undegraded":"2023-09-23T15:15:20.139065+0000","last_fullsized":"2023-09-23T15:15:20.139065+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":729,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.134717+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:15:20.134717+0000","last_peered":"2023-09-23T15:15:20.134717+0000","last_clean":"2023-09-23T15:15:20.134717+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:15:20.134717+0000","last_undegraded":"2023-09-23T15:15:20.134717+0000","last_fullsized":"2023-09-23T15:15:20.134717+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":170,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101677+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:15:20.101677+0000","last_peered":"2023-09-23T15:15:20.101677+0000","last_clean":"2023-09-23T15:15:20.101677+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:15:20.101677+0000","last_undegraded":"2023-09-23T15:15:20.101677+0000","last_fullsized":"2023-09-23T15:15:20.101677+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":103,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139411+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:15:20.139411+0000","last_peered":"2023-09-23T15:15:20.139411+0000","last_clean":"2023-09-23T15:15:20.139411+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:15:20.139411+0000","last_undegraded":"2023-09-23T15:15:20.139411+0000","last_fullsized":"2023-09-23T15:15:20.139411+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"211'10885","reported_seq":10933,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.144398+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:15:34.144398+0000","last_peered":"2023-09-23T15:15:34.144398+0000","last_clean":"2023-09-23T15:15:34.144398+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:15:34.144398+0000","last_undegraded":"2023-09-23T15:15:34.144398+0000","last_fullsized":"2023-09-23T15:15:34.144398+0000","mapping_epoch":171,"log_start":"191'5600","ondisk_log_start":"191'5600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5285,"log_dups_size":0,"ondisk_log_size":5285,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114098176,"num_objects":1741,"num_object_clones":0,"num_object_copies":5223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10885,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":499,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.140944+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:15:20.140944+0000","last_peered":"2023-09-23T15:15:20.140944+0000","last_clean":"2023-09-23T15:15:20.140944+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:15:20.140944+0000","last_undegraded":"2023-09-23T15:15:20.140944+0000","last_fullsized":"2023-09-23T15:15:20.140944+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":823,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.131955+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:15:20.131955+0000","last_peered":"2023-09-23T15:15:20.131955+0000","last_clean":"2023-09-23T15:15:20.131955+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:15:20.131955+0000","last_undegraded":"2023-09-23T15:15:20.131955+0000","last_fullsized":"2023-09-23T15:15:20.131955+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":83,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137583+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:15:20.137583+0000","last_peered":"2023-09-23T15:15:20.137583+0000","last_clean":"2023-09-23T15:15:20.137583+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:15:20.137583+0000","last_undegraded":"2023-09-23T15:15:20.137583+0000","last_fullsized":"2023-09-23T15:15:20.137583+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":107,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137955+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:15:20.137955+0000","last_peered":"2023-09-23T15:15:20.137955+0000","last_clean":"2023-09-23T15:15:20.137955+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:15:20.137955+0000","last_undegraded":"2023-09-23T15:15:20.137955+0000","last_fullsized":"2023-09-23T15:15:20.137955+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"211'10697","reported_seq":10745,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.628741+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:15:34.628741+0000","last_peered":"2023-09-23T15:15:34.628741+0000","last_clean":"2023-09-23T15:15:34.628741+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:15:34.628741+0000","last_undegraded":"2023-09-23T15:15:34.628741+0000","last_fullsized":"2023-09-23T15:15:34.628741+0000","mapping_epoch":171,"log_start":"175'600","ondisk_log_start":"175'600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10097,"log_dups_size":0,"ondisk_log_size":10097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110297088,"num_objects":1683,"num_object_clones":0,"num_object_copies":5049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10697,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":498,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.131801+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:15:20.131801+0000","last_peered":"2023-09-23T15:15:20.131801+0000","last_clean":"2023-09-23T15:15:20.131801+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:15:20.131801+0000","last_undegraded":"2023-09-23T15:15:20.131801+0000","last_fullsized":"2023-09-23T15:15:20.131801+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007133+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:15:31.007133+0000","last_peered":"2023-09-23T15:15:31.007133+0000","last_clean":"2023-09-23T15:15:31.007133+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:15:31.007133+0000","last_undegraded":"2023-09-23T15:15:31.007133+0000","last_fullsized":"2023-09-23T15:15:31.007133+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":125,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.172600+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:15:20.172600+0000","last_peered":"2023-09-23T15:15:20.172600+0000","last_clean":"2023-09-23T15:15:20.172600+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:15:20.172600+0000","last_undegraded":"2023-09-23T15:15:20.172600+0000","last_fullsized":"2023-09-23T15:15:20.172600+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":161,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.099344+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:15:20.099344+0000","last_peered":"2023-09-23T15:15:20.099344+0000","last_clean":"2023-09-23T15:15:20.099344+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:15:20.099344+0000","last_undegraded":"2023-09-23T15:15:20.099344+0000","last_fullsized":"2023-09-23T15:15:20.099344+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"211'10916","reported_seq":10967,"reported_epoch":211,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:34.616040+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:15:34.616040+0000","last_peered":"2023-09-23T15:15:34.616040+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:15:34.616040+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"180'4500","ondisk_log_start":"180'4500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6416,"log_dups_size":0,"ondisk_log_size":6416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113246208,"num_objects":1728,"num_object_clones":0,"num_object_copies":5184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10916,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-3224}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137331+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:15:20.137331+0000","last_peered":"2023-09-23T15:15:20.137331+0000","last_clean":"2023-09-23T15:15:20.137331+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:15:20.137331+0000","last_undegraded":"2023-09-23T15:15:20.137331+0000","last_fullsized":"2023-09-23T15:15:20.137331+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":294,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137325+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:15:20.137325+0000","last_peered":"2023-09-23T15:15:20.137325+0000","last_clean":"2023-09-23T15:15:20.137325+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:15:20.137325+0000","last_undegraded":"2023-09-23T15:15:20.137325+0000","last_fullsized":"2023-09-23T15:15:20.137325+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":156,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138976+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:20.138976+0000","last_peered":"2023-09-23T15:15:20.138976+0000","last_clean":"2023-09-23T15:15:20.138976+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:20.138976+0000","last_undegraded":"2023-09-23T15:15:20.138976+0000","last_fullsized":"2023-09-23T15:15:20.138976+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:29.461887+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:15:29.461887+0000","last_peered":"2023-09-23T15:15:29.461887+0000","last_clean":"2023-09-23T15:15:29.461887+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:15:29.461887+0000","last_undegraded":"2023-09-23T15:15:29.461887+0000","last_fullsized":"2023-09-23T15:15:29.461887+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"211'10749","reported_seq":10799,"reported_epoch":211,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:33.517389+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:15:33.517389+0000","last_peered":"2023-09-23T15:15:33.517389+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:15:33.517389+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"175'1600","ondisk_log_start":"175'1600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9149,"log_dups_size":0,"ondisk_log_size":9149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115277824,"num_objects":1759,"num_object_clones":0,"num_object_copies":5277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10749,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-3115}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138652+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:15:20.138652+0000","last_peered":"2023-09-23T15:15:20.138652+0000","last_clean":"2023-09-23T15:15:20.138652+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:15:20.138652+0000","last_undegraded":"2023-09-23T15:15:20.138652+0000","last_fullsized":"2023-09-23T15:15:20.138652+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":816,"reported_epoch":210,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:20.140261+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:15:20.140261+0000","last_peered":"2023-09-23T15:15:20.140261+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:15:20.140261+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139568+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:15:20.139568+0000","last_peered":"2023-09-23T15:15:20.139568+0000","last_clean":"2023-09-23T15:15:20.139568+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:15:20.139568+0000","last_undegraded":"2023-09-23T15:15:20.139568+0000","last_fullsized":"2023-09-23T15:15:20.139568+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":161,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137627+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:20.137627+0000","last_peered":"2023-09-23T15:15:20.137627+0000","last_clean":"2023-09-23T15:15:20.137627+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:20.137627+0000","last_undegraded":"2023-09-23T15:15:20.137627+0000","last_fullsized":"2023-09-23T15:15:20.137627+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"211'10600","reported_seq":10647,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.357052+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:15:34.357052+0000","last_peered":"2023-09-23T15:15:34.357052+0000","last_clean":"2023-09-23T15:15:34.357052+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:15:34.357052+0000","last_undegraded":"2023-09-23T15:15:34.357052+0000","last_fullsized":"2023-09-23T15:15:34.357052+0000","mapping_epoch":171,"log_start":"178'3000","ondisk_log_start":"178'3000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7600,"log_dups_size":0,"ondisk_log_size":7600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111542272,"num_objects":1702,"num_object_clones":0,"num_object_copies":5106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10600,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.140365+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:15:20.140365+0000","last_peered":"2023-09-23T15:15:20.140365+0000","last_clean":"2023-09-23T15:15:20.140365+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:15:20.140365+0000","last_undegraded":"2023-09-23T15:15:20.140365+0000","last_fullsized":"2023-09-23T15:15:20.140365+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":187,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.172972+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:15:20.172972+0000","last_peered":"2023-09-23T15:15:20.172972+0000","last_clean":"2023-09-23T15:15:20.172972+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:15:20.172972+0000","last_undegraded":"2023-09-23T15:15:20.172972+0000","last_fullsized":"2023-09-23T15:15:20.172972+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":671,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137868+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:15:20.137868+0000","last_peered":"2023-09-23T15:15:20.137868+0000","last_clean":"2023-09-23T15:15:20.137868+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:15:20.137868+0000","last_undegraded":"2023-09-23T15:15:20.137868+0000","last_fullsized":"2023-09-23T15:15:20.137868+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":191,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.171985+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:15:20.171985+0000","last_peered":"2023-09-23T15:15:20.171985+0000","last_clean":"2023-09-23T15:15:20.171985+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:15:20.171985+0000","last_undegraded":"2023-09-23T15:15:20.171985+0000","last_fullsized":"2023-09-23T15:15:20.171985+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"211'10932","reported_seq":10987,"reported_epoch":211,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:34.208647+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:15:34.208647+0000","last_peered":"2023-09-23T15:15:34.208647+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:15:34.208647+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"191'5600","ondisk_log_start":"191'5600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5332,"log_dups_size":0,"ondisk_log_size":5332,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111280128,"num_objects":1698,"num_object_clones":0,"num_object_copies":5094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10932,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-3268}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":487,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.138004+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:15:20.138004+0000","last_peered":"2023-09-23T15:15:20.138004+0000","last_clean":"2023-09-23T15:15:20.138004+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:15:20.138004+0000","last_undegraded":"2023-09-23T15:15:20.138004+0000","last_fullsized":"2023-09-23T15:15:20.138004+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":687,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.099733+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:15:20.099733+0000","last_peered":"2023-09-23T15:15:20.099733+0000","last_clean":"2023-09-23T15:15:20.099733+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:15:20.099733+0000","last_undegraded":"2023-09-23T15:15:20.099733+0000","last_fullsized":"2023-09-23T15:15:20.099733+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":174,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:29.461910+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:15:29.461910+0000","last_peered":"2023-09-23T15:15:29.461910+0000","last_clean":"2023-09-23T15:15:29.461910+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:15:29.461910+0000","last_undegraded":"2023-09-23T15:15:29.461910+0000","last_fullsized":"2023-09-23T15:15:29.461910+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":152,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101571+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:15:20.101571+0000","last_peered":"2023-09-23T15:15:20.101571+0000","last_clean":"2023-09-23T15:15:20.101571+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:15:20.101571+0000","last_undegraded":"2023-09-23T15:15:20.101571+0000","last_fullsized":"2023-09-23T15:15:20.101571+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":163,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.137560+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:15:20.137560+0000","last_peered":"2023-09-23T15:15:20.137560+0000","last_clean":"2023-09-23T15:15:20.137560+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:15:20.137560+0000","last_undegraded":"2023-09-23T15:15:20.137560+0000","last_fullsized":"2023-09-23T15:15:20.137560+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":210,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.132632+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:15:20.132632+0000","last_peered":"2023-09-23T15:15:20.132632+0000","last_clean":"2023-09-23T15:15:20.132632+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:15:20.132632+0000","last_undegraded":"2023-09-23T15:15:20.132632+0000","last_fullsized":"2023-09-23T15:15:20.132632+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":189,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.132831+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:15:20.132831+0000","last_peered":"2023-09-23T15:15:20.132831+0000","last_clean":"2023-09-23T15:15:20.132831+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:15:20.132831+0000","last_undegraded":"2023-09-23T15:15:20.132831+0000","last_fullsized":"2023-09-23T15:15:20.132831+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101775+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:15:20.101775+0000","last_peered":"2023-09-23T15:15:20.101775+0000","last_clean":"2023-09-23T15:15:20.101775+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:15:20.101775+0000","last_undegraded":"2023-09-23T15:15:20.101775+0000","last_fullsized":"2023-09-23T15:15:20.101775+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":155,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.135298+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:15:20.135298+0000","last_peered":"2023-09-23T15:15:20.135298+0000","last_clean":"2023-09-23T15:15:20.135298+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:15:20.135298+0000","last_undegraded":"2023-09-23T15:15:20.135298+0000","last_fullsized":"2023-09-23T15:15:20.135298+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101694+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:15:20.101694+0000","last_peered":"2023-09-23T15:15:20.101694+0000","last_clean":"2023-09-23T15:15:20.101694+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:15:20.101694+0000","last_undegraded":"2023-09-23T15:15:20.101694+0000","last_fullsized":"2023-09-23T15:15:20.101694+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":182,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.173948+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:15:20.173948+0000","last_peered":"2023-09-23T15:15:20.173948+0000","last_clean":"2023-09-23T15:15:20.173948+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:15:20.173948+0000","last_undegraded":"2023-09-23T15:15:20.173948+0000","last_fullsized":"2023-09-23T15:15:20.173948+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":196,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.171080+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:15:20.171080+0000","last_peered":"2023-09-23T15:15:20.171080+0000","last_clean":"2023-09-23T15:15:20.171080+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:15:20.171080+0000","last_undegraded":"2023-09-23T15:15:20.171080+0000","last_fullsized":"2023-09-23T15:15:20.171080+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":159,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.139146+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:20.139146+0000","last_peered":"2023-09-23T15:15:20.139146+0000","last_clean":"2023-09-23T15:15:20.139146+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:20.139146+0000","last_undegraded":"2023-09-23T15:15:20.139146+0000","last_fullsized":"2023-09-23T15:15:20.139146+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":210,"state":"active+clean","last_fresh":"2023-09-23T15:15:20.101465+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:15:20.101465+0000","last_peered":"2023-09-23T15:15:20.101465+0000","last_clean":"2023-09-23T15:15:20.101465+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:15:20.101465+0000","last_undegraded":"2023-09-23T15:15:20.101465+0000","last_fullsized":"2023-09-23T15:15:20.101465+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1797324824,"num_objects":27426,"num_object_clones":0,"num_object_copies":82278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22738,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27426,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173352,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6962,"num_bytes_recovered":456261632,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5276971008,"data_stored":5276958792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120352,"ondisk_log_size":120352,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201530,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463166,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1059464,"kb_used_data":893248,"kb_used_omap":0,"kb_used_meta":166144,"kb_avail":92685688,"statfs":{"total":95995035648,"available":94910144512,"internally_reserved":0,"allocated":914685952,"data_stored":913981576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":170131456},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.617,"15min":2.6779999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":6.367,"5min":41.491,"15min":108.39400000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.8130000000000002,"15min":2.8180000000000001},"min":{"1min":0.29299999999999998,"5min":0.223,"15min":0.223},"max":{"1min":12.634,"5min":43.151000000000003,"15min":108.452},"last":0.436}]},{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":4.5350000000000001,"5min":3.6030000000000002,"15min":3.448},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":43.377000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":1.048},{"interface":"front","average":{"1min":7.2919999999999998,"5min":4.3440000000000003,"15min":3.8519999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":42.704000000000001,"5min":42.704000000000001,"15min":42.704000000000001},"last":5.4480000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":13.1,"5min":6.4619999999999997,"15min":5.702},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":112.352,"5min":112.352,"15min":180.374},"last":2.3340000000000001},{"interface":"front","average":{"1min":13.564,"5min":6.4489999999999998,"15min":5.7329999999999997},"min":{"1min":0.45200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":117.667,"5min":117.667,"15min":181.685},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":12.955,"5min":4.3739999999999997,"15min":1.992},"min":{"1min":0.441,"5min":0.249,"15min":0.249},"max":{"1min":81.087000000000003,"5min":81.087000000000003,"15min":81.087000000000003},"last":3.4750000000000001},{"interface":"front","average":{"1min":12.819000000000001,"5min":4.4420000000000002,"15min":1.982},"min":{"1min":0.374,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":80.977999999999994,"5min":80.977999999999994,"15min":80.977999999999994},"last":1.5549999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4870000000000001,"15min":1.0309999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.75800000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.6040000000000001,"15min":1.121},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7590000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.33400000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":2.5230000000000001,"15min":1.4059999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":2.976,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":2.5310000000000001,"15min":1.452},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7859999999999996,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:14:36 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.381,"15min":1},"min":{"1min":0.33900000000000002,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":3.181,"5min":7.5910000000000002,"15min":17},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.82699999999999996,"5min":1.3939999999999999,"15min":0.97499999999999998},"min":{"1min":0.39700000000000002,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":3.754,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.54300000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015204,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":720860,"kb_used_data":554196,"kb_used_omap":6,"kb_used_meta":166649,"kb_avail":93024292,"statfs":{"total":95995035648,"available":95256875008,"internally_reserved":0,"allocated":567496704,"data_stored":566487909,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":170648651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":3.6080000000000001,"15min":3.738},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.0839999999999996,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.5609999999999999,"5min":4.4000000000000004,"15min":4.54},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.558,"5min":16.131,"15min":16.131},"last":12.592000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0229999999999997,"5min":4.3070000000000004,"15min":4.1870000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.757999999999999,"5min":13.843,"15min":13.843},"last":9.7989999999999995},{"interface":"front","average":{"1min":4.3380000000000001,"5min":4.2460000000000004,"15min":4.2300000000000004},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.97099999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9539999999999997,"5min":9.6270000000000007,"15min":10.406000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.27,"5min":123.426,"15min":123.426},"last":0.54300000000000004},{"interface":"front","average":{"1min":3.5499999999999998,"5min":10.762,"15min":11.964},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1010000000000009,"5min":123.32899999999999,"15min":123.32899999999999},"last":12.564}]},{"osd":4,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.7090000000000001,"15min":2.7000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.7199999999999998,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":9.8409999999999993},{"interface":"front","average":{"1min":3.129,"5min":3.472,"15min":3.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.779,"5min":16.779,"15min":16.779},"last":5.5979999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":3.419,"15min":3.6589999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8259999999999996,"5min":16.166,"15min":16.166},"last":9.2230000000000008},{"interface":"front","average":{"1min":3.0190000000000001,"5min":2.9209999999999998,"15min":2.9049999999999998},"min":{"1min":0.35399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7640000000000002,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":9.8680000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":3.488,"15min":3.702},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.8010000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":9.2530000000000001},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.8050000000000002,"15min":2.7669999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.7930000000000001,"5min":7.7930000000000001,"15min":7.7930000000000001},"last":12.448}]},{"osd":7,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.1749999999999998,"15min":3.1629999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.8179999999999996,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":9.8919999999999995},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2909999999999999,"15min":3.3330000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":1.8580000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893253,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":731760,"kb_used_data":442280,"kb_used_omap":8,"kb_used_meta":289463,"kb_avail":93013392,"statfs":{"total":95995035648,"available":95245713408,"internally_reserved":0,"allocated":452894720,"data_stored":451983163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":296410708},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.875,"15min":4.1529999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.5099999999999998,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.28599999999999998},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.274,"15min":4.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4249999999999998,"5min":31.378,"15min":31.378},"last":7.5590000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":15.058999999999999,"5min":7.9790000000000001,"15min":3.3519999999999999},"min":{"1min":0.311,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":93.873999999999995,"5min":122.20099999999999,"15min":122.20099999999999},"last":6.3849999999999998},{"interface":"front","average":{"1min":13.999000000000001,"5min":6.9569999999999999,"15min":2.891},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":94.652000000000001,"5min":122.071,"15min":122.071},"last":9.1170000000000009}]},{"osd":3,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":8.2420000000000009,"5min":6.8710000000000004,"15min":2.8610000000000002},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":72.137,"5min":99.204999999999998,"15min":99.204999999999998},"last":2.9340000000000002},{"interface":"front","average":{"1min":13.818,"5min":6.9850000000000003,"15min":3.2029999999999998},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":95.795000000000002,"5min":95.795000000000002,"15min":95.795000000000002},"last":5.46}]},{"osd":4,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.9319999999999999,"15min":1.595},"min":{"1min":0.46200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0499999999999998,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.0150000000000001,"15min":1.5980000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9809999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.9420000000000002,"15min":1.6479999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2370000000000001,"5min":109.369,"15min":109.369},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.532,"15min":1.8049999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.1260000000000003,"5min":109.387,"15min":109.387},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.9590000000000001,"15min":1.6319999999999999},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.032,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.171,"5min":2.8359999999999999,"15min":1.5580000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2910000000000004,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.0870000000000002,"15min":1.6759999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2640000000000002,"5min":99.549999999999997,"15min":99.549999999999997},"last":2.141},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.4199999999999999,"15min":1.4890000000000001},"min":{"1min":0.497,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.093,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.5}]}]},{"osd":2,"up_from":105,"seq":450971566200,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1345008,"kb_used_data":1018024,"kb_used_omap":11,"kb_used_meta":326964,"kb_avail":92400144,"statfs":{"total":95995035648,"available":94617747456,"internally_reserved":0,"allocated":1042456576,"data_stored":1041548745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":334811920},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":10.598000000000001,"5min":4.6280000000000001,"15min":1.9299999999999999},"min":{"1min":0.215,"5min":0.184,"15min":0.184},"max":{"1min":87.132999999999996,"5min":87.132999999999996,"15min":87.132999999999996},"last":3.085},{"interface":"front","average":{"1min":7.6420000000000003,"5min":3.6579999999999999,"15min":1.595},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":84.820999999999998,"5min":84.820999999999998,"15min":84.820999999999998},"last":2.9380000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:14:47 2023","interfaces":[{"interface":"back","average":{"1min":9.2129999999999992,"5min":5.8739999999999997,"15min":5.3179999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":84.641000000000005,"5min":84.641000000000005,"15min":84.641000000000005},"last":2.589},{"interface":"front","average":{"1min":9.2040000000000006,"5min":3.8799999999999999,"15min":2.992},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":84.751999999999995,"5min":84.751999999999995,"15min":84.751999999999995},"last":2.3340000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:15:29 2023","interfaces":[{"interface":"back","average":{"1min":14.097,"5min":6.1619999999999999,"15min":2.661},"min":{"1min":0.70399999999999996,"5min":0.219,"15min":0.219},"max":{"1min":86.332999999999998,"5min":87.108000000000004,"15min":87.108000000000004},"last":23.254000000000001},{"interface":"front","average":{"1min":12.151999999999999,"5min":5.577,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":8.8010000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.54,"5min":3.6659999999999999,"15min":1.7649999999999999},"min":{"1min":0.42299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":84.608000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":1.3400000000000001},{"interface":"front","average":{"1min":8.8379999999999992,"5min":4.8460000000000001,"15min":2.254},"min":{"1min":0.41099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":2.2440000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6260000000000003,"5min":3.6459999999999999,"15min":1.8779999999999999},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":84.498000000000005,"5min":84.498000000000005,"15min":84.498000000000005},"last":2.3199999999999998},{"interface":"front","average":{"1min":6.4429999999999996,"5min":3.5600000000000001,"15min":1.825},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":84.664000000000001,"5min":84.664000000000001,"15min":84.664000000000001},"last":1.401}]},{"osd":6,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9589999999999996,"5min":3.9089999999999998,"15min":1.9490000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":84.715999999999994,"5min":84.715999999999994,"15min":84.715999999999994},"last":1.3779999999999999},{"interface":"front","average":{"1min":9.0410000000000004,"5min":3.847,"15min":1.9430000000000001},"min":{"1min":0.44,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":87.055000000000007,"5min":87.055000000000007,"15min":87.055000000000007},"last":2.2029999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8410000000000002,"5min":3.2010000000000001,"15min":1.7350000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":84.596000000000004,"5min":84.596000000000004,"15min":84.596000000000004},"last":2.286},{"interface":"front","average":{"1min":6.7770000000000001,"5min":3.274,"15min":1.7210000000000001},"min":{"1min":0.46999999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":84.783000000000001,"5min":84.783000000000001,"15min":84.783000000000001},"last":1.425}]}]},{"osd":3,"up_from":110,"seq":472446402674,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1515968,"kb_used_data":1210616,"kb_used_omap":3,"kb_used_meta":305340,"kb_avail":92229184,"statfs":{"total":95995035648,"available":94442684416,"internally_reserved":0,"allocated":1239670784,"data_stored":1238801657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":312668329},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":3.3100000000000001,"15min":2.0110000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":13.064,"5min":61.744,"15min":79.611000000000004},"last":13.374000000000001},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.2130000000000001,"15min":2.0249999999999999},"min":{"1min":0.40200000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":12.348000000000001,"5min":61.686,"15min":79.656000000000006},"last":18.190999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":6.2789999999999999,"5min":4.9240000000000004,"15min":4.6980000000000004},"min":{"1min":0.34200000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.873999999999999,"5min":28.315000000000001,"15min":28.315000000000001},"last":12.566000000000001},{"interface":"front","average":{"1min":4.7350000000000003,"5min":4.8230000000000004,"15min":4.8380000000000001},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.395,"5min":14.427,"15min":14.427},"last":0.30199999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8600000000000003,"5min":4.6020000000000003,"15min":3.6909999999999998},"min":{"1min":0.432,"5min":0.22,"15min":0.22},"max":{"1min":61.732999999999997,"5min":63.009999999999998,"15min":109.077},"last":13.342000000000001},{"interface":"front","average":{"1min":8.202,"5min":5.101,"15min":3.9460000000000002},"min":{"1min":0.47999999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":60.305999999999997,"5min":69.715999999999994,"15min":119.72799999999999},"last":16.431000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.6139999999999999,"15min":1.4319999999999999},"min":{"1min":0.36199999999999999,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":54.051000000000002,"5min":54.051000000000002,"15min":54.051000000000002},"last":12.958},{"interface":"front","average":{"1min":5.2939999999999996,"5min":2.7029999999999998,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":52.228999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.34999999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":2.399,"15min":1.5740000000000001},"min":{"1min":0.40899999999999997,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":8.0459999999999994,"5min":8.9450000000000003,"15min":56.960999999999999},"last":12.999000000000001},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.552,"15min":1.806},"min":{"1min":0.49299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.8730000000000002,"5min":10.738,"15min":56.981999999999999},"last":12.667999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":3.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.42899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":78.102000000000004,"5min":78.102000000000004,"15min":78.102000000000004},"last":13.029999999999999},{"interface":"front","average":{"1min":6.5919999999999996,"5min":3.327,"15min":1.962},"min":{"1min":0.69299999999999995,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":78.028000000000006,"5min":78.028000000000006,"15min":78.028000000000006},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":2.5110000000000001,"15min":1.71},"min":{"1min":0.52200000000000002,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":5.923,"5min":10.603,"15min":14.885999999999999},"last":12.701000000000001},{"interface":"front","average":{"1min":3.016,"5min":2.2160000000000002,"15min":1.363},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.109,"5min":14.351000000000001,"15min":14.351000000000001},"last":13.295}]}]},{"osd":4,"up_from":31,"seq":133143986439,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":756100,"kb_used_data":558396,"kb_used_omap":0,"kb_used_meta":197696,"kb_avail":92989052,"statfs":{"total":95995035648,"available":95220789248,"internally_reserved":0,"allocated":571797504,"data_stored":571096703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":202440704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":3.375,"15min":2.5289999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":65.988,"5min":66.605999999999995,"15min":122.69199999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":5.4029999999999996,"5min":3.3639999999999999,"15min":2.6309999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":65.396000000000001,"5min":66.384,"15min":122.745},"last":0.32300000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":4.2539999999999996,"15min":4.4500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.783999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":2.1619999999999999},{"interface":"front","average":{"1min":3.5699999999999998,"5min":4.7750000000000004,"15min":4.976},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":28.815000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":2.77}]},{"osd":2,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.5140000000000002,"15min":2.883},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.670999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":4.5599999999999996},{"interface":"front","average":{"1min":3.0369999999999999,"5min":4.984,"15min":3.1829999999999998},"min":{"1min":0.41399999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":17.756,"5min":147.88900000000001,"15min":147.88900000000001},"last":2.8530000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":9.3309999999999995,"15min":3.7869999999999999},"min":{"1min":0.36799999999999999,"5min":0.252,"15min":0.252},"max":{"1min":96.447000000000003,"5min":163.97499999999999,"15min":163.97499999999999},"last":3.073},{"interface":"front","average":{"1min":7.0019999999999998,"5min":9.1080000000000005,"15min":3.5750000000000002},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":95.944999999999993,"5min":164.447,"15min":164.447},"last":5.0300000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:14:41 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.452,"15min":1.1950000000000001},"min":{"1min":0.25900000000000001,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":1.1479999999999999,"5min":14.073,"15min":28.891999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.208,"15min":1.0600000000000001},"min":{"1min":0.28499999999999998,"5min":0.252,"15min":0.252},"max":{"1min":2.577,"5min":8.9920000000000009,"15min":32.100999999999999},"last":0.38900000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:15:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2169999999999996,"5min":4.2329999999999997,"15min":2.0449999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":86.091999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.40999999999999998},{"interface":"front","average":{"1min":5.4509999999999996,"5min":3.984,"15min":1.9319999999999999},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":86.048000000000002,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.34100000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.91800000000000004,"15min":1.129},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":0.75900000000000001,"5min":7.7389999999999999,"15min":18.23},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.3400000000000001,"15min":1.179},"min":{"1min":0.375,"5min":0.314,"15min":0.314},"max":{"1min":6.1399999999999997,"5min":14.044,"15min":14.653},"last":0.45300000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724802,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":741244,"kb_used_data":546100,"kb_used_omap":0,"kb_used_meta":195136,"kb_avail":93003908,"statfs":{"total":95995035648,"available":95236001792,"internally_reserved":0,"allocated":559206400,"data_stored":558509578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":199819264},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1829999999999998,"15min":1.292},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.9530000000000003,"5min":17.044,"15min":24.962},"last":4.9500000000000002},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.9039999999999999,"15min":1.226},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":6.2549999999999999,"5min":18.814,"15min":25.317},"last":12.531000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5030000000000001,"5min":2.2389999999999999,"15min":1.861},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.082999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":6.4560000000000004},{"interface":"front","average":{"1min":2.9510000000000001,"5min":2.4049999999999998,"15min":2.3140000000000001},"min":{"1min":0.39200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":17.963999999999999,"5min":17.963999999999999,"15min":17.963999999999999},"last":6.9740000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":4.7460000000000004,"15min":2.2989999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":33.68,"5min":97.087999999999994,"15min":97.087999999999994},"last":0.46600000000000003},{"interface":"front","average":{"1min":4.0510000000000002,"5min":4.8639999999999999,"15min":2.2930000000000001},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":33.707999999999998,"5min":97.305999999999997,"15min":97.305999999999997},"last":0.49199999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":5.4550000000000001,"5min":3.7829999999999999,"15min":1.8360000000000001},"min":{"1min":0.31900000000000001,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":32.305999999999997,"5min":53.088999999999999,"15min":53.088999999999999},"last":12.974},{"interface":"front","average":{"1min":5.0970000000000004,"5min":3.5249999999999999,"15min":1.734},"min":{"1min":0.46400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":24.052,"5min":53.764000000000003,"15min":53.764000000000003},"last":12.738}]},{"osd":4,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":2.1480000000000001,"15min":1.1639999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.369,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.2349999999999999,"15min":1.238},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.88200000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.34300000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.494,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":11.076000000000001,"15min":15.196},"last":0.371},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.3859999999999999,"15min":1.012},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":1.367,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.437}]},{"osd":7,"last update":"Sat Sep 23 15:14:31 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":1.8580000000000001,"15min":1.228},"min":{"1min":0.36099999999999999,"5min":0.33300000000000002,"15min":0.28299999999999997},"max":{"1min":1.331,"5min":11.353999999999999,"15min":15.301},"last":0.504},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.9119999999999999,"15min":1.212},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":1.8460000000000001,"5min":15.762,"15min":15.762},"last":0.46000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":443813888,"data_stored":443809816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":558567424,"data_stored":558563352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1035141120,"data_stored":1035141120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1227292672,"data_stored":1227292672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":562626560,"data_stored":562626560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":552472576,"data_stored":552468504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":897056768,"data_stored":897056768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:15:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:40 smithi027 ceph-mon[195233]: pgmap v578: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:40.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:40.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:40 smithi017 ceph-mon[215646]: pgmap v578: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:40.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:40 smithi017 ceph-mon[222878]: pgmap v578: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 524 op/s; 22741/82359 objects degraded (27.612%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T15:15:40.429 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:40.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:40.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:40.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:40.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:40.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:40.898 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:40.953 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:40.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:40.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:41.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:41.062 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:15:41.062 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:15:41.090 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:41.142 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:41.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:41.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:41.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:41.345 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:41.811 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:41.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:41.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:41.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:42.034 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:42.115 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:15:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:42 smithi027 ceph-mon[195233]: pgmap v579: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/217240637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22600/74154 objects degraded (30.477%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:42.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[215646]: pgmap v579: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:42.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/217240637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22600/74154 objects degraded (30.477%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:42.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[222878]: pgmap v579: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:42.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/217240637' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:15:42.382 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22600/74154 objects degraded (30.477%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:43.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:43.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[215646]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[215646]: failed to encode map e212 with expected crc 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[222878]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T15:15:43.191 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:43 smithi017 ceph-mon[222878]: failed to encode map e212 with expected crc 2023-09-23T15:15:43.222 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:43.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:15:43.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:43.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:43.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:15:43.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:43 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:15:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:43 smithi027 ceph-mon[195233]: osdmap e212: 8 total, 8 up, 7 in 2023-09-23T15:15:43.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:43 smithi027 ceph-mon[195233]: failed to encode map e212 with expected crc 2023-09-23T15:15:43.469 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:43.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:43.731 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:43.757 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:44.057 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:44.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:44.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:44.169 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:44.199 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:44.254 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:44.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:44.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:44.350 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:44.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:44.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[215646]: pgmap v581: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:44.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:44.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:44.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[222878]: pgmap v581: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:44.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:44.437 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:44.440 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:44.476 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:44 smithi027 ceph-mon[195233]: pgmap v581: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.1 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 540 op/s; 22600/74154 objects degraded (30.477%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:15:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:44.805 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:44.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:44.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:44.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:45.029 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:45.073 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:45.125 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:45.192 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:45.225 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:45.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:45.249 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:45.293 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:45.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:45.334 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:45.361 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:45.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:45.413 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:45.414 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:45.419 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/71337024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:45.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:45.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:45.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.437 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.438 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.438 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.438 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.438 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:15:45.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/71337024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:45.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/71337024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:45.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:15:45.520 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:45.546 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:45.590 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:45.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:45.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:45.793 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:46.131 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:46.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:46.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:46.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:46.336 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:46 smithi017 ceph-mon[215646]: pgmap v582: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 22600/70596 objects degraded (32.013%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:46.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:46 smithi017 ceph-mon[222878]: pgmap v582: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 22600/70596 objects degraded (32.013%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:46.346 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:46.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:46 smithi027 ceph-mon[195233]: pgmap v582: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 22600/70596 objects degraded (32.013%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:15:46.737 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:46.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:46.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:46.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:46.944 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:47.071 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:47.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:47.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:47.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:47.203 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:47.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:47 smithi017 ceph-mon[215646]: pgmap v583: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:47.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22600/70596 objects degraded (32.013%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:47 smithi017 ceph-mon[222878]: pgmap v583: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:47.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22600/70596 objects degraded (32.013%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:47 smithi027 ceph-mon[195233]: pgmap v583: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22600/70596 objects degraded (32.013%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:47.687 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:47.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:47.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:47.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:47.825 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:47.880 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:47.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:47.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:47.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:48.034 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:15:48.034 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:15:48.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.034 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:180590 6.d e212/203 6:bdcfd3ff:::benchmark_data_smithi017_350392_object80199:head v 212'11256, mlcod=212'11254)", 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:15:48.018930+0000", 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0063382890000000004, 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063718359999999996, 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:15:48.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018930+0000", 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018930+0000", 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018926+0000", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999957 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018941+0000", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4632e-05 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018956+0000", 2023-09-23T15:15:48.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4791e-05 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.018964+0000", 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1459999999999996e-06 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.020193+0000", 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00122901 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.020223+0000", 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9828e-05 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:180592 6.d e212/203)", 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:15:48.025151+0000", 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00011769699999999999, 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224019, 2023-09-23T15:15:48.040 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025151+0000", 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:15:48.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025151+0000", 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025149+0000", 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:15:48.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025165+0000", 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6138999999999999e-05 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025188+0000", 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2166000000000001e-05 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:15:48.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:15:48.025198+0000", 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0229e-05 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2023-09-23T15:15:48.044 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:15:48.093 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:48.297 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:48.307 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:48] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.43.0" 2023-09-23T15:15:48.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:48.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:48.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:48.496 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:48.882 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:48.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:48.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:48.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:49.023 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:49.068 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:15:49.111 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:49.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:49 smithi017 ceph-mon[222878]: pgmap v584: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:49 smithi017 ceph-mon[222878]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:49.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:49 smithi017 ceph-mon[215646]: pgmap v584: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:49.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:49 smithi017 ceph-mon[215646]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:49.732 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:49.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:15:49.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:49.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:49.784 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:49.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:15:49.839 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:49 smithi027 ceph-mon[195233]: pgmap v584: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 22439/66099 objects degraded (33.948%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T15:15:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:49 smithi027 ceph-mon[195233]: from='client.45359 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:15:49.867 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:49.924 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:49.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:49.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:50.029 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:50.073 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":584,"stamp":"2023-09-23T15:15:48.417243+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1446877404,"num_objects":22033,"num_object_clones":0,"num_object_copies":66099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22439,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22033,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":179800,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7323,"num_bytes_recovered":481400999,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124685,"ondisk_log_size":124685,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5925084,"kb_used_data":4280356,"kb_used_omap":30,"kb_used_meta":1644129,"kb_avail":650290980,"statfs":{"total":671965249536,"available":665897963520,"internally_reserved":0,"allocated":4383084544,"data_stored":4377253365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1683588696},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-355205120,"num_objects":-5420,"num_object_clones":0,"num_object_copies":-16260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-302,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5420,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":302,"num_bytes_recovered":19791872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008783"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":206,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.982039+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:15:30.982039+0000","last_peered":"2023-09-23T15:15:30.982039+0000","last_clean":"2023-09-23T15:15:30.982039+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:15:30.982039+0000","last_undegraded":"2023-09-23T15:15:30.982039+0000","last_fullsized":"2023-09-23T15:15:30.982039+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":140,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985875+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:15:30.985875+0000","last_peered":"2023-09-23T15:15:30.985875+0000","last_clean":"2023-09-23T15:15:30.985875+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:15:30.985875+0000","last_undegraded":"2023-09-23T15:15:30.985875+0000","last_fullsized":"2023-09-23T15:15:30.985875+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":89,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.001724+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:15:31.001724+0000","last_peered":"2023-09-23T15:15:31.001724+0000","last_clean":"2023-09-23T15:15:31.001724+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:15:31.001724+0000","last_undegraded":"2023-09-23T15:15:31.001724+0000","last_fullsized":"2023-09-23T15:15:31.001724+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":177,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018694+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:15:31.018694+0000","last_peered":"2023-09-23T15:15:31.018694+0000","last_clean":"2023-09-23T15:15:31.018694+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:15:31.018694+0000","last_undegraded":"2023-09-23T15:15:31.018694+0000","last_fullsized":"2023-09-23T15:15:31.018694+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":209,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984583+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:15:30.984583+0000","last_peered":"2023-09-23T15:15:30.984583+0000","last_clean":"2023-09-23T15:15:30.984583+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:15:30.984583+0000","last_undegraded":"2023-09-23T15:15:30.984583+0000","last_fullsized":"2023-09-23T15:15:30.984583+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":186,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.990834+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:15:30.990834+0000","last_peered":"2023-09-23T15:15:30.990834+0000","last_clean":"2023-09-23T15:15:30.990834+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:15:30.990834+0000","last_undegraded":"2023-09-23T15:15:30.990834+0000","last_fullsized":"2023-09-23T15:15:30.990834+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":156,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.019252+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:15:31.019252+0000","last_peered":"2023-09-23T15:15:31.019252+0000","last_clean":"2023-09-23T15:15:31.019252+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:15:31.019252+0000","last_undegraded":"2023-09-23T15:15:31.019252+0000","last_fullsized":"2023-09-23T15:15:31.019252+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":192,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.696575+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:15:34.696575+0000","last_peered":"2023-09-23T15:15:34.696575+0000","last_clean":"2023-09-23T15:15:34.696575+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:15:34.696575+0000","last_undegraded":"2023-09-23T15:15:34.696575+0000","last_fullsized":"2023-09-23T15:15:34.696575+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018147+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:15:31.018147+0000","last_peered":"2023-09-23T15:15:31.018147+0000","last_clean":"2023-09-23T15:15:31.018147+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:15:31.018147+0000","last_undegraded":"2023-09-23T15:15:31.018147+0000","last_fullsized":"2023-09-23T15:15:31.018147+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":202,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985180+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:15:30.985180+0000","last_peered":"2023-09-23T15:15:30.985180+0000","last_clean":"2023-09-23T15:15:30.985180+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:15:30.985180+0000","last_undegraded":"2023-09-23T15:15:30.985180+0000","last_fullsized":"2023-09-23T15:15:30.985180+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":135,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.696610+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:15:34.696610+0000","last_peered":"2023-09-23T15:15:34.696610+0000","last_clean":"2023-09-23T15:15:34.696610+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:15:34.696610+0000","last_undegraded":"2023-09-23T15:15:34.696610+0000","last_fullsized":"2023-09-23T15:15:34.696610+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":74,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985813+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:15:30.985813+0000","last_peered":"2023-09-23T15:15:30.985813+0000","last_clean":"2023-09-23T15:15:30.985813+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:15:30.985813+0000","last_undegraded":"2023-09-23T15:15:30.985813+0000","last_fullsized":"2023-09-23T15:15:30.985813+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":84,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.997154+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:30.997154+0000","last_peered":"2023-09-23T15:15:30.997154+0000","last_clean":"2023-09-23T15:15:30.997154+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:30.997154+0000","last_undegraded":"2023-09-23T15:15:30.997154+0000","last_fullsized":"2023-09-23T15:15:30.997154+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":196,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979724+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:15:30.979724+0000","last_peered":"2023-09-23T15:15:30.979724+0000","last_clean":"2023-09-23T15:15:30.979724+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:15:30.979724+0000","last_undegraded":"2023-09-23T15:15:30.979724+0000","last_fullsized":"2023-09-23T15:15:30.979724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979882+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:15:30.979882+0000","last_peered":"2023-09-23T15:15:30.979882+0000","last_clean":"2023-09-23T15:15:30.979882+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:15:30.979882+0000","last_undegraded":"2023-09-23T15:15:30.979882+0000","last_fullsized":"2023-09-23T15:15:30.979882+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007443+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:15:31.007443+0000","last_peered":"2023-09-23T15:15:31.007443+0000","last_clean":"2023-09-23T15:15:31.007443+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:15:31.007443+0000","last_undegraded":"2023-09-23T15:15:31.007443+0000","last_fullsized":"2023-09-23T15:15:31.007443+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":74,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979934+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:15:30.979934+0000","last_peered":"2023-09-23T15:15:30.979934+0000","last_clean":"2023-09-23T15:15:30.979934+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:15:30.979934+0000","last_undegraded":"2023-09-23T15:15:30.979934+0000","last_fullsized":"2023-09-23T15:15:30.979934+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":203,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.696675+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:15:34.696675+0000","last_peered":"2023-09-23T15:15:34.696675+0000","last_clean":"2023-09-23T15:15:34.696675+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:15:34.696675+0000","last_undegraded":"2023-09-23T15:15:34.696675+0000","last_fullsized":"2023-09-23T15:15:34.696675+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007022+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:15:31.007022+0000","last_peered":"2023-09-23T15:15:31.007022+0000","last_clean":"2023-09-23T15:15:31.007022+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:15:31.007022+0000","last_undegraded":"2023-09-23T15:15:31.007022+0000","last_fullsized":"2023-09-23T15:15:31.007022+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.023011+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:15:31.023011+0000","last_peered":"2023-09-23T15:15:31.023011+0000","last_clean":"2023-09-23T15:15:31.023011+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:15:31.023011+0000","last_undegraded":"2023-09-23T15:15:31.023011+0000","last_fullsized":"2023-09-23T15:15:31.023011+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":180,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985318+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:15:30.985318+0000","last_peered":"2023-09-23T15:15:30.985318+0000","last_clean":"2023-09-23T15:15:30.985318+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:15:30.985318+0000","last_undegraded":"2023-09-23T15:15:30.985318+0000","last_fullsized":"2023-09-23T15:15:30.985318+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.017750+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:15:31.017750+0000","last_peered":"2023-09-23T15:15:31.017750+0000","last_clean":"2023-09-23T15:15:31.017750+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:15:31.017750+0000","last_undegraded":"2023-09-23T15:15:31.017750+0000","last_fullsized":"2023-09-23T15:15:31.017750+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":58,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979612+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:15:30.979612+0000","last_peered":"2023-09-23T15:15:30.979612+0000","last_clean":"2023-09-23T15:15:30.979612+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:15:30.979612+0000","last_undegraded":"2023-09-23T15:15:30.979612+0000","last_fullsized":"2023-09-23T15:15:30.979612+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018769+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:15:31.018769+0000","last_peered":"2023-09-23T15:15:31.018769+0000","last_clean":"2023-09-23T15:15:31.018769+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:15:31.018769+0000","last_undegraded":"2023-09-23T15:15:31.018769+0000","last_fullsized":"2023-09-23T15:15:31.018769+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.023297+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:15:31.023297+0000","last_peered":"2023-09-23T15:15:31.023297+0000","last_clean":"2023-09-23T15:15:31.023297+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:15:31.023297+0000","last_undegraded":"2023-09-23T15:15:31.023297+0000","last_fullsized":"2023-09-23T15:15:31.023297+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":178,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.989941+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:15:30.989941+0000","last_peered":"2023-09-23T15:15:30.989941+0000","last_clean":"2023-09-23T15:15:30.989941+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:15:30.989941+0000","last_undegraded":"2023-09-23T15:15:30.989941+0000","last_fullsized":"2023-09-23T15:15:30.989941+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":202,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985755+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:15:30.985755+0000","last_peered":"2023-09-23T15:15:30.985755+0000","last_clean":"2023-09-23T15:15:30.985755+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:15:30.985755+0000","last_undegraded":"2023-09-23T15:15:30.985755+0000","last_fullsized":"2023-09-23T15:15:30.985755+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"212'11096","reported_seq":11143,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.714292+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:15:44.714292+0000","last_peered":"2023-09-23T15:15:44.714292+0000","last_clean":"2023-09-23T15:15:44.714292+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:15:44.714292+0000","last_undegraded":"2023-09-23T15:15:44.714292+0000","last_fullsized":"2023-09-23T15:15:44.714292+0000","mapping_epoch":171,"log_start":"175'1000","ondisk_log_start":"175'1000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10096,"log_dups_size":0,"ondisk_log_size":10096,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85065728,"num_objects":1298,"num_object_clones":0,"num_object_copies":3894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11096,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.019960+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:15:31.019960+0000","last_peered":"2023-09-23T15:15:31.019960+0000","last_clean":"2023-09-23T15:15:31.019960+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:15:31.019960+0000","last_undegraded":"2023-09-23T15:15:31.019960+0000","last_fullsized":"2023-09-23T15:15:31.019960+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":86,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.982767+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:15:30.982767+0000","last_peered":"2023-09-23T15:15:30.982767+0000","last_clean":"2023-09-23T15:15:30.982767+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:15:30.982767+0000","last_undegraded":"2023-09-23T15:15:30.982767+0000","last_fullsized":"2023-09-23T15:15:30.982767+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":163,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984864+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:15:30.984864+0000","last_peered":"2023-09-23T15:15:30.984864+0000","last_clean":"2023-09-23T15:15:30.984864+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:15:30.984864+0000","last_undegraded":"2023-09-23T15:15:30.984864+0000","last_fullsized":"2023-09-23T15:15:30.984864+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"212'11327","reported_seq":16113,"reported_epoch":212,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:44.688826+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:15:44.688826+0000","last_peered":"2023-09-23T15:15:44.688826+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:15:44.688826+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"181'4900","ondisk_log_start":"181'4900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6427,"log_dups_size":0,"ondisk_log_size":6427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87752704,"num_objects":1339,"num_object_clones":0,"num_object_copies":4017,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2634,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1339,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2364,"num_bytes_recovered":154927104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1295}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":479,"reported_epoch":211,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:31.006954+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:15:31.006954+0000","last_peered":"2023-09-23T15:15:31.006954+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:15:31.006954+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":86,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.982561+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:15:30.982561+0000","last_peered":"2023-09-23T15:15:30.982561+0000","last_clean":"2023-09-23T15:15:30.982561+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:15:30.982561+0000","last_undegraded":"2023-09-23T15:15:30.982561+0000","last_fullsized":"2023-09-23T15:15:30.982561+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979522+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:15:30.979522+0000","last_peered":"2023-09-23T15:15:30.979522+0000","last_clean":"2023-09-23T15:15:30.979522+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:15:30.979522+0000","last_undegraded":"2023-09-23T15:15:30.979522+0000","last_fullsized":"2023-09-23T15:15:30.979522+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"212'11026","reported_seq":11077,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:43.582403+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:15:43.582403+0000","last_peered":"2023-09-23T15:15:43.582403+0000","last_clean":"2023-09-23T15:15:43.582403+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:15:43.582403+0000","last_undegraded":"2023-09-23T15:15:43.582403+0000","last_fullsized":"2023-09-23T15:15:43.582403+0000","mapping_epoch":171,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9126,"log_dups_size":0,"ondisk_log_size":9126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92798976,"num_objects":1416,"num_object_clones":0,"num_object_copies":4248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11026,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018957+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:15:31.018957+0000","last_peered":"2023-09-23T15:15:31.018957+0000","last_clean":"2023-09-23T15:15:31.018957+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:15:31.018957+0000","last_undegraded":"2023-09-23T15:15:31.018957+0000","last_fullsized":"2023-09-23T15:15:31.018957+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":72,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985321+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:15:30.985321+0000","last_peered":"2023-09-23T15:15:30.985321+0000","last_clean":"2023-09-23T15:15:30.985321+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:15:30.985321+0000","last_undegraded":"2023-09-23T15:15:30.985321+0000","last_fullsized":"2023-09-23T15:15:30.985321+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":200,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.990240+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:15:30.990240+0000","last_peered":"2023-09-23T15:15:30.990240+0000","last_clean":"2023-09-23T15:15:30.990240+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:15:30.990240+0000","last_undegraded":"2023-09-23T15:15:30.990240+0000","last_fullsized":"2023-09-23T15:15:30.990240+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"212'11265","reported_seq":11316,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.444413+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:15:44.444413+0000","last_peered":"2023-09-23T15:15:44.444413+0000","last_clean":"2023-09-23T15:15:44.444413+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:15:44.444413+0000","last_undegraded":"2023-09-23T15:15:44.444413+0000","last_fullsized":"2023-09-23T15:15:44.444413+0000","mapping_epoch":171,"log_start":"192'5700","ondisk_log_start":"192'5700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5565,"log_dups_size":0,"ondisk_log_size":5565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11265,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018371+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:15:31.018371+0000","last_peered":"2023-09-23T15:15:31.018371+0000","last_clean":"2023-09-23T15:15:31.018371+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:15:31.018371+0000","last_undegraded":"2023-09-23T15:15:31.018371+0000","last_fullsized":"2023-09-23T15:15:31.018371+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985168+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:30.985168+0000","last_peered":"2023-09-23T15:15:30.985168+0000","last_clean":"2023-09-23T15:15:30.985168+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:15:30.985168+0000","last_undegraded":"2023-09-23T15:15:30.985168+0000","last_fullsized":"2023-09-23T15:15:30.985168+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.017856+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:15:31.017856+0000","last_peered":"2023-09-23T15:15:31.017856+0000","last_clean":"2023-09-23T15:15:31.017856+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:15:31.017856+0000","last_undegraded":"2023-09-23T15:15:31.017856+0000","last_fullsized":"2023-09-23T15:15:31.017856+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"212'11456","reported_seq":11511,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:44.337843+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:15:44.337843+0000","last_peered":"2023-09-23T15:15:44.337843+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:15:44.337843+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"179'3900","ondisk_log_start":"179'3900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7556,"log_dups_size":0,"ondisk_log_size":7556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11456,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-3554}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007346+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:15:31.007346+0000","last_peered":"2023-09-23T15:15:31.007346+0000","last_clean":"2023-09-23T15:15:31.007346+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:15:31.007346+0000","last_undegraded":"2023-09-23T15:15:31.007346+0000","last_fullsized":"2023-09-23T15:15:31.007346+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":175,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.986617+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:15:30.986617+0000","last_peered":"2023-09-23T15:15:30.986617+0000","last_clean":"2023-09-23T15:15:30.986617+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:15:30.986617+0000","last_undegraded":"2023-09-23T15:15:30.986617+0000","last_fullsized":"2023-09-23T15:15:30.986617+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979811+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:15:30.979811+0000","last_peered":"2023-09-23T15:15:30.979811+0000","last_clean":"2023-09-23T15:15:30.979811+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:15:30.979811+0000","last_undegraded":"2023-09-23T15:15:30.979811+0000","last_fullsized":"2023-09-23T15:15:30.979811+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"212'11208","reported_seq":11259,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:43.582085+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:15:43.582085+0000","last_peered":"2023-09-23T15:15:43.582085+0000","last_clean":"2023-09-23T15:15:43.582085+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:15:43.582085+0000","last_undegraded":"2023-09-23T15:15:43.582085+0000","last_fullsized":"2023-09-23T15:15:43.582085+0000","mapping_epoch":171,"log_start":"176'2100","ondisk_log_start":"176'2100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11208,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":505,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007055+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:15:31.007055+0000","last_peered":"2023-09-23T15:15:31.007055+0000","last_clean":"2023-09-23T15:15:31.007055+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:15:31.007055+0000","last_undegraded":"2023-09-23T15:15:31.007055+0000","last_fullsized":"2023-09-23T15:15:31.007055+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.990146+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:15:30.990146+0000","last_peered":"2023-09-23T15:15:30.990146+0000","last_clean":"2023-09-23T15:15:30.990146+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:15:30.990146+0000","last_undegraded":"2023-09-23T15:15:30.990146+0000","last_fullsized":"2023-09-23T15:15:30.990146+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979625+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:15:30.979625+0000","last_peered":"2023-09-23T15:15:30.979625+0000","last_clean":"2023-09-23T15:15:30.979625+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:15:30.979625+0000","last_undegraded":"2023-09-23T15:15:30.979625+0000","last_fullsized":"2023-09-23T15:15:30.979625+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"212'11215","reported_seq":11265,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.679090+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:15:44.679090+0000","last_peered":"2023-09-23T15:15:44.679090+0000","last_clean":"2023-09-23T15:15:44.679090+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:15:44.679090+0000","last_undegraded":"2023-09-23T15:15:44.679090+0000","last_fullsized":"2023-09-23T15:15:44.679090+0000","mapping_epoch":171,"log_start":"181'4800","ondisk_log_start":"181'4800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89325568,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11215,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.017760+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:15:31.017760+0000","last_peered":"2023-09-23T15:15:31.017760+0000","last_clean":"2023-09-23T15:15:31.017760+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:15:31.017760+0000","last_undegraded":"2023-09-23T15:15:31.017760+0000","last_fullsized":"2023-09-23T15:15:31.017760+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":195,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979414+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:15:30.979414+0000","last_peered":"2023-09-23T15:15:30.979414+0000","last_clean":"2023-09-23T15:15:30.979414+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:15:30.979414+0000","last_undegraded":"2023-09-23T15:15:30.979414+0000","last_fullsized":"2023-09-23T15:15:30.979414+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.982647+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:15:30.982647+0000","last_peered":"2023-09-23T15:15:30.982647+0000","last_clean":"2023-09-23T15:15:30.982647+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:15:30.982647+0000","last_undegraded":"2023-09-23T15:15:30.982647+0000","last_fullsized":"2023-09-23T15:15:30.982647+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"212'11158","reported_seq":21028,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.691522+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:15:44.691522+0000","last_peered":"2023-09-23T15:15:44.691522+0000","last_clean":"2023-09-23T15:15:44.691522+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:15:44.691522+0000","last_undegraded":"2023-09-23T15:15:44.691522+0000","last_fullsized":"2023-09-23T15:15:44.691522+0000","mapping_epoch":203,"log_start":"175'1100","ondisk_log_start":"175'1100","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10058,"log_dups_size":0,"ondisk_log_size":10058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89980952,"num_objects":1374,"num_object_clones":0,"num_object_copies":4122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1374,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11158,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":500,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.023222+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:15:31.023222+0000","last_peered":"2023-09-23T15:15:31.023222+0000","last_clean":"2023-09-23T15:15:31.023222+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:15:31.023222+0000","last_undegraded":"2023-09-23T15:15:31.023222+0000","last_fullsized":"2023-09-23T15:15:31.023222+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984349+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:15:30.984349+0000","last_peered":"2023-09-23T15:15:30.984349+0000","last_clean":"2023-09-23T15:15:30.984349+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:15:30.984349+0000","last_undegraded":"2023-09-23T15:15:30.984349+0000","last_fullsized":"2023-09-23T15:15:30.984349+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":89,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.996673+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:30.996673+0000","last_peered":"2023-09-23T15:15:30.996673+0000","last_clean":"2023-09-23T15:15:30.996673+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:30.996673+0000","last_undegraded":"2023-09-23T15:15:30.996673+0000","last_fullsized":"2023-09-23T15:15:30.996673+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"212'11227","reported_seq":11278,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.444448+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:15:44.444448+0000","last_peered":"2023-09-23T15:15:44.444448+0000","last_clean":"2023-09-23T15:15:44.444448+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:15:44.444448+0000","last_undegraded":"2023-09-23T15:15:44.444448+0000","last_fullsized":"2023-09-23T15:15:44.444448+0000","mapping_epoch":171,"log_start":"191'5600","ondisk_log_start":"191'5600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5627,"log_dups_size":0,"ondisk_log_size":5627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87883776,"num_objects":1341,"num_object_clones":0,"num_object_copies":4023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11227,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.019042+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:15:31.019042+0000","last_peered":"2023-09-23T15:15:31.019042+0000","last_clean":"2023-09-23T15:15:31.019042+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:15:31.019042+0000","last_undegraded":"2023-09-23T15:15:31.019042+0000","last_fullsized":"2023-09-23T15:15:31.019042+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":388,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.998555+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:15:30.998555+0000","last_peered":"2023-09-23T15:15:30.998555+0000","last_clean":"2023-09-23T15:15:30.998555+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:15:30.998555+0000","last_undegraded":"2023-09-23T15:15:30.998555+0000","last_fullsized":"2023-09-23T15:15:30.998555+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":188,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985221+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:30.985221+0000","last_peered":"2023-09-23T15:15:30.985221+0000","last_clean":"2023-09-23T15:15:30.985221+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:15:30.985221+0000","last_undegraded":"2023-09-23T15:15:30.985221+0000","last_fullsized":"2023-09-23T15:15:30.985221+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":133,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.987289+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:15:30.987289+0000","last_peered":"2023-09-23T15:15:30.987289+0000","last_clean":"2023-09-23T15:15:30.987289+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:15:30.987289+0000","last_undegraded":"2023-09-23T15:15:30.987289+0000","last_fullsized":"2023-09-23T15:15:30.987289+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"212'11011","reported_seq":11062,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.618775+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:15:44.618775+0000","last_peered":"2023-09-23T15:15:44.618775+0000","last_clean":"2023-09-23T15:15:44.618775+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:15:44.618775+0000","last_undegraded":"2023-09-23T15:15:44.618775+0000","last_fullsized":"2023-09-23T15:15:44.618775+0000","mapping_epoch":171,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88014848,"num_objects":1343,"num_object_clones":0,"num_object_copies":4029,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1343,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11011,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":489,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.100123+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:15:42.100123+0000","last_peered":"2023-09-23T15:15:42.100123+0000","last_clean":"2023-09-23T15:15:42.100123+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:15:42.100123+0000","last_undegraded":"2023-09-23T15:15:42.100123+0000","last_fullsized":"2023-09-23T15:15:42.100123+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":730,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985109+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:15:30.985109+0000","last_peered":"2023-09-23T15:15:30.985109+0000","last_clean":"2023-09-23T15:15:30.985109+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:15:30.985109+0000","last_undegraded":"2023-09-23T15:15:30.985109+0000","last_fullsized":"2023-09-23T15:15:30.985109+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979830+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:15:30.979830+0000","last_peered":"2023-09-23T15:15:30.979830+0000","last_clean":"2023-09-23T15:15:30.979830+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:15:30.979830+0000","last_undegraded":"2023-09-23T15:15:30.979830+0000","last_fullsized":"2023-09-23T15:15:30.979830+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":104,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.998236+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:15:30.998236+0000","last_peered":"2023-09-23T15:15:30.998236+0000","last_clean":"2023-09-23T15:15:30.998236+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:15:30.998236+0000","last_undegraded":"2023-09-23T15:15:30.998236+0000","last_fullsized":"2023-09-23T15:15:30.998236+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"212'11227","reported_seq":11276,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.049210+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:15:44.049210+0000","last_peered":"2023-09-23T15:15:44.049210+0000","last_clean":"2023-09-23T15:15:44.049210+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:15:44.049210+0000","last_undegraded":"2023-09-23T15:15:44.049210+0000","last_fullsized":"2023-09-23T15:15:44.049210+0000","mapping_epoch":171,"log_start":"193'5900","ondisk_log_start":"193'5900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5327,"log_dups_size":0,"ondisk_log_size":5327,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11227,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":501,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.100365+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:15:42.100365+0000","last_peered":"2023-09-23T15:15:42.100365+0000","last_clean":"2023-09-23T15:15:42.100365+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:15:42.100365+0000","last_undegraded":"2023-09-23T15:15:42.100365+0000","last_fullsized":"2023-09-23T15:15:42.100365+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":824,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.992979+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:15:30.992979+0000","last_peered":"2023-09-23T15:15:30.992979+0000","last_clean":"2023-09-23T15:15:30.992979+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:15:30.992979+0000","last_undegraded":"2023-09-23T15:15:30.992979+0000","last_fullsized":"2023-09-23T15:15:30.992979+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":84,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.996049+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:15:30.996049+0000","last_peered":"2023-09-23T15:15:30.996049+0000","last_clean":"2023-09-23T15:15:30.996049+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:15:30.996049+0000","last_undegraded":"2023-09-23T15:15:30.996049+0000","last_fullsized":"2023-09-23T15:15:30.996049+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":108,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.996604+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:15:30.996604+0000","last_peered":"2023-09-23T15:15:30.996604+0000","last_clean":"2023-09-23T15:15:30.996604+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:15:30.996604+0000","last_undegraded":"2023-09-23T15:15:30.996604+0000","last_fullsized":"2023-09-23T15:15:30.996604+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"212'10994","reported_seq":11043,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.636609+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:15:44.636609+0000","last_peered":"2023-09-23T15:15:44.636609+0000","last_clean":"2023-09-23T15:15:44.636609+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:15:44.636609+0000","last_undegraded":"2023-09-23T15:15:44.636609+0000","last_fullsized":"2023-09-23T15:15:44.636609+0000","mapping_epoch":171,"log_start":"175'900","ondisk_log_start":"175'900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90832896,"num_objects":1386,"num_object_clones":0,"num_object_copies":4158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10994,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":499,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985048+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:15:30.985048+0000","last_peered":"2023-09-23T15:15:30.985048+0000","last_clean":"2023-09-23T15:15:30.985048+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:15:30.985048+0000","last_undegraded":"2023-09-23T15:15:30.985048+0000","last_fullsized":"2023-09-23T15:15:30.985048+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":501,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:34.462487+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:15:34.462487+0000","last_peered":"2023-09-23T15:15:34.462487+0000","last_clean":"2023-09-23T15:15:34.462487+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:15:34.462487+0000","last_undegraded":"2023-09-23T15:15:34.462487+0000","last_fullsized":"2023-09-23T15:15:34.462487+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985533+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:15:30.985533+0000","last_peered":"2023-09-23T15:15:30.985533+0000","last_clean":"2023-09-23T15:15:30.985533+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:15:30.985533+0000","last_undegraded":"2023-09-23T15:15:30.985533+0000","last_fullsized":"2023-09-23T15:15:30.985533+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.990510+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:15:30.990510+0000","last_peered":"2023-09-23T15:15:30.990510+0000","last_clean":"2023-09-23T15:15:30.990510+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:15:30.990510+0000","last_undegraded":"2023-09-23T15:15:30.990510+0000","last_fullsized":"2023-09-23T15:15:30.990510+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"212'11273","reported_seq":11324,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:44.692148+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:15:44.692148+0000","last_peered":"2023-09-23T15:15:44.692148+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:15:44.692148+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"181'4800","ondisk_log_start":"181'4800","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89849856,"num_objects":1371,"num_object_clones":0,"num_object_copies":4113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11273,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-3581}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":489,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.099961+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:15:42.099961+0000","last_peered":"2023-09-23T15:15:42.099961+0000","last_clean":"2023-09-23T15:15:42.099961+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:15:42.099961+0000","last_undegraded":"2023-09-23T15:15:42.099961+0000","last_fullsized":"2023-09-23T15:15:42.099961+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":295,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.995789+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:15:30.995789+0000","last_peered":"2023-09-23T15:15:30.995789+0000","last_clean":"2023-09-23T15:15:30.995789+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:15:30.995789+0000","last_undegraded":"2023-09-23T15:15:30.995789+0000","last_fullsized":"2023-09-23T15:15:30.995789+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":157,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.007376+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:31.007376+0000","last_peered":"2023-09-23T15:15:31.007376+0000","last_clean":"2023-09-23T15:15:31.007376+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:31.007376+0000","last_undegraded":"2023-09-23T15:15:31.007376+0000","last_fullsized":"2023-09-23T15:15:31.007376+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018499+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:15:31.018499+0000","last_peered":"2023-09-23T15:15:31.018499+0000","last_clean":"2023-09-23T15:15:31.018499+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:15:31.018499+0000","last_undegraded":"2023-09-23T15:15:31.018499+0000","last_fullsized":"2023-09-23T15:15:31.018499+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"212'11098","reported_seq":11149,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:43.565365+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:15:43.565365+0000","last_peered":"2023-09-23T15:15:43.565365+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:15:43.565365+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"176'2000","ondisk_log_start":"176'2000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9098,"log_dups_size":0,"ondisk_log_size":9098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92405760,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11098,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-3464}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":488,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018854+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:15:31.018854+0000","last_peered":"2023-09-23T15:15:31.018854+0000","last_clean":"2023-09-23T15:15:31.018854+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:15:31.018854+0000","last_undegraded":"2023-09-23T15:15:31.018854+0000","last_fullsized":"2023-09-23T15:15:31.018854+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":818,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:42.105088+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:15:42.105088+0000","last_peered":"2023-09-23T15:15:42.105088+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:15:42.105088+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":84,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.998431+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:15:30.998431+0000","last_peered":"2023-09-23T15:15:30.998431+0000","last_clean":"2023-09-23T15:15:30.998431+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:15:30.998431+0000","last_undegraded":"2023-09-23T15:15:30.998431+0000","last_fullsized":"2023-09-23T15:15:30.998431+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":162,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.017999+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:31.017999+0000","last_peered":"2023-09-23T15:15:31.017999+0000","last_clean":"2023-09-23T15:15:31.017999+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:31.017999+0000","last_undegraded":"2023-09-23T15:15:31.017999+0000","last_fullsized":"2023-09-23T15:15:31.017999+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"212'10931","reported_seq":10980,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:44.237568+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:15:44.237568+0000","last_peered":"2023-09-23T15:15:44.237568+0000","last_clean":"2023-09-23T15:15:44.237568+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:15:44.237568+0000","last_undegraded":"2023-09-23T15:15:44.237568+0000","last_fullsized":"2023-09-23T15:15:44.237568+0000","mapping_epoch":171,"log_start":"178'3400","ondisk_log_start":"178'3400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7531,"log_dups_size":0,"ondisk_log_size":7531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89849856,"num_objects":1371,"num_object_clones":0,"num_object_copies":4113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10931,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":489,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.105257+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:15:42.105257+0000","last_peered":"2023-09-23T15:15:42.105257+0000","last_clean":"2023-09-23T15:15:42.105257+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:15:42.105257+0000","last_undegraded":"2023-09-23T15:15:42.105257+0000","last_fullsized":"2023-09-23T15:15:42.105257+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":188,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985688+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:15:30.985688+0000","last_peered":"2023-09-23T15:15:30.985688+0000","last_clean":"2023-09-23T15:15:30.985688+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:15:30.985688+0000","last_undegraded":"2023-09-23T15:15:30.985688+0000","last_fullsized":"2023-09-23T15:15:30.985688+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":672,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.999618+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:15:30.999618+0000","last_peered":"2023-09-23T15:15:30.999618+0000","last_clean":"2023-09-23T15:15:30.999618+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:15:30.999618+0000","last_undegraded":"2023-09-23T15:15:30.999618+0000","last_fullsized":"2023-09-23T15:15:30.999618+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":192,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984273+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:15:30.984273+0000","last_peered":"2023-09-23T15:15:30.984273+0000","last_clean":"2023-09-23T15:15:30.984273+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:15:30.984273+0000","last_undegraded":"2023-09-23T15:15:30.984273+0000","last_fullsized":"2023-09-23T15:15:30.984273+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"212'11260","reported_seq":11316,"reported_epoch":212,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:44.194183+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:15:44.194183+0000","last_peered":"2023-09-23T15:15:44.194183+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:15:44.194183+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"193'5900","ondisk_log_start":"193'5900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5360,"log_dups_size":0,"ondisk_log_size":5360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89784320,"num_objects":1370,"num_object_clones":0,"num_object_copies":4110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11260,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-3596}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":489,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.100664+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:15:42.100664+0000","last_peered":"2023-09-23T15:15:42.100664+0000","last_clean":"2023-09-23T15:15:42.100664+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:15:42.100664+0000","last_undegraded":"2023-09-23T15:15:42.100664+0000","last_fullsized":"2023-09-23T15:15:42.100664+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":688,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.990727+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:15:30.990727+0000","last_peered":"2023-09-23T15:15:30.990727+0000","last_clean":"2023-09-23T15:15:30.990727+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:15:30.990727+0000","last_undegraded":"2023-09-23T15:15:30.990727+0000","last_fullsized":"2023-09-23T15:15:30.990727+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":175,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.018611+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:15:31.018611+0000","last_peered":"2023-09-23T15:15:31.018611+0000","last_clean":"2023-09-23T15:15:31.018611+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:15:31.018611+0000","last_undegraded":"2023-09-23T15:15:31.018611+0000","last_fullsized":"2023-09-23T15:15:31.018611+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979493+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:15:30.979493+0000","last_peered":"2023-09-23T15:15:30.979493+0000","last_clean":"2023-09-23T15:15:30.979493+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:15:30.979493+0000","last_undegraded":"2023-09-23T15:15:30.979493+0000","last_fullsized":"2023-09-23T15:15:30.979493+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":164,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.017953+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:15:31.017953+0000","last_peered":"2023-09-23T15:15:31.017953+0000","last_clean":"2023-09-23T15:15:31.017953+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:15:31.017953+0000","last_undegraded":"2023-09-23T15:15:31.017953+0000","last_fullsized":"2023-09-23T15:15:31.017953+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":212,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.110086+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:15:42.110086+0000","last_peered":"2023-09-23T15:15:42.110086+0000","last_clean":"2023-09-23T15:15:42.110086+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:15:42.110086+0000","last_undegraded":"2023-09-23T15:15:42.110086+0000","last_fullsized":"2023-09-23T15:15:42.110086+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":190,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985265+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:15:30.985265+0000","last_peered":"2023-09-23T15:15:30.985265+0000","last_clean":"2023-09-23T15:15:30.985265+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:15:30.985265+0000","last_undegraded":"2023-09-23T15:15:30.985265+0000","last_fullsized":"2023-09-23T15:15:30.985265+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":204,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979975+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:15:30.979975+0000","last_peered":"2023-09-23T15:15:30.979975+0000","last_clean":"2023-09-23T15:15:30.979975+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:15:30.979975+0000","last_undegraded":"2023-09-23T15:15:30.979975+0000","last_fullsized":"2023-09-23T15:15:30.979975+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":156,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985222+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:15:30.985222+0000","last_peered":"2023-09-23T15:15:30.985222+0000","last_clean":"2023-09-23T15:15:30.985222+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:15:30.985222+0000","last_undegraded":"2023-09-23T15:15:30.985222+0000","last_fullsized":"2023-09-23T15:15:30.985222+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":187,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.983016+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:15:30.983016+0000","last_peered":"2023-09-23T15:15:30.983016+0000","last_clean":"2023-09-23T15:15:30.983016+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:15:30.983016+0000","last_undegraded":"2023-09-23T15:15:30.983016+0000","last_fullsized":"2023-09-23T15:15:30.983016+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":183,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.984883+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:15:30.984883+0000","last_peered":"2023-09-23T15:15:30.984883+0000","last_clean":"2023-09-23T15:15:30.984883+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:15:30.984883+0000","last_undegraded":"2023-09-23T15:15:30.984883+0000","last_fullsized":"2023-09-23T15:15:30.984883+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":197,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.985148+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:15:30.985148+0000","last_peered":"2023-09-23T15:15:30.985148+0000","last_clean":"2023-09-23T15:15:30.985148+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:15:30.985148+0000","last_undegraded":"2023-09-23T15:15:30.985148+0000","last_fullsized":"2023-09-23T15:15:30.985148+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":160,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:31.019331+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:31.019331+0000","last_peered":"2023-09-23T15:15:31.019331+0000","last_clean":"2023-09-23T15:15:31.019331+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:31.019331+0000","last_undegraded":"2023-09-23T15:15:31.019331+0000","last_fullsized":"2023-09-23T15:15:31.019331+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":180,"reported_epoch":211,"state":"active+clean","last_fresh":"2023-09-23T15:15:30.979314+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:15:30.979314+0000","last_peered":"2023-09-23T15:15:30.979314+0000","last_clean":"2023-09-23T15:15:30.979314+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:15:30.979314+0000","last_undegraded":"2023-09-23T15:15:30.979314+0000","last_fullsized":"2023-09-23T15:15:30.979314+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1442119704,"num_objects":22006,"num_object_clones":0,"num_object_copies":66018,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":22436,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22006,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178772,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7264,"num_bytes_recovered":476053504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4306513920,"data_stored":4306501704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120272,"ondisk_log_size":120272,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201532,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463168,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":869908,"kb_used_data":705420,"kb_used_omap":0,"kb_used_meta":164480,"kb_avail":92875244,"statfs":{"total":95995035648,"available":95104249856,"internally_reserved":0,"allocated":722350080,"data_stored":721641689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":168427520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":2.617,"15min":2.6779999999999999},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":6.367,"5min":41.491,"15min":108.39400000000001},"last":3.2879999999999998},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.8130000000000002,"15min":2.8180000000000001},"min":{"1min":0.29299999999999998,"5min":0.223,"15min":0.223},"max":{"1min":12.634,"5min":43.151000000000003,"15min":108.452},"last":3.21}]},{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":4.5350000000000001,"5min":3.6030000000000002,"15min":3.448},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":43.377000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":3.0760000000000001},{"interface":"front","average":{"1min":7.2919999999999998,"5min":4.3440000000000003,"15min":3.8519999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":42.704000000000001,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.502}]},{"osd":2,"last update":"Sat Sep 23 15:14:58 2023","interfaces":[{"interface":"back","average":{"1min":13.1,"5min":6.4619999999999997,"15min":5.702},"min":{"1min":0.33600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":112.352,"5min":112.352,"15min":180.374},"last":1.8959999999999999},{"interface":"front","average":{"1min":13.564,"5min":6.4489999999999998,"15min":5.7329999999999997},"min":{"1min":0.45200000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":117.667,"5min":117.667,"15min":181.685},"last":3.399}]},{"osd":3,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":15.903,"5min":7.0270000000000001,"15min":3.012},"min":{"1min":0.627,"5min":0.249,"15min":0.249},"max":{"1min":102.254,"5min":102.254,"15min":102.254},"last":3.4900000000000002},{"interface":"front","average":{"1min":17.141999999999999,"5min":7.4619999999999997,"15min":3.0840000000000001},"min":{"1min":0.46100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":98.941999999999993,"5min":98.941999999999993,"15min":98.941999999999993},"last":3.9049999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4870000000000001,"15min":1.0309999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.75800000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.6040000000000001,"15min":1.121},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7590000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.48899999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":2.5230000000000001,"15min":1.4059999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":2.976,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":2.5310000000000001,"15min":1.452},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7859999999999996,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.2689999999999999,"15min":1.032},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.125,"5min":7.5910000000000002,"15min":17},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1659999999999999,"15min":0.995},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":2.524,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.47399999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015206,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":676136,"kb_used_data":517664,"kb_used_omap":6,"kb_used_meta":158457,"kb_avail":93069016,"statfs":{"total":95995035648,"available":95302672384,"internally_reserved":0,"allocated":530087936,"data_stored":529075126,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":162260043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8250000000000002,"5min":3.6080000000000001,"15min":3.738},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":7.0839999999999996,"5min":16.626000000000001,"15min":16.626000000000001},"last":4.5919999999999996},{"interface":"front","average":{"1min":3.5609999999999999,"5min":4.4000000000000004,"15min":4.54},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":10.558,"5min":16.131,"15min":16.131},"last":1.9490000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":5.0229999999999997,"5min":4.3070000000000004,"15min":4.1870000000000003},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":13.757999999999999,"5min":13.843,"15min":13.843},"last":1.821},{"interface":"front","average":{"1min":4.3380000000000001,"5min":4.2460000000000004,"15min":4.2300000000000004},"min":{"1min":0.26200000000000001,"5min":0.253,"15min":0.253},"max":{"1min":16.882000000000001,"5min":16.882000000000001,"15min":16.882000000000001},"last":4.5149999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":4.9539999999999997,"5min":9.6270000000000007,"15min":10.406000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.27,"5min":123.426,"15min":123.426},"last":4.54},{"interface":"front","average":{"1min":3.5499999999999998,"5min":10.762,"15min":11.964},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.1010000000000009,"5min":123.32899999999999,"15min":123.32899999999999},"last":2.1880000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7629999999999999,"5min":2.7090000000000001,"15min":2.7000000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":7.7199999999999998,"5min":7.8330000000000002,"15min":7.8330000000000002},"last":1.887},{"interface":"front","average":{"1min":3.129,"5min":3.472,"15min":3.5289999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.779,"5min":16.779,"15min":16.779},"last":4.4050000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":1.9790000000000001,"5min":3.419,"15min":3.6589999999999998},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":6.8259999999999996,"5min":16.166,"15min":16.166},"last":1.9810000000000001},{"interface":"front","average":{"1min":3.0190000000000001,"5min":2.9209999999999998,"15min":2.9049999999999998},"min":{"1min":0.35399999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.7640000000000002,"5min":7.8600000000000003,"15min":7.8600000000000003},"last":5.125}]},{"osd":6,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":2.2040000000000002,"5min":3.488,"15min":3.702},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.8010000000000002,"5min":16.074000000000002,"15min":16.074000000000002},"last":2.004},{"interface":"front","average":{"1min":3.0299999999999998,"5min":2.8050000000000002,"15min":2.7669999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":7.7930000000000001,"5min":7.7930000000000001,"15min":7.7930000000000001},"last":1.8640000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2490000000000001,"5min":3.1749999999999998,"15min":3.1629999999999998},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":7.8179999999999996,"5min":7.8890000000000002,"15min":7.8890000000000002},"last":1.9119999999999999},{"interface":"front","average":{"1min":3.0390000000000001,"5min":3.2909999999999999,"15min":3.3330000000000002},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.841000000000001,"5min":16.841000000000001,"15min":16.841000000000001},"last":4.484}]}]},{"osd":0,"up_from":95,"seq":408021893255,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649280,"kb_used_data":355448,"kb_used_omap":8,"kb_used_meta":293559,"kb_avail":93095872,"statfs":{"total":95995035648,"available":95330172928,"internally_reserved":0,"allocated":363978752,"data_stored":363059415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":300605012},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:14:45 2023","interfaces":[{"interface":"back","average":{"1min":2.2069999999999999,"5min":3.875,"15min":4.1529999999999996},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":5.5099999999999998,"5min":31.510999999999999,"15min":31.510999999999999},"last":3.306},{"interface":"front","average":{"1min":2.5329999999999999,"5min":4.274,"15min":4.5640000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.4249999999999998,"5min":31.378,"15min":31.378},"last":2.5979999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":15.058999999999999,"5min":7.9790000000000001,"15min":3.3519999999999999},"min":{"1min":0.311,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":93.873999999999995,"5min":122.20099999999999,"15min":122.20099999999999},"last":12.339},{"interface":"front","average":{"1min":13.999000000000001,"5min":6.9569999999999999,"15min":2.891},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":94.652000000000001,"5min":122.071,"15min":122.071},"last":7.1779999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":8.2420000000000009,"5min":6.8710000000000004,"15min":2.8610000000000002},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":72.137,"5min":99.204999999999998,"15min":99.204999999999998},"last":14.089},{"interface":"front","average":{"1min":13.818,"5min":6.9850000000000003,"15min":3.2029999999999998},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":95.795000000000002,"5min":95.795000000000002,"15min":95.795000000000002},"last":14.289}]},{"osd":4,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.9319999999999999,"15min":1.595},"min":{"1min":0.46200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0499999999999998,"5min":31.341999999999999,"15min":31.341999999999999},"last":2.5619999999999998},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.0150000000000001,"15min":1.5980000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9809999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":3.4100000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.9420000000000002,"15min":1.6479999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2370000000000001,"5min":109.369,"15min":109.369},"last":7.117},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.532,"15min":1.8049999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.1260000000000003,"5min":109.387,"15min":109.387},"last":2.5379999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.9590000000000001,"15min":1.6319999999999999},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.032,"5min":99.114999999999995,"15min":99.114999999999995},"last":3.4830000000000001},{"interface":"front","average":{"1min":1.171,"5min":2.8359999999999999,"15min":1.5580000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2910000000000004,"5min":98.921999999999997,"15min":98.921999999999997},"last":7.0739999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.0870000000000002,"15min":1.6759999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2640000000000002,"5min":99.549999999999997,"15min":99.549999999999997},"last":7.1509999999999998},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.4199999999999999,"15min":1.4890000000000001},"min":{"1min":0.497,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.093,"5min":31.225999999999999,"15min":31.225999999999999},"last":2.5800000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566202,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1137596,"kb_used_data":818548,"kb_used_omap":11,"kb_used_meta":318772,"kb_avail":92607556,"statfs":{"total":95995035648,"available":94830137344,"internally_reserved":0,"allocated":838193152,"data_stored":837281306,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":326423312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":10.598000000000001,"5min":4.6280000000000001,"15min":1.9299999999999999},"min":{"1min":0.215,"5min":0.184,"15min":0.184},"max":{"1min":87.132999999999996,"5min":87.132999999999996,"15min":87.132999999999996},"last":2.165},{"interface":"front","average":{"1min":7.6420000000000003,"5min":3.6579999999999999,"15min":1.595},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":84.820999999999998,"5min":84.820999999999998,"15min":84.820999999999998},"last":5.5919999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:14:47 2023","interfaces":[{"interface":"back","average":{"1min":9.2129999999999992,"5min":5.8739999999999997,"15min":5.3179999999999996},"min":{"1min":0.34899999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":84.641000000000005,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.89200000000000002},{"interface":"front","average":{"1min":9.2040000000000006,"5min":3.8799999999999999,"15min":2.992},"min":{"1min":0.34399999999999997,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":84.751999999999995,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.42699999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:29 2023","interfaces":[{"interface":"back","average":{"1min":14.097,"5min":6.1619999999999999,"15min":2.661},"min":{"1min":0.70399999999999996,"5min":0.219,"15min":0.219},"max":{"1min":86.332999999999998,"5min":87.108000000000004,"15min":87.108000000000004},"last":0.627},{"interface":"front","average":{"1min":12.151999999999999,"5min":5.577,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":1.3520000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.54,"5min":3.6659999999999999,"15min":1.7649999999999999},"min":{"1min":0.42299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":84.608000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.47299999999999998},{"interface":"front","average":{"1min":8.8379999999999992,"5min":4.8460000000000001,"15min":2.254},"min":{"1min":0.41099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6260000000000003,"5min":3.6459999999999999,"15min":1.8779999999999999},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":84.498000000000005,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.96199999999999997},{"interface":"front","average":{"1min":6.4429999999999996,"5min":3.5600000000000001,"15min":1.825},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":84.664000000000001,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.502}]},{"osd":6,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9589999999999996,"5min":3.9089999999999998,"15min":1.9490000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":84.715999999999994,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.69899999999999995},{"interface":"front","average":{"1min":9.0410000000000004,"5min":3.847,"15min":1.9430000000000001},"min":{"1min":0.44,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":87.055000000000007,"5min":87.055000000000007,"15min":87.055000000000007},"last":2.1299999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8410000000000002,"5min":3.2010000000000001,"15min":1.7350000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":84.596000000000004,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.99299999999999999},{"interface":"front","average":{"1min":6.7770000000000001,"5min":3.274,"15min":1.7210000000000001},"min":{"1min":0.46999999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":84.783000000000001,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.67000000000000004}]}]},{"osd":3,"up_from":110,"seq":472446402676,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303628,"kb_used_data":990084,"kb_used_omap":3,"kb_used_meta":313532,"kb_avail":92441524,"statfs":{"total":95995035648,"available":94660120576,"internally_reserved":0,"allocated":1013846016,"data_stored":1012972874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":321056937},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":3.3100000000000001,"15min":2.0110000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":13.064,"5min":61.744,"15min":79.611000000000004},"last":0.26200000000000001},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.2130000000000001,"15min":2.0249999999999999},"min":{"1min":0.40200000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":12.348000000000001,"5min":61.686,"15min":79.656000000000006},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:14:48 2023","interfaces":[{"interface":"back","average":{"1min":6.2789999999999999,"5min":4.9240000000000004,"15min":4.6980000000000004},"min":{"1min":0.34200000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":25.873999999999999,"5min":28.315000000000001,"15min":28.315000000000001},"last":6.1710000000000003},{"interface":"front","average":{"1min":4.7350000000000003,"5min":4.8230000000000004,"15min":4.8380000000000001},"min":{"1min":0.311,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.395,"5min":14.427,"15min":14.427},"last":0.44}]},{"osd":2,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8600000000000003,"5min":4.6020000000000003,"15min":3.6909999999999998},"min":{"1min":0.432,"5min":0.22,"15min":0.22},"max":{"1min":61.732999999999997,"5min":63.009999999999998,"15min":109.077},"last":5.556},{"interface":"front","average":{"1min":8.202,"5min":5.101,"15min":3.9460000000000002},"min":{"1min":0.47999999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":60.305999999999997,"5min":69.715999999999994,"15min":119.72799999999999},"last":0.68799999999999994}]},{"osd":4,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.6139999999999999,"15min":1.4319999999999999},"min":{"1min":0.36199999999999999,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":54.051000000000002,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.627},{"interface":"front","average":{"1min":5.2939999999999996,"5min":2.7029999999999998,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":52.228999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":5.6230000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":2.399,"15min":1.5740000000000001},"min":{"1min":0.40899999999999997,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":8.0459999999999994,"5min":8.9450000000000003,"15min":56.960999999999999},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.552,"15min":1.806},"min":{"1min":0.49299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.8730000000000002,"5min":10.738,"15min":56.981999999999999},"last":5.6840000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":3.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.42899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":78.102000000000004,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.66000000000000003},{"interface":"front","average":{"1min":6.5919999999999996,"5min":3.327,"15min":1.962},"min":{"1min":0.69299999999999995,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":78.028000000000006,"5min":78.028000000000006,"15min":78.028000000000006},"last":5.6580000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":2.5110000000000001,"15min":1.71},"min":{"1min":0.52200000000000002,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":5.923,"5min":10.603,"15min":14.885999999999999},"last":5.7190000000000003},{"interface":"front","average":{"1min":3.016,"5min":2.2160000000000002,"15min":1.363},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.109,"5min":14.351000000000001,"15min":14.351000000000001},"last":0.47999999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986441,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642704,"kb_used_data":451912,"kb_used_omap":0,"kb_used_meta":190784,"kb_avail":93102448,"statfs":{"total":95995035648,"available":95336906752,"internally_reserved":0,"allocated":462757888,"data_stored":462053072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":195362816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":3.375,"15min":2.5289999999999999},"min":{"1min":0.24099999999999999,"5min":0.219,"15min":0.219},"max":{"1min":65.988,"5min":66.605999999999995,"15min":122.69199999999999},"last":1.361},{"interface":"front","average":{"1min":5.4029999999999996,"5min":3.3639999999999999,"15min":2.6309999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":65.396000000000001,"5min":66.384,"15min":122.745},"last":0.39600000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":3.0779999999999998,"5min":4.2539999999999996,"15min":4.4500000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":28.783999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.54300000000000004},{"interface":"front","average":{"1min":3.5699999999999998,"5min":4.7750000000000004,"15min":4.976},"min":{"1min":0.27400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":28.815000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":10.114000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:14:57 2023","interfaces":[{"interface":"back","average":{"1min":2.2610000000000001,"5min":4.5140000000000002,"15min":2.883},"min":{"1min":0.27200000000000002,"5min":0.255,"15min":0.255},"max":{"1min":17.670999999999999,"5min":148.08000000000001,"15min":148.08000000000001},"last":4.9669999999999996},{"interface":"front","average":{"1min":3.0369999999999999,"5min":4.984,"15min":3.1829999999999998},"min":{"1min":0.41399999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":17.756,"5min":147.88900000000001,"15min":147.88900000000001},"last":1.085}]},{"osd":3,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":9.3309999999999995,"15min":3.7869999999999999},"min":{"1min":0.36799999999999999,"5min":0.252,"15min":0.252},"max":{"1min":96.447000000000003,"5min":163.97499999999999,"15min":163.97499999999999},"last":3.3300000000000001},{"interface":"front","average":{"1min":7.0019999999999998,"5min":9.1080000000000005,"15min":3.5750000000000002},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":95.944999999999993,"5min":164.447,"15min":164.447},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:15:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.966,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":87.912999999999997,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.29899999999999999},{"interface":"front","average":{"1min":5.4960000000000004,"5min":1.9450000000000001,"15min":1.395},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":88.602999999999994,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 15:15:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2169999999999996,"5min":4.2329999999999997,"15min":2.0449999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":86.091999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":5.4509999999999996,"5min":3.984,"15min":1.9319999999999999},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":86.048000000000002,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.41099999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.91800000000000004,"15min":1.129},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":0.75900000000000001,"5min":7.7389999999999999,"15min":18.23},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.3400000000000001,"15min":1.179},"min":{"1min":0.375,"5min":0.314,"15min":0.314},"max":{"1min":6.1399999999999997,"5min":14.044,"15min":14.653},"last":0.42499999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724804,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645832,"kb_used_data":441280,"kb_used_omap":0,"kb_used_meta":204544,"kb_avail":93099320,"statfs":{"total":95995035648,"available":95333703680,"internally_reserved":0,"allocated":451870720,"data_stored":451169883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":209453056},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":2.1829999999999998,"15min":1.292},"min":{"1min":0.22500000000000001,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":5.9530000000000003,"5min":17.044,"15min":24.962},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.2850000000000001,"5min":1.9039999999999999,"15min":1.226},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.222},"max":{"1min":6.2549999999999999,"5min":18.814,"15min":25.317},"last":5.032}]},{"osd":1,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":4.5030000000000001,"5min":2.2389999999999999,"15min":1.861},"min":{"1min":0.254,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":18.082999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":1.867},{"interface":"front","average":{"1min":2.9510000000000001,"5min":2.4049999999999998,"15min":2.3140000000000001},"min":{"1min":0.39200000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":17.963999999999999,"5min":17.963999999999999,"15min":17.963999999999999},"last":9.1010000000000009}]},{"osd":2,"last update":"Sat Sep 23 15:15:00 2023","interfaces":[{"interface":"back","average":{"1min":3.5920000000000001,"5min":4.7460000000000004,"15min":2.2989999999999999},"min":{"1min":0.30099999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":33.68,"5min":97.087999999999994,"15min":97.087999999999994},"last":5.157},{"interface":"front","average":{"1min":4.0510000000000002,"5min":4.8639999999999999,"15min":2.2930000000000001},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.254},"max":{"1min":33.707999999999998,"5min":97.305999999999997,"15min":97.305999999999997},"last":5.2789999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":10.388,"5min":5.0510000000000002,"15min":2.488},"min":{"1min":0.54900000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.570999999999998,"5min":80.570999999999998,"15min":80.570999999999998},"last":4.71},{"interface":"front","average":{"1min":9.8190000000000008,"5min":5.1440000000000001,"15min":2.3460000000000001},"min":{"1min":0.44400000000000001,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":80.551000000000002,"5min":80.551000000000002,"15min":80.551000000000002},"last":4.7320000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:14:46 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":2.1480000000000001,"15min":1.1639999999999999},"min":{"1min":0.28399999999999997,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.369,"5min":46.024000000000001,"15min":46.024000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.40799999999999997,"5min":2.2349999999999999,"15min":1.238},"min":{"1min":0.27200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.88200000000000001,"5min":45.795000000000002,"15min":45.795000000000002},"last":0.54100000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.494,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":11.076000000000001,"15min":15.196},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.3859999999999999,"15min":1.012},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":1.367,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.58599999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.3380000000000001,"15min":1.288},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.28599999999999998},"max":{"1min":8.2200000000000006,"5min":9.6560000000000006,"15min":15.301},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.141,"5min":1.3700000000000001,"15min":1.252},"min":{"1min":0.39900000000000002,"5min":0.311,"15min":0.311},"max":{"1min":5.6559999999999997,"5min":15.762,"15min":15.762},"last":0.47699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":355799040,"data_stored":355794968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":521801728,"data_stored":521797656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":829292544,"data_stored":829292544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":981860352,"data_stored":981860352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":453509120,"data_stored":453509120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":445059072,"data_stored":445055000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":719192064,"data_stored":719192064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:15:50.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:15:50.124 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:50.290 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:50.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:50.416 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:50.519 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:50.898 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:50.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:51.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:15:51.033 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:51.094 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:15:51.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:51.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:15:51.199 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:51.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:51.299 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:51.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:51.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:51.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:51.563 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:51.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:51 smithi027 ceph-mon[195233]: pgmap v585: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 22290/58449 objects degraded (38.136%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:51.688 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:51.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:51.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:51.735 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:51.744 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:51.804 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:15:51.806 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:51.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:51 smithi017 ceph-mon[215646]: pgmap v585: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 22290/58449 objects degraded (38.136%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:51.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:51 smithi017 ceph-mon[222878]: pgmap v585: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 523 op/s; 22290/58449 objects degraded (38.136%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:51.921 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:52.164 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:15:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:52.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:15:52.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:52.315 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:52.373 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:52.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:52.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22290/58449 objects degraded (38.136%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:52.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:52.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:52.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:52.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:52.859 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22290/58449 objects degraded (38.136%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:52.859 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22290/58449 objects degraded (38.136%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:52.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:52.985 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:53.044 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:53.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:53.050 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:53.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:15:53.241 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:53.974 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:15:53.994 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:15:54.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:54.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:15:54.104 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:54.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[222878]: pgmap v586: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 22290/58449 objects degraded (38.136%); 998 KiB/s, 30 objects/s recovering 2023-09-23T15:15:54.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3754048691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.212 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.215 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[215646]: pgmap v586: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 22290/58449 objects degraded (38.136%); 998 KiB/s, 30 objects/s recovering 2023-09-23T15:15:54.215 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3754048691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.215 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:53 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.225 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:53 smithi027 ceph-mon[195233]: pgmap v586: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 506 op/s; 22290/58449 objects degraded (38.136%); 998 KiB/s, 30 objects/s recovering 2023-09-23T15:15:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3754048691' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:53 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:15:54.983 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:54.983 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: failed to encode map e213 with expected crc 2023-09-23T15:15:54.983 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:54.983 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T15:15:54.983 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: failed to encode map e213 with expected crc 2023-09-23T15:15:54.984 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[215646]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:55.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T15:15:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:54 smithi027 ceph-mon[195233]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:15:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:15:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: osdmap e213: 8 total, 8 up, 7 in 2023-09-23T15:15:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:54 smithi017 ceph-mon[222878]: failed to encode map e213 with expected crc 2023-09-23T15:15:55.809 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:55.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:55.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:15:55.949 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:56.005 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:15:56.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:56.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:15:56.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:56.222 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:55 smithi017 ceph-mon[215646]: pgmap v588: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 22290/55122 objects degraded (40.438%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:56.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:56 smithi017 ceph-mon[222878]: pgmap v588: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 22290/55122 objects degraded (40.438%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:56.230 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:56.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:55 smithi027 ceph-mon[195233]: pgmap v588: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.1 GiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 515 op/s; 22290/55122 objects degraded (40.438%); 1.0 MiB/s, 30 objects/s recovering 2023-09-23T15:15:56.423 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:56.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:56.435 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:56.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:56.537 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:15:56.574 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:56.585 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:15:56.630 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:15:56.631 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:56.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:56.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:56.665 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:15:56.698 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:15:56.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:15:56.749 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:15:56.780 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:56.802 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:15:56.802 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:15:56.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:15:56.849 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:15:56.851 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:15:56.851 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.851 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.851 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.852 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:15:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:56.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:15:56.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:57.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1809877037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:57.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22290/55122 objects degraded (40.438%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:57.044 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:57.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1809877037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22290/55122 objects degraded (40.438%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:57.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1809877037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:15:57.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22290/55122 objects degraded (40.438%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:15:57.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:15:57.697 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:15:57.698 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22134/50781 objects degraded (43.587%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":765,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":213,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":16927,"data_bytes":1112250588,"bytes_used":5038170112,"bytes_avail":666927079424,"bytes_total":671965249536,"degraded_objects":22134,"degraded_total":50781,"degraded_ratio":0.4358716842913688,"recovering_objects_per_sec":30,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":305,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":510},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (6m)\n [==========================..] (remaining: 28s)","progress":0.93577980995178223,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:15:58.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:15:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:15:58] "GET /metrics HTTP/1.1" 200 36319 "" "Prometheus/2.43.0" 2023-09-23T15:15:58.324 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:58.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:58 smithi017 ceph-mon[222878]: pgmap v589: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:15:58.325 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2419977291' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:58.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:58 smithi017 ceph-mon[215646]: pgmap v589: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:15:58.326 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2419977291' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:58.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:58.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:58 smithi027 ceph-mon[195233]: pgmap v589: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:15:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2419977291' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:15:58.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:15:58.502 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:58.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:15:59.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:15:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:59.289 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:15:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:59.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:15:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:15:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:59.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:15:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:15:59.394 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:15:59.412 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:15:59.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:15:59.426 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:59.475 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:15:59.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:59.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:15:59.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:15:59.692 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:15:59.728 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:59.771 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:15:59.865 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:15:59.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:15:59.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:15:59.989 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:00.021 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:00.077 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:00.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:00.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:00.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:00.216 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:00.272 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:00.279 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:00 smithi027 ceph-mon[195233]: pgmap v590: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:00.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:00.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:00.300 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:00 smithi017 ceph-mon[215646]: pgmap v590: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:00.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:00 smithi017 ceph-mon[222878]: pgmap v590: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 510 op/s; 22134/50781 objects degraded (43.587%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:00.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:00.411 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:00.466 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:00.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:00.470 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:00.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:00.669 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:01.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:01.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:01.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:01.333 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:01.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:16:01.834 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:01.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:01.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:01.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:02.046 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:02.150 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:02.208 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:02.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:02.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:02.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:02.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:02 smithi017 ceph-mon[215646]: pgmap v591: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:02.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21992/43191 objects degraded (50.918%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:02.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:02 smithi017 ceph-mon[222878]: pgmap v591: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:02.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21992/43191 objects degraded (50.918%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:02.437 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:02 smithi027 ceph-mon[195233]: pgmap v591: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21992/43191 objects degraded (50.918%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:02.695 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:02.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:02.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:02.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:02.917 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:02.923 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:03.026 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:03.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:03.130 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:03.161 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:03.218 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:03.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:03.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:03.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:03.418 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:04.126 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:16:04.126 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:16:04.174 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:16:04.342 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:04.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:04.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:04.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[222878]: pgmap v592: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:04.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1341517797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:04.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:04.439 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[215646]: pgmap v592: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:04.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1341517797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:04.440 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:04.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:04.559 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:04 smithi027 ceph-mon[195233]: pgmap v592: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 903 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 508 op/s; 21992/43191 objects degraded (50.918%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1341517797' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:05.302 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:05.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:05.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:05.413 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.442 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.442 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:05 smithi027 ceph-mon[195233]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.445 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[215646]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: osdmap e214: 8 total, 8 up, 7 in 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:05 smithi017 ceph-mon[222878]: failed to encode map e214 with expected crc 2023-09-23T15:16:05.514 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:05.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:05.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:05.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:05.699 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:06.100 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:06.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:06.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:06.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:06.242 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:06.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:06 smithi017 ceph-mon[215646]: pgmap v594: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 826 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 21992/39525 objects degraded (55.641%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:06.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:06 smithi017 ceph-mon[222878]: pgmap v594: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 826 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 21992/39525 objects degraded (55.641%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:06.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:16:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:06 smithi027 ceph-mon[195233]: pgmap v594: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 826 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 519 op/s; 21992/39525 objects degraded (55.641%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:06.705 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:06.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:06.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:06.825 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:06.932 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:07.129 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:07.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:07.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:07.242 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:07.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:16:07.276 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:07.326 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:07.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:07.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:07.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:07.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:07 smithi017 ceph-mon[215646]: pgmap v595: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:07.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21992/39525 objects degraded (55.641%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:07.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:07 smithi017 ceph-mon[222878]: pgmap v595: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:07.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21992/39525 objects degraded (55.641%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:07.560 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:07.560 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:07.598 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:16:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:07 smithi027 ceph-mon[195233]: pgmap v595: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21992/39525 objects degraded (55.641%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:07.756 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:07.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:07.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:07.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:07.938 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:08.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:08] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:16:08.370 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:08.370 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:08.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:08.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:08.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:08.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:08.399 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:08.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.434 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:16:08.467 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:08 smithi017 ceph-mon[222878]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:08.467 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2440955646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:08.474 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:08 smithi017 ceph-mon[215646]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:08.474 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2440955646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:08.484 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:08 smithi027 ceph-mon[195233]: from='client.55282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2440955646' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:08.706 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:08.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:08.713 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:08.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:08.889 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:09.347 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":595,"stamp":"2023-09-23T15:16:06.431700+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":765958364,"num_objects":11643,"num_object_clones":0,"num_object_copies":34929,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21834,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11643,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":190190,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7928,"num_bytes_recovered":481400999,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124575,"ondisk_log_size":124575,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4054548,"kb_used_data":2369244,"kb_used_omap":30,"kb_used_meta":1685089,"kb_avail":652161516,"statfs":{"total":671965249536,"available":667813392384,"internally_reserved":0,"allocated":2426105856,"data_stored":2420227247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1725531736},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-346292224,"num_objects":-5284,"num_object_clones":0,"num_object_copies":-15852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5284,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":300,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.010207"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":208,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943643+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:15:53.943643+0000","last_peered":"2023-09-23T15:15:53.943643+0000","last_clean":"2023-09-23T15:15:53.943643+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:15:53.943643+0000","last_undegraded":"2023-09-23T15:15:53.943643+0000","last_fullsized":"2023-09-23T15:15:53.943643+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":142,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950677+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:15:53.950677+0000","last_peered":"2023-09-23T15:15:53.950677+0000","last_clean":"2023-09-23T15:15:53.950677+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:15:53.950677+0000","last_undegraded":"2023-09-23T15:15:53.950677+0000","last_fullsized":"2023-09-23T15:15:53.950677+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":90,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.102657+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:15:42.102657+0000","last_peered":"2023-09-23T15:15:42.102657+0000","last_clean":"2023-09-23T15:15:42.102657+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:15:42.102657+0000","last_undegraded":"2023-09-23T15:15:42.102657+0000","last_fullsized":"2023-09-23T15:15:42.102657+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960612+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:15:53.960612+0000","last_peered":"2023-09-23T15:15:53.960612+0000","last_clean":"2023-09-23T15:15:53.960612+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:15:53.960612+0000","last_undegraded":"2023-09-23T15:15:53.960612+0000","last_fullsized":"2023-09-23T15:15:53.960612+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.750886+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:15:54.750886+0000","last_peered":"2023-09-23T15:15:54.750886+0000","last_clean":"2023-09-23T15:15:54.750886+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:15:54.750886+0000","last_undegraded":"2023-09-23T15:15:54.750886+0000","last_fullsized":"2023-09-23T15:15:54.750886+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":188,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964217+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:15:53.964217+0000","last_peered":"2023-09-23T15:15:53.964217+0000","last_clean":"2023-09-23T15:15:53.964217+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:15:53.964217+0000","last_undegraded":"2023-09-23T15:15:53.964217+0000","last_fullsized":"2023-09-23T15:15:53.964217+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961153+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:15:53.961153+0000","last_peered":"2023-09-23T15:15:53.961153+0000","last_clean":"2023-09-23T15:15:53.961153+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:15:53.961153+0000","last_undegraded":"2023-09-23T15:15:53.961153+0000","last_fullsized":"2023-09-23T15:15:53.961153+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950489+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:15:53.950489+0000","last_peered":"2023-09-23T15:15:53.950489+0000","last_clean":"2023-09-23T15:15:53.950489+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:15:53.950489+0000","last_undegraded":"2023-09-23T15:15:53.950489+0000","last_fullsized":"2023-09-23T15:15:53.950489+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":490,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960239+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:15:53.960239+0000","last_peered":"2023-09-23T15:15:53.960239+0000","last_clean":"2023-09-23T15:15:53.960239+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:15:53.960239+0000","last_undegraded":"2023-09-23T15:15:53.960239+0000","last_fullsized":"2023-09-23T15:15:53.960239+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":204,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959464+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:15:53.959464+0000","last_peered":"2023-09-23T15:15:53.959464+0000","last_clean":"2023-09-23T15:15:53.959464+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:15:53.959464+0000","last_undegraded":"2023-09-23T15:15:53.959464+0000","last_fullsized":"2023-09-23T15:15:53.959464+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":137,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950222+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:15:53.950222+0000","last_peered":"2023-09-23T15:15:53.950222+0000","last_clean":"2023-09-23T15:15:53.950222+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:15:53.950222+0000","last_undegraded":"2023-09-23T15:15:53.950222+0000","last_fullsized":"2023-09-23T15:15:53.950222+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":76,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950612+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:15:53.950612+0000","last_peered":"2023-09-23T15:15:53.950612+0000","last_clean":"2023-09-23T15:15:53.950612+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:15:53.950612+0000","last_undegraded":"2023-09-23T15:15:53.950612+0000","last_fullsized":"2023-09-23T15:15:53.950612+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":85,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.102188+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:42.102188+0000","last_peered":"2023-09-23T15:15:42.102188+0000","last_clean":"2023-09-23T15:15:42.102188+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:42.102188+0000","last_undegraded":"2023-09-23T15:15:42.102188+0000","last_fullsized":"2023-09-23T15:15:42.102188+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":198,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943599+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:15:53.943599+0000","last_peered":"2023-09-23T15:15:53.943599+0000","last_clean":"2023-09-23T15:15:53.943599+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:15:53.943599+0000","last_undegraded":"2023-09-23T15:15:53.943599+0000","last_fullsized":"2023-09-23T15:15:53.943599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943616+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:15:53.943616+0000","last_peered":"2023-09-23T15:15:53.943616+0000","last_clean":"2023-09-23T15:15:53.943616+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:15:53.943616+0000","last_undegraded":"2023-09-23T15:15:53.943616+0000","last_fullsized":"2023-09-23T15:15:53.943616+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":502,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964028+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:15:53.964028+0000","last_peered":"2023-09-23T15:15:53.964028+0000","last_clean":"2023-09-23T15:15:53.964028+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:15:53.964028+0000","last_undegraded":"2023-09-23T15:15:53.964028+0000","last_fullsized":"2023-09-23T15:15:53.964028+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":76,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943677+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:15:53.943677+0000","last_peered":"2023-09-23T15:15:53.943677+0000","last_clean":"2023-09-23T15:15:53.943677+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:15:53.943677+0000","last_undegraded":"2023-09-23T15:15:53.943677+0000","last_fullsized":"2023-09-23T15:15:53.943677+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.955897+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:15:53.955897+0000","last_peered":"2023-09-23T15:15:53.955897+0000","last_clean":"2023-09-23T15:15:53.955897+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:15:53.955897+0000","last_undegraded":"2023-09-23T15:15:53.955897+0000","last_fullsized":"2023-09-23T15:15:53.955897+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":502,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961188+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:15:53.961188+0000","last_peered":"2023-09-23T15:15:53.961188+0000","last_clean":"2023-09-23T15:15:53.961188+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:15:53.961188+0000","last_undegraded":"2023-09-23T15:15:53.961188+0000","last_fullsized":"2023-09-23T15:15:53.961188+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960663+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:15:53.960663+0000","last_peered":"2023-09-23T15:15:53.960663+0000","last_clean":"2023-09-23T15:15:53.960663+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:15:53.960663+0000","last_undegraded":"2023-09-23T15:15:53.960663+0000","last_fullsized":"2023-09-23T15:15:53.960663+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959599+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:15:53.959599+0000","last_peered":"2023-09-23T15:15:53.959599+0000","last_clean":"2023-09-23T15:15:53.959599+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:15:53.959599+0000","last_undegraded":"2023-09-23T15:15:53.959599+0000","last_fullsized":"2023-09-23T15:15:53.959599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":490,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959433+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:15:53.959433+0000","last_peered":"2023-09-23T15:15:53.959433+0000","last_clean":"2023-09-23T15:15:53.959433+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:15:53.959433+0000","last_undegraded":"2023-09-23T15:15:53.959433+0000","last_fullsized":"2023-09-23T15:15:53.959433+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943558+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:15:53.943558+0000","last_peered":"2023-09-23T15:15:53.943558+0000","last_clean":"2023-09-23T15:15:53.943558+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:15:53.943558+0000","last_undegraded":"2023-09-23T15:15:53.943558+0000","last_fullsized":"2023-09-23T15:15:53.943558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960691+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:15:53.960691+0000","last_peered":"2023-09-23T15:15:53.960691+0000","last_clean":"2023-09-23T15:15:53.960691+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:15:53.960691+0000","last_undegraded":"2023-09-23T15:15:53.960691+0000","last_fullsized":"2023-09-23T15:15:53.960691+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":502,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960985+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:15:53.960985+0000","last_peered":"2023-09-23T15:15:53.960985+0000","last_clean":"2023-09-23T15:15:53.960985+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:15:53.960985+0000","last_undegraded":"2023-09-23T15:15:53.960985+0000","last_fullsized":"2023-09-23T15:15:53.960985+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963149+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:15:53.963149+0000","last_peered":"2023-09-23T15:15:53.963149+0000","last_clean":"2023-09-23T15:15:53.963149+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:15:53.963149+0000","last_undegraded":"2023-09-23T15:15:53.963149+0000","last_fullsized":"2023-09-23T15:15:53.963149+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.751052+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:15:54.751052+0000","last_peered":"2023-09-23T15:15:54.751052+0000","last_clean":"2023-09-23T15:15:54.751052+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:15:54.751052+0000","last_undegraded":"2023-09-23T15:15:54.751052+0000","last_fullsized":"2023-09-23T15:15:54.751052+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"214'11711","reported_seq":11762,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.682703+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:16:04.682703+0000","last_peered":"2023-09-23T15:16:04.682703+0000","last_clean":"2023-09-23T15:16:04.682703+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:16:04.682703+0000","last_undegraded":"2023-09-23T15:16:04.682703+0000","last_fullsized":"2023-09-23T15:16:04.682703+0000","mapping_epoch":171,"log_start":"175'1700","ondisk_log_start":"175'1700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10011,"log_dups_size":0,"ondisk_log_size":10011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44761088,"num_objects":683,"num_object_clones":0,"num_object_copies":2049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11711,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.152749+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:16:04.152749+0000","last_peered":"2023-09-23T15:16:04.152749+0000","last_clean":"2023-09-23T15:16:04.152749+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:16:04.152749+0000","last_undegraded":"2023-09-23T15:16:04.152749+0000","last_fullsized":"2023-09-23T15:16:04.152749+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":88,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.944029+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:15:53.944029+0000","last_peered":"2023-09-23T15:15:53.944029+0000","last_clean":"2023-09-23T15:15:53.944029+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:15:53.944029+0000","last_undegraded":"2023-09-23T15:15:53.944029+0000","last_fullsized":"2023-09-23T15:15:53.944029+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959174+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:15:53.959174+0000","last_peered":"2023-09-23T15:15:53.959174+0000","last_clean":"2023-09-23T15:15:53.959174+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:15:53.959174+0000","last_undegraded":"2023-09-23T15:15:53.959174+0000","last_fullsized":"2023-09-23T15:15:53.959174+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"214'11966","reported_seq":17964,"reported_epoch":214,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.753503+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:16:04.753503+0000","last_peered":"2023-09-23T15:16:04.753503+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:16:04.753503+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"190'5500","ondisk_log_start":"190'5500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6466,"log_dups_size":0,"ondisk_log_size":6466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45875200,"num_objects":700,"num_object_clones":0,"num_object_copies":2100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2029,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11966,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2969,"num_bytes_recovered":154927104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1329}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":482,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.150470+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:16:04.150470+0000","last_peered":"2023-09-23T15:16:04.150470+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:16:04.150470+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":88,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943997+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:15:53.943997+0000","last_peered":"2023-09-23T15:15:53.943997+0000","last_clean":"2023-09-23T15:15:53.943997+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:15:53.943997+0000","last_undegraded":"2023-09-23T15:15:53.943997+0000","last_fullsized":"2023-09-23T15:15:53.943997+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943456+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:15:53.943456+0000","last_peered":"2023-09-23T15:15:53.943456+0000","last_clean":"2023-09-23T15:15:53.943456+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:15:53.943456+0000","last_undegraded":"2023-09-23T15:15:53.943456+0000","last_fullsized":"2023-09-23T15:15:53.943456+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"213'11676","reported_seq":11728,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:16:03.650520+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:16:03.650520+0000","last_peered":"2023-09-23T15:16:03.650520+0000","last_clean":"2023-09-23T15:16:03.650520+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:16:03.650520+0000","last_undegraded":"2023-09-23T15:16:03.650520+0000","last_fullsized":"2023-09-23T15:16:03.650520+0000","mapping_epoch":171,"log_start":"177'2500","ondisk_log_start":"177'2500","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9176,"log_dups_size":0,"ondisk_log_size":9176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50200576,"num_objects":766,"num_object_clones":0,"num_object_copies":2298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11676,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.151332+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:16:04.151332+0000","last_peered":"2023-09-23T15:16:04.151332+0000","last_clean":"2023-09-23T15:16:04.151332+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:16:04.151332+0000","last_undegraded":"2023-09-23T15:16:04.151332+0000","last_fullsized":"2023-09-23T15:16:04.151332+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":74,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959577+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:15:53.959577+0000","last_peered":"2023-09-23T15:15:53.959577+0000","last_clean":"2023-09-23T15:15:53.959577+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:15:53.959577+0000","last_undegraded":"2023-09-23T15:15:53.959577+0000","last_fullsized":"2023-09-23T15:15:53.959577+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963880+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:15:53.963880+0000","last_peered":"2023-09-23T15:15:53.963880+0000","last_clean":"2023-09-23T15:15:53.963880+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:15:53.963880+0000","last_undegraded":"2023-09-23T15:15:53.963880+0000","last_fullsized":"2023-09-23T15:15:53.963880+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"214'11909","reported_seq":11962,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.450716+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:16:04.450716+0000","last_peered":"2023-09-23T15:16:04.450716+0000","last_clean":"2023-09-23T15:16:04.450716+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:16:04.450716+0000","last_undegraded":"2023-09-23T15:16:04.450716+0000","last_fullsized":"2023-09-23T15:16:04.450716+0000","mapping_epoch":171,"log_start":"195'6300","ondisk_log_start":"195'6300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5609,"log_dups_size":0,"ondisk_log_size":5609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11909,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.150884+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:16:04.150884+0000","last_peered":"2023-09-23T15:16:04.150884+0000","last_clean":"2023-09-23T15:16:04.150884+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:16:04.150884+0000","last_undegraded":"2023-09-23T15:16:04.150884+0000","last_fullsized":"2023-09-23T15:16:04.150884+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959513+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:53.959513+0000","last_peered":"2023-09-23T15:15:53.959513+0000","last_clean":"2023-09-23T15:15:53.959513+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:15:53.959513+0000","last_undegraded":"2023-09-23T15:15:53.959513+0000","last_fullsized":"2023-09-23T15:15:53.959513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":174,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465941+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:15:54.465941+0000","last_peered":"2023-09-23T15:15:54.465941+0000","last_clean":"2023-09-23T15:15:54.465941+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:15:54.465941+0000","last_undegraded":"2023-09-23T15:15:54.465941+0000","last_fullsized":"2023-09-23T15:15:54.465941+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"214'12149","reported_seq":12203,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.349310+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:16:04.349310+0000","last_peered":"2023-09-23T15:16:04.349310+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:16:04.349310+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"180'4600","ondisk_log_start":"180'4600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50003968,"num_objects":763,"num_object_clones":0,"num_object_copies":2289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12149,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-4247}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465823+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:15:54.465823+0000","last_peered":"2023-09-23T15:15:54.465823+0000","last_clean":"2023-09-23T15:15:54.465823+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:15:54.465823+0000","last_undegraded":"2023-09-23T15:15:54.465823+0000","last_fullsized":"2023-09-23T15:15:54.465823+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.953257+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:15:53.953257+0000","last_peered":"2023-09-23T15:15:53.953257+0000","last_clean":"2023-09-23T15:15:53.953257+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:15:53.953257+0000","last_undegraded":"2023-09-23T15:15:53.953257+0000","last_fullsized":"2023-09-23T15:15:53.953257+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943637+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:15:53.943637+0000","last_peered":"2023-09-23T15:15:53.943637+0000","last_clean":"2023-09-23T15:15:53.943637+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:15:53.943637+0000","last_undegraded":"2023-09-23T15:15:53.943637+0000","last_fullsized":"2023-09-23T15:15:53.943637+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"213'11845","reported_seq":11897,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:16:03.634070+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:16:03.634070+0000","last_peered":"2023-09-23T15:16:03.634070+0000","last_clean":"2023-09-23T15:16:03.634070+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:16:03.634070+0000","last_undegraded":"2023-09-23T15:16:03.634070+0000","last_fullsized":"2023-09-23T15:16:03.634070+0000","mapping_epoch":171,"log_start":"177'2700","ondisk_log_start":"177'2700","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9145,"log_dups_size":0,"ondisk_log_size":9145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48037888,"num_objects":733,"num_object_clones":0,"num_object_copies":2199,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":733,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11845,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":508,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465780+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:15:54.465780+0000","last_peered":"2023-09-23T15:15:54.465780+0000","last_clean":"2023-09-23T15:15:54.465780+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:15:54.465780+0000","last_undegraded":"2023-09-23T15:15:54.465780+0000","last_fullsized":"2023-09-23T15:15:54.465780+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963417+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:15:53.963417+0000","last_peered":"2023-09-23T15:15:53.963417+0000","last_clean":"2023-09-23T15:15:53.963417+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:15:53.963417+0000","last_undegraded":"2023-09-23T15:15:53.963417+0000","last_fullsized":"2023-09-23T15:15:53.963417+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943731+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:15:53.943731+0000","last_peered":"2023-09-23T15:15:53.943731+0000","last_clean":"2023-09-23T15:15:53.943731+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:15:53.943731+0000","last_undegraded":"2023-09-23T15:15:53.943731+0000","last_fullsized":"2023-09-23T15:15:53.943731+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"214'11859","reported_seq":11912,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.765698+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:16:04.765698+0000","last_peered":"2023-09-23T15:16:04.765698+0000","last_clean":"2023-09-23T15:16:04.765698+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:16:04.765698+0000","last_undegraded":"2023-09-23T15:16:04.765698+0000","last_fullsized":"2023-09-23T15:16:04.765698+0000","mapping_epoch":171,"log_start":"190'5400","ondisk_log_start":"190'5400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6459,"log_dups_size":0,"ondisk_log_size":6459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47120384,"num_objects":719,"num_object_clones":0,"num_object_copies":2157,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":719,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11859,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.149936+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:16:04.149936+0000","last_peered":"2023-09-23T15:16:04.149936+0000","last_clean":"2023-09-23T15:16:04.149936+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:16:04.149936+0000","last_undegraded":"2023-09-23T15:16:04.149936+0000","last_fullsized":"2023-09-23T15:16:04.149936+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943370+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:15:53.943370+0000","last_peered":"2023-09-23T15:15:53.943370+0000","last_clean":"2023-09-23T15:15:53.943370+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:15:53.943370+0000","last_undegraded":"2023-09-23T15:15:53.943370+0000","last_fullsized":"2023-09-23T15:15:53.943370+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.944061+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:15:53.944061+0000","last_peered":"2023-09-23T15:15:53.944061+0000","last_clean":"2023-09-23T15:15:53.944061+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:15:53.944061+0000","last_undegraded":"2023-09-23T15:15:53.944061+0000","last_fullsized":"2023-09-23T15:15:53.944061+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"214'11830","reported_seq":21702,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.706092+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:16:04.706092+0000","last_peered":"2023-09-23T15:16:04.706092+0000","last_clean":"2023-09-23T15:16:04.706092+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:16:04.706092+0000","last_undegraded":"2023-09-23T15:16:04.706092+0000","last_fullsized":"2023-09-23T15:16:04.706092+0000","mapping_epoch":203,"log_start":"175'1800","ondisk_log_start":"175'1800","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10030,"log_dups_size":0,"ondisk_log_size":10030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45940760,"num_objects":702,"num_object_clones":0,"num_object_copies":2106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":702,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11830,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465887+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:15:54.465887+0000","last_peered":"2023-09-23T15:15:54.465887+0000","last_clean":"2023-09-23T15:15:54.465887+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:15:54.465887+0000","last_undegraded":"2023-09-23T15:15:54.465887+0000","last_fullsized":"2023-09-23T15:15:54.465887+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950295+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:15:53.950295+0000","last_peered":"2023-09-23T15:15:53.950295+0000","last_clean":"2023-09-23T15:15:53.950295+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:15:53.950295+0000","last_undegraded":"2023-09-23T15:15:53.950295+0000","last_fullsized":"2023-09-23T15:15:53.950295+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":90,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.107279+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:42.107279+0000","last_peered":"2023-09-23T15:15:42.107279+0000","last_clean":"2023-09-23T15:15:42.107279+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:42.107279+0000","last_undegraded":"2023-09-23T15:15:42.107279+0000","last_fullsized":"2023-09-23T15:15:42.107279+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"214'11868","reported_seq":11922,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.421481+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:16:04.421481+0000","last_peered":"2023-09-23T15:16:04.421481+0000","last_clean":"2023-09-23T15:16:04.421481+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:16:04.421481+0000","last_undegraded":"2023-09-23T15:16:04.421481+0000","last_fullsized":"2023-09-23T15:16:04.421481+0000","mapping_epoch":171,"log_start":"195'6300","ondisk_log_start":"195'6300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45875200,"num_objects":700,"num_object_clones":0,"num_object_copies":2100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.466116+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:15:54.466116+0000","last_peered":"2023-09-23T15:15:54.466116+0000","last_clean":"2023-09-23T15:15:54.466116+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:15:54.466116+0000","last_undegraded":"2023-09-23T15:15:54.466116+0000","last_fullsized":"2023-09-23T15:15:54.466116+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":389,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.107989+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:15:42.107989+0000","last_peered":"2023-09-23T15:15:42.107989+0000","last_clean":"2023-09-23T15:15:42.107989+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:15:42.107989+0000","last_undegraded":"2023-09-23T15:15:42.107989+0000","last_fullsized":"2023-09-23T15:15:42.107989+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959515+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:53.959515+0000","last_peered":"2023-09-23T15:15:53.959515+0000","last_clean":"2023-09-23T15:15:53.959515+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:15:53.959515+0000","last_undegraded":"2023-09-23T15:15:53.959515+0000","last_fullsized":"2023-09-23T15:15:53.959515+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.109736+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:15:42.109736+0000","last_peered":"2023-09-23T15:15:42.109736+0000","last_clean":"2023-09-23T15:15:42.109736+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:15:42.109736+0000","last_undegraded":"2023-09-23T15:15:42.109736+0000","last_fullsized":"2023-09-23T15:15:42.109736+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"214'11649","reported_seq":11703,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.769347+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:16:04.769347+0000","last_peered":"2023-09-23T15:16:04.769347+0000","last_clean":"2023-09-23T15:16:04.769347+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:16:04.769347+0000","last_undegraded":"2023-09-23T15:16:04.769347+0000","last_fullsized":"2023-09-23T15:16:04.769347+0000","mapping_epoch":171,"log_start":"189'5200","ondisk_log_start":"189'5200","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6449,"log_dups_size":0,"ondisk_log_size":6449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46202880,"num_objects":705,"num_object_clones":0,"num_object_copies":2115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11649,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964131+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:15:53.964131+0000","last_peered":"2023-09-23T15:15:53.964131+0000","last_clean":"2023-09-23T15:15:53.964131+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:15:53.964131+0000","last_undegraded":"2023-09-23T15:15:53.964131+0000","last_fullsized":"2023-09-23T15:15:53.964131+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":732,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959346+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:15:53.959346+0000","last_peered":"2023-09-23T15:15:53.959346+0000","last_clean":"2023-09-23T15:15:53.959346+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:15:53.959346+0000","last_undegraded":"2023-09-23T15:15:53.959346+0000","last_fullsized":"2023-09-23T15:15:53.959346+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943609+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:15:53.943609+0000","last_peered":"2023-09-23T15:15:53.943609+0000","last_clean":"2023-09-23T15:15:53.943609+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:15:53.943609+0000","last_undegraded":"2023-09-23T15:15:53.943609+0000","last_fullsized":"2023-09-23T15:15:53.943609+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":105,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.107696+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:15:42.107696+0000","last_peered":"2023-09-23T15:15:42.107696+0000","last_clean":"2023-09-23T15:15:42.107696+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:15:42.107696+0000","last_undegraded":"2023-09-23T15:15:42.107696+0000","last_fullsized":"2023-09-23T15:15:42.107696+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"214'11871","reported_seq":11922,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.213885+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:16:04.213885+0000","last_peered":"2023-09-23T15:16:04.213885+0000","last_clean":"2023-09-23T15:16:04.213885+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:16:04.213885+0000","last_undegraded":"2023-09-23T15:16:04.213885+0000","last_fullsized":"2023-09-23T15:16:04.213885+0000","mapping_epoch":171,"log_start":"197'6600","ondisk_log_start":"197'6600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5271,"log_dups_size":0,"ondisk_log_size":5271,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11871,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960751+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:15:53.960751+0000","last_peered":"2023-09-23T15:15:53.960751+0000","last_clean":"2023-09-23T15:15:53.960751+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:15:53.960751+0000","last_undegraded":"2023-09-23T15:15:53.960751+0000","last_fullsized":"2023-09-23T15:15:53.960751+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":825,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.119576+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:15:42.119576+0000","last_peered":"2023-09-23T15:15:42.119576+0000","last_clean":"2023-09-23T15:15:42.119576+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:15:42.119576+0000","last_undegraded":"2023-09-23T15:15:42.119576+0000","last_fullsized":"2023-09-23T15:15:42.119576+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":85,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.101646+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:15:42.101646+0000","last_peered":"2023-09-23T15:15:42.101646+0000","last_clean":"2023-09-23T15:15:42.101646+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:15:42.101646+0000","last_undegraded":"2023-09-23T15:15:42.101646+0000","last_fullsized":"2023-09-23T15:15:42.101646+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":109,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.107200+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:15:42.107200+0000","last_peered":"2023-09-23T15:15:42.107200+0000","last_clean":"2023-09-23T15:15:42.107200+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:15:42.107200+0000","last_undegraded":"2023-09-23T15:15:42.107200+0000","last_fullsized":"2023-09-23T15:15:42.107200+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"214'11659","reported_seq":11710,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.672759+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:16:04.672759+0000","last_peered":"2023-09-23T15:16:04.672759+0000","last_clean":"2023-09-23T15:16:04.672759+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:16:04.672759+0000","last_undegraded":"2023-09-23T15:16:04.672759+0000","last_fullsized":"2023-09-23T15:16:04.672759+0000","mapping_epoch":171,"log_start":"175'1600","ondisk_log_start":"175'1600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10059,"log_dups_size":0,"ondisk_log_size":10059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47251456,"num_objects":721,"num_object_clones":0,"num_object_copies":2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11659,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":501,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959356+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:15:53.959356+0000","last_peered":"2023-09-23T15:15:53.959356+0000","last_clean":"2023-09-23T15:15:53.959356+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:15:53.959356+0000","last_undegraded":"2023-09-23T15:15:53.959356+0000","last_fullsized":"2023-09-23T15:15:53.959356+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961421+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:15:53.961421+0000","last_peered":"2023-09-23T15:15:53.961421+0000","last_clean":"2023-09-23T15:15:53.961421+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:15:53.961421+0000","last_undegraded":"2023-09-23T15:15:53.961421+0000","last_fullsized":"2023-09-23T15:15:53.961421+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.951319+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:15:53.951319+0000","last_peered":"2023-09-23T15:15:53.951319+0000","last_clean":"2023-09-23T15:15:53.951319+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:15:53.951319+0000","last_undegraded":"2023-09-23T15:15:53.951319+0000","last_fullsized":"2023-09-23T15:15:53.951319+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964346+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:15:53.964346+0000","last_peered":"2023-09-23T15:15:53.964346+0000","last_clean":"2023-09-23T15:15:53.964346+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:15:53.964346+0000","last_undegraded":"2023-09-23T15:15:53.964346+0000","last_fullsized":"2023-09-23T15:15:53.964346+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"214'11931","reported_seq":11985,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.725575+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:16:04.725575+0000","last_peered":"2023-09-23T15:16:04.725575+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:16:04.725575+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"190'5500","ondisk_log_start":"190'5500","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6431,"log_dups_size":0,"ondisk_log_size":6431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46727168,"num_objects":713,"num_object_clones":0,"num_object_copies":2139,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":713,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11931,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-4239}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959592+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:15:53.959592+0000","last_peered":"2023-09-23T15:15:53.959592+0000","last_clean":"2023-09-23T15:15:53.959592+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:15:53.959592+0000","last_undegraded":"2023-09-23T15:15:53.959592+0000","last_fullsized":"2023-09-23T15:15:53.959592+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":296,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.101404+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:15:42.101404+0000","last_peered":"2023-09-23T15:15:42.101404+0000","last_clean":"2023-09-23T15:15:42.101404+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:15:42.101404+0000","last_undegraded":"2023-09-23T15:15:42.101404+0000","last_fullsized":"2023-09-23T15:15:42.101404+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":159,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963945+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:53.963945+0000","last_peered":"2023-09-23T15:15:53.963945+0000","last_clean":"2023-09-23T15:15:53.963945+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:53.963945+0000","last_undegraded":"2023-09-23T15:15:53.963945+0000","last_fullsized":"2023-09-23T15:15:53.963945+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960446+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:15:53.960446+0000","last_peered":"2023-09-23T15:15:53.960446+0000","last_clean":"2023-09-23T15:15:53.960446+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:15:53.960446+0000","last_undegraded":"2023-09-23T15:15:53.960446+0000","last_fullsized":"2023-09-23T15:15:53.960446+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"213'11719","reported_seq":11771,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:03.583786+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:16:03.583786+0000","last_peered":"2023-09-23T15:16:03.583786+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:16:03.583786+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"177'2600","ondisk_log_start":"177'2600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9119,"log_dups_size":0,"ondisk_log_size":9119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51707904,"num_objects":789,"num_object_clones":0,"num_object_copies":2367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11719,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-4085}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.466088+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:15:54.466088+0000","last_peered":"2023-09-23T15:15:54.466088+0000","last_clean":"2023-09-23T15:15:54.466088+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:15:54.466088+0000","last_undegraded":"2023-09-23T15:15:54.466088+0000","last_fullsized":"2023-09-23T15:15:54.466088+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":820,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:53.961575+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:15:53.961575+0000","last_peered":"2023-09-23T15:15:53.961575+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:15:53.961575+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":85,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.107873+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:15:42.107873+0000","last_peered":"2023-09-23T15:15:42.107873+0000","last_clean":"2023-09-23T15:15:42.107873+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:15:42.107873+0000","last_undegraded":"2023-09-23T15:15:42.107873+0000","last_fullsized":"2023-09-23T15:15:42.107873+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959894+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:53.959894+0000","last_peered":"2023-09-23T15:15:53.959894+0000","last_clean":"2023-09-23T15:15:53.959894+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:53.959894+0000","last_undegraded":"2023-09-23T15:15:53.959894+0000","last_fullsized":"2023-09-23T15:15:53.959894+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"214'11606","reported_seq":11657,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.338043+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:16:04.338043+0000","last_peered":"2023-09-23T15:16:04.338043+0000","last_clean":"2023-09-23T15:16:04.338043+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:16:04.338043+0000","last_undegraded":"2023-09-23T15:16:04.338043+0000","last_fullsized":"2023-09-23T15:16:04.338043+0000","mapping_epoch":171,"log_start":"179'4100","ondisk_log_start":"179'4100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7506,"log_dups_size":0,"ondisk_log_size":7506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45613056,"num_objects":696,"num_object_clones":0,"num_object_copies":2088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11606,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961729+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:15:53.961729+0000","last_peered":"2023-09-23T15:15:53.961729+0000","last_clean":"2023-09-23T15:15:53.961729+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:15:53.961729+0000","last_undegraded":"2023-09-23T15:15:53.961729+0000","last_fullsized":"2023-09-23T15:15:53.961729+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960200+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:15:53.960200+0000","last_peered":"2023-09-23T15:15:53.960200+0000","last_clean":"2023-09-23T15:15:53.960200+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:15:53.960200+0000","last_undegraded":"2023-09-23T15:15:53.960200+0000","last_fullsized":"2023-09-23T15:15:53.960200+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":673,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:42.108453+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:15:42.108453+0000","last_peered":"2023-09-23T15:15:42.108453+0000","last_clean":"2023-09-23T15:15:42.108453+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:15:42.108453+0000","last_undegraded":"2023-09-23T15:15:42.108453+0000","last_fullsized":"2023-09-23T15:15:42.108453+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950563+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:15:53.950563+0000","last_peered":"2023-09-23T15:15:53.950563+0000","last_clean":"2023-09-23T15:15:53.950563+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:15:53.950563+0000","last_undegraded":"2023-09-23T15:15:53.950563+0000","last_fullsized":"2023-09-23T15:15:53.950563+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"214'11914","reported_seq":11972,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.199227+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:16:04.199227+0000","last_peered":"2023-09-23T15:16:04.199227+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:16:04.199227+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"197'6600","ondisk_log_start":"197'6600","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5314,"log_dups_size":0,"ondisk_log_size":5314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46923776,"num_objects":716,"num_object_clones":0,"num_object_copies":2148,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":716,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11914,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-4250}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960194+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:15:53.960194+0000","last_peered":"2023-09-23T15:15:53.960194+0000","last_clean":"2023-09-23T15:15:53.960194+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:15:53.960194+0000","last_undegraded":"2023-09-23T15:15:53.960194+0000","last_fullsized":"2023-09-23T15:15:53.960194+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":690,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964069+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:15:53.964069+0000","last_peered":"2023-09-23T15:15:53.964069+0000","last_clean":"2023-09-23T15:15:53.964069+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:15:53.964069+0000","last_undegraded":"2023-09-23T15:15:53.964069+0000","last_fullsized":"2023-09-23T15:15:53.964069+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960530+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:15:53.960530+0000","last_peered":"2023-09-23T15:15:53.960530+0000","last_clean":"2023-09-23T15:15:53.960530+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:15:53.960530+0000","last_undegraded":"2023-09-23T15:15:53.960530+0000","last_fullsized":"2023-09-23T15:15:53.960530+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943446+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:15:53.943446+0000","last_peered":"2023-09-23T15:15:53.943446+0000","last_clean":"2023-09-23T15:15:53.943446+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:15:53.943446+0000","last_undegraded":"2023-09-23T15:15:53.943446+0000","last_fullsized":"2023-09-23T15:15:53.943446+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":166,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959823+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:15:53.959823+0000","last_peered":"2023-09-23T15:15:53.959823+0000","last_clean":"2023-09-23T15:15:53.959823+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:15:53.959823+0000","last_undegraded":"2023-09-23T15:15:53.959823+0000","last_fullsized":"2023-09-23T15:15:53.959823+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":213,"reported_epoch":212,"state":"active+clean","last_fresh":"2023-09-23T15:15:43.600500+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:15:43.600500+0000","last_peered":"2023-09-23T15:15:43.600500+0000","last_clean":"2023-09-23T15:15:43.600500+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:15:43.600500+0000","last_undegraded":"2023-09-23T15:15:43.600500+0000","last_fullsized":"2023-09-23T15:15:43.600500+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959675+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:15:53.959675+0000","last_peered":"2023-09-23T15:15:53.959675+0000","last_clean":"2023-09-23T15:15:53.959675+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:15:53.959675+0000","last_undegraded":"2023-09-23T15:15:53.959675+0000","last_fullsized":"2023-09-23T15:15:53.959675+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943701+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:15:53.943701+0000","last_peered":"2023-09-23T15:15:53.943701+0000","last_clean":"2023-09-23T15:15:53.943701+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:15:53.943701+0000","last_undegraded":"2023-09-23T15:15:53.943701+0000","last_fullsized":"2023-09-23T15:15:53.943701+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959469+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:15:53.959469+0000","last_peered":"2023-09-23T15:15:53.959469+0000","last_clean":"2023-09-23T15:15:53.959469+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:15:53.959469+0000","last_undegraded":"2023-09-23T15:15:53.959469+0000","last_fullsized":"2023-09-23T15:15:53.959469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943491+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:15:53.943491+0000","last_peered":"2023-09-23T15:15:53.943491+0000","last_clean":"2023-09-23T15:15:53.943491+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:15:53.943491+0000","last_undegraded":"2023-09-23T15:15:53.943491+0000","last_fullsized":"2023-09-23T15:15:53.943491+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":185,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.951061+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:15:53.951061+0000","last_peered":"2023-09-23T15:15:53.951061+0000","last_clean":"2023-09-23T15:15:53.951061+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:15:53.951061+0000","last_undegraded":"2023-09-23T15:15:53.951061+0000","last_fullsized":"2023-09-23T15:15:53.951061+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":200,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.750923+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:15:54.750923+0000","last_peered":"2023-09-23T15:15:54.750923+0000","last_clean":"2023-09-23T15:15:54.750923+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:15:54.750923+0000","last_undegraded":"2023-09-23T15:15:54.750923+0000","last_fullsized":"2023-09-23T15:15:54.750923+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":162,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961242+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:53.961242+0000","last_peered":"2023-09-23T15:15:53.961242+0000","last_clean":"2023-09-23T15:15:53.961242+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:53.961242+0000","last_undegraded":"2023-09-23T15:15:53.961242+0000","last_fullsized":"2023-09-23T15:15:53.961242+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943305+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:15:53.943305+0000","last_peered":"2023-09-23T15:15:53.943305+0000","last_clean":"2023-09-23T15:15:53.943305+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:15:53.943305+0000","last_undegraded":"2023-09-23T15:15:53.943305+0000","last_fullsized":"2023-09-23T15:15:53.943305+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":761200664,"num_objects":11616,"num_object_clones":0,"num_object_copies":34848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21831,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11616,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189162,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7869,"num_bytes_recovered":476053504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2328637440,"data_stored":2328625224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120162,"ondisk_log_size":120162,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201536,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463172,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":567392,"kb_used_data":394712,"kb_used_omap":0,"kb_used_meta":172672,"kb_avail":93177760,"statfs":{"total":95995035648,"available":95414026240,"internally_reserved":0,"allocated":404185088,"data_stored":403472682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":176816128},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.1469999999999998,"15min":2.7120000000000002},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.2519999999999998,"5min":41.491,"15min":108.39400000000001},"last":0.23899999999999999},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.4649999999999999,"15min":2.9289999999999998},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.1029999999999998,"5min":43.151000000000003,"15min":108.452},"last":0.27200000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":3.395,"15min":3.379},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.266,"5min":43.377000000000002,"15min":43.377000000000002},"last":1.8129999999999999},{"interface":"front","average":{"1min":3.2589999999999999,"5min":4.274,"15min":3.8290000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.4060000000000006,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.97999999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5129999999999999,"5min":5.7489999999999997,"15min":6.0949999999999998},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":68.828000000000003,"5min":112.352,"15min":180.374},"last":2.6549999999999998},{"interface":"front","average":{"1min":5.681,"5min":5.8140000000000001,"15min":6.0709999999999997},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":68.730999999999995,"5min":117.667,"15min":181.685},"last":2.0800000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":15.903,"5min":7.0270000000000001,"15min":3.012},"min":{"1min":0.627,"5min":0.249,"15min":0.249},"max":{"1min":102.254,"5min":102.254,"15min":102.254},"last":2.133},{"interface":"front","average":{"1min":17.141999999999999,"5min":7.4619999999999997,"15min":3.0840000000000001},"min":{"1min":0.46100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":98.941999999999993,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4870000000000001,"15min":1.0309999999999999},"min":{"1min":0.248,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.75800000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.312},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.6040000000000001,"15min":1.121},"min":{"1min":0.30599999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7590000000000003,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.33700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:09 2023","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":2.5230000000000001,"15min":1.4059999999999999},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":2.976,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":2.5310000000000001,"15min":1.452},"min":{"1min":0.36399999999999999,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.7859999999999996,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.2689999999999999,"15min":1.032},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.125,"5min":7.5910000000000002,"15min":17},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1659999999999999,"15min":0.995},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":2.524,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.442}]}]},{"osd":1,"up_from":183,"seq":785979015209,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":499576,"kb_used_data":328816,"kb_used_omap":6,"kb_used_meta":170745,"kb_avail":93245576,"statfs":{"total":95995035648,"available":95483469824,"internally_reserved":0,"allocated":336707584,"data_stored":335686994,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":174842955},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":3.4049999999999998,"15min":3.6709999999999998},"min":{"1min":0.40200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.338000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.19900000000000001},{"interface":"front","average":{"1min":5.6859999999999999,"5min":4.6150000000000002,"15min":4.6109999999999998},"min":{"1min":0.29599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.414999999999999,"5min":16.414999999999999,"15min":16.414999999999999},"last":0.246}]},{"osd":2,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":4.1429999999999998,"15min":4.133},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.315,"5min":13.843,"15min":13.843},"last":0.26300000000000001},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.0709999999999997,"15min":4.1719999999999997},"min":{"1min":0.434,"5min":0.253,"15min":0.253},"max":{"1min":16.364999999999998,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.30199999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":8.3599999999999994,"15min":9.984},"min":{"1min":0.40899999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.617999999999999,"5min":123.426,"15min":123.426},"last":211.405},{"interface":"front","average":{"1min":5.3920000000000003,"5min":9.3279999999999994,"15min":11.486000000000001},"min":{"1min":0.68100000000000005,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.451000000000001,"5min":123.32899999999999,"15min":123.32899999999999},"last":210.988}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8409999999999993,"5min":9.8409999999999993,"15min":9.8409999999999993},"last":0.46800000000000003},{"interface":"front","average":{"1min":2.823,"5min":3.3250000000000002,"15min":3.48},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.228000000000002,"5min":16.779,"15min":16.779},"last":0.54900000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":3.4340000000000002,"15min":3.6640000000000001},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.898999999999999,"5min":16.166,"15min":16.166},"last":0.48199999999999998},{"interface":"front","average":{"1min":3.5179999999999998,"5min":3.0449999999999999,"15min":2.9460000000000002},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8680000000000003,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.57199999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":3.4209999999999998,"15min":3.6800000000000002},"min":{"1min":0.67500000000000004,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":14.867000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":0.51700000000000002},{"interface":"front","average":{"1min":3.7240000000000002,"5min":3,"15min":2.8319999999999999},"min":{"1min":0.438,"5min":0.377,"15min":0.377},"max":{"1min":12.448,"5min":12.448,"15min":12.448},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":3.1379999999999999,"15min":3.1499999999999999},"min":{"1min":0.47199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.60299999999999998},{"interface":"front","average":{"1min":3.5169999999999999,"5min":3.323,"15min":3.343},"min":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.280999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.50800000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893259,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":499356,"kb_used_data":191828,"kb_used_omap":8,"kb_used_meta":307511,"kb_avail":93245796,"statfs":{"total":95995035648,"available":95483695104,"internally_reserved":0,"allocated":196431872,"data_stored":195500740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":314891860},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":3.298,"15min":3.96},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4109999999999996,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.9770000000000001,"5min":3.7280000000000002,"15min":4.3819999999999997},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5590000000000002,"5min":31.378,"15min":31.378},"last":1.6779999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":15.058999999999999,"5min":7.9790000000000001,"15min":3.3519999999999999},"min":{"1min":0.311,"5min":0.17000000000000001,"15min":0.17000000000000001},"max":{"1min":93.873999999999995,"5min":122.20099999999999,"15min":122.20099999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":13.999000000000001,"5min":6.9569999999999999,"15min":2.891},"min":{"1min":0.25600000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":94.652000000000001,"5min":122.071,"15min":122.071},"last":3.653}]},{"osd":3,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":8.2420000000000009,"5min":6.8710000000000004,"15min":2.8610000000000002},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":72.137,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.77400000000000002},{"interface":"front","average":{"1min":13.818,"5min":6.9850000000000003,"15min":3.2029999999999998},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":95.795000000000002,"5min":95.795000000000002,"15min":95.795000000000002},"last":0.71699999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.768,"5min":2.9319999999999999,"15min":1.595},"min":{"1min":0.46200000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.0499999999999998,"5min":31.341999999999999,"15min":31.341999999999999},"last":1.758},{"interface":"front","average":{"1min":1.5369999999999999,"5min":3.0150000000000001,"15min":1.5980000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":6.9809999999999999,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.79300000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.9420000000000002,"15min":1.6479999999999999},"min":{"1min":0.34799999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.2370000000000001,"5min":109.369,"15min":109.369},"last":0.69099999999999995},{"interface":"front","average":{"1min":1.7809999999999999,"5min":3.532,"15min":1.8049999999999999},"min":{"1min":0.41899999999999998,"5min":0.35399999999999998,"15min":0.28599999999999998},"max":{"1min":7.1260000000000003,"5min":109.387,"15min":109.387},"last":1.7310000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.516,"5min":2.9590000000000001,"15min":1.6319999999999999},"min":{"1min":0.35899999999999999,"5min":0.309,"15min":0.309},"max":{"1min":7.032,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.81000000000000005},{"interface":"front","average":{"1min":1.171,"5min":2.8359999999999999,"15min":1.5580000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":5.2910000000000004,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.4219999999999999,"5min":3.0870000000000002,"15min":1.6759999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.2640000000000002,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.73399999999999999},{"interface":"front","average":{"1min":1.8380000000000001,"5min":2.4199999999999999,"15min":1.4890000000000001},"min":{"1min":0.497,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":7.093,"5min":31.225999999999999,"15min":31.225999999999999},"last":1.7869999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566206,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":787080,"kb_used_data":450240,"kb_used_omap":11,"kb_used_meta":336820,"kb_avail":92958072,"statfs":{"total":95995035648,"available":95189065728,"internally_reserved":0,"allocated":461045760,"data_stored":460129899,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":344904464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":10.598000000000001,"5min":4.6280000000000001,"15min":1.9299999999999999},"min":{"1min":0.215,"5min":0.184,"15min":0.184},"max":{"1min":87.132999999999996,"5min":87.132999999999996,"15min":87.132999999999996},"last":16.085000000000001},{"interface":"front","average":{"1min":7.6420000000000003,"5min":3.6579999999999999,"15min":1.595},"min":{"1min":0.35499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":84.820999999999998,"5min":84.820999999999998,"15min":84.820999999999998},"last":15.833}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":5.492,"15min":5.1900000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.058999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":31.513999999999999},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.782,"15min":2.96},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.962999999999999,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 15:15:29 2023","interfaces":[{"interface":"back","average":{"1min":14.097,"5min":6.1619999999999999,"15min":2.661},"min":{"1min":0.70399999999999996,"5min":0.219,"15min":0.219},"max":{"1min":86.332999999999998,"5min":87.108000000000004,"15min":87.108000000000004},"last":10.922000000000001},{"interface":"front","average":{"1min":12.151999999999999,"5min":5.577,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":31.440999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.54,"5min":3.6659999999999999,"15min":1.7649999999999999},"min":{"1min":0.42299999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":84.608000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.55300000000000005},{"interface":"front","average":{"1min":8.8379999999999992,"5min":4.8460000000000001,"15min":2.254},"min":{"1min":0.41099999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":87.019999999999996,"5min":87.019999999999996,"15min":87.019999999999996},"last":2.302}]},{"osd":5,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.6260000000000003,"5min":3.6459999999999999,"15min":1.8779999999999999},"min":{"1min":0.36299999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":84.498000000000005,"5min":84.498000000000005,"15min":84.498000000000005},"last":16.193999999999999},{"interface":"front","average":{"1min":6.4429999999999996,"5min":3.5600000000000001,"15min":1.825},"min":{"1min":0.44800000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":84.664000000000001,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.505}]},{"osd":6,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.9589999999999996,"5min":3.9089999999999998,"15min":1.9490000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":84.715999999999994,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.47399999999999998},{"interface":"front","average":{"1min":9.0410000000000004,"5min":3.847,"15min":1.9430000000000001},"min":{"1min":0.44,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":87.055000000000007,"5min":87.055000000000007,"15min":87.055000000000007},"last":2.4079999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:03 2023","interfaces":[{"interface":"back","average":{"1min":6.8410000000000002,"5min":3.2010000000000001,"15min":1.7350000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":84.596000000000004,"5min":84.596000000000004,"15min":84.596000000000004},"last":16.239000000000001},{"interface":"front","average":{"1min":6.7770000000000001,"5min":3.274,"15min":1.7210000000000001},"min":{"1min":0.46999999999999997,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":84.783000000000001,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.52600000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402680,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833436,"kb_used_data":527956,"kb_used_omap":3,"kb_used_meta":305340,"kb_avail":92911716,"statfs":{"total":95995035648,"available":95141597184,"internally_reserved":0,"allocated":540626944,"data_stored":539746022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":312668329},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":3.3100000000000001,"15min":2.0110000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":13.064,"5min":61.744,"15min":79.611000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.2130000000000001,"15min":2.0249999999999999},"min":{"1min":0.40200000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":12.348000000000001,"5min":61.686,"15min":79.656000000000006},"last":4.5780000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:15:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":4.702,"15min":4.6239999999999997},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.566000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":3.5339999999999998,"5min":4.5609999999999999,"15min":4.7510000000000003},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.539000000000001,"5min":21.539000000000001,"15min":21.539000000000001},"last":0.35099999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8600000000000003,"5min":4.6020000000000003,"15min":3.6909999999999998},"min":{"1min":0.432,"5min":0.22,"15min":0.22},"max":{"1min":61.732999999999997,"5min":63.009999999999998,"15min":109.077},"last":0.44800000000000001},{"interface":"front","average":{"1min":8.202,"5min":5.101,"15min":3.9460000000000002},"min":{"1min":0.47999999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":60.305999999999997,"5min":69.715999999999994,"15min":119.72799999999999},"last":4.5490000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.6139999999999999,"15min":1.4319999999999999},"min":{"1min":0.36199999999999999,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":54.051000000000002,"5min":54.051000000000002,"15min":54.051000000000002},"last":4.5220000000000002},{"interface":"front","average":{"1min":5.2939999999999996,"5min":2.7029999999999998,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":52.228999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":1.0509999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":2.399,"15min":1.5740000000000001},"min":{"1min":0.40899999999999997,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":8.0459999999999994,"5min":8.9450000000000003,"15min":56.960999999999999},"last":4.4960000000000004},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.552,"15min":1.806},"min":{"1min":0.49299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.8730000000000002,"5min":10.738,"15min":56.981999999999999},"last":1.1279999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":3.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.42899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":78.102000000000004,"5min":78.102000000000004,"15min":78.102000000000004},"last":4.452},{"interface":"front","average":{"1min":6.5919999999999996,"5min":3.327,"15min":1.962},"min":{"1min":0.69299999999999995,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":78.028000000000006,"5min":78.028000000000006,"15min":78.028000000000006},"last":1.079}]},{"osd":7,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":2.5110000000000001,"15min":1.71},"min":{"1min":0.52200000000000002,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":5.923,"5min":10.603,"15min":14.885999999999999},"last":1.101},{"interface":"front","average":{"1min":3.016,"5min":2.2160000000000002,"15min":1.363},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.109,"5min":14.351000000000001,"15min":14.351000000000001},"last":0.42199999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986445,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":432028,"kb_used_data":242900,"kb_used_omap":0,"kb_used_meta":189120,"kb_avail":93313124,"statfs":{"total":95995035648,"available":95552638976,"internally_reserved":0,"allocated":248729600,"data_stored":248020769,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":193658880},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":2.5830000000000002,"15min":2.6200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.8250000000000002,"5min":65.988,"15min":122.69199999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.552,"5min":2.5219999999999998,"15min":2.698},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":5.7149999999999999,"5min":65.396000000000001,"15min":122.745},"last":3.1419999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":3.6920000000000002,"15min":4.2629999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7539999999999996,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":2.8860000000000001,"5min":4.3369999999999997,"15min":4.8300000000000001},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.114000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":5.4790000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":4.1260000000000003,"15min":3.1080000000000001},"min":{"1min":0.37,"5min":0.255,"15min":0.255},"max":{"1min":12.595000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":2.4420000000000002},{"interface":"front","average":{"1min":3.2869999999999999,"5min":4.383,"15min":3.3530000000000002},"min":{"1min":0.34300000000000003,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":7.173,"5min":147.88900000000001,"15min":147.88900000000001},"last":3.1800000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":9.3309999999999995,"15min":3.7869999999999999},"min":{"1min":0.36799999999999999,"5min":0.252,"15min":0.252},"max":{"1min":96.447000000000003,"5min":163.97499999999999,"15min":163.97499999999999},"last":1.3959999999999999},{"interface":"front","average":{"1min":7.0019999999999998,"5min":9.1080000000000005,"15min":3.5750000000000002},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":95.944999999999993,"5min":164.447,"15min":164.447},"last":5.9809999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:15:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.966,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":87.912999999999997,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.32800000000000001},{"interface":"front","average":{"1min":5.4960000000000004,"5min":1.9450000000000001,"15min":1.395},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":88.602999999999994,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:15:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2169999999999996,"5min":4.2329999999999997,"15min":2.0449999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":86.091999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":5.4509999999999996,"5min":3.984,"15min":1.9319999999999999},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":86.048000000000002,"5min":210.78899999999999,"15min":210.78899999999999},"last":2.9089999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.91800000000000004,"15min":1.129},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":0.75900000000000001,"5min":7.7389999999999999,"15min":18.23},"last":0.435},{"interface":"front","average":{"1min":1.079,"5min":1.3400000000000001,"15min":1.179},"min":{"1min":0.375,"5min":0.314,"15min":0.314},"max":{"1min":6.1399999999999997,"5min":14.044,"15min":14.653},"last":0.497}]}]},{"osd":5,"up_from":39,"seq":167503724808,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435680,"kb_used_data":232792,"kb_used_omap":0,"kb_used_meta":202880,"kb_avail":93309472,"statfs":{"total":95995035648,"available":95548899328,"internally_reserved":0,"allocated":238379008,"data_stored":237670141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":207749120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.9430000000000001,"15min":1.4079999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":7.8319999999999999,"5min":16.981999999999999,"15min":24.962},"last":0.36599999999999999},{"interface":"front","average":{"1min":3.972,"5min":2.1589999999999998,"15min":1.4550000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":12.531000000000001,"5min":18.814,"15min":25.317},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.3730000000000002,"15min":1.9059999999999999},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.6110000000000007,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.248},{"interface":"front","average":{"1min":4.1109999999999998,"5min":2.7730000000000001,"15min":2.4369999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.423,"5min":17.963999999999999,"15min":17.963999999999999},"last":1.3009999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.266,"15min":2.4609999999999999},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3889999999999993,"5min":52.414000000000001,"15min":97.087999999999994},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.5070000000000001,"5min":3.3570000000000002,"15min":2.4209999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":52.375999999999998,"15min":97.305999999999997},"last":0.27900000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":10.388,"5min":5.0510000000000002,"15min":2.488},"min":{"1min":0.54900000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.570999999999998,"5min":80.570999999999998,"15min":80.570999999999998},"last":0.96599999999999997},{"interface":"front","average":{"1min":9.8190000000000008,"5min":5.1440000000000001,"15min":2.3460000000000001},"min":{"1min":0.44400000000000001,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":80.551000000000002,"5min":80.551000000000002,"15min":80.551000000000002},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":8.7479999999999993,"5min":3.2589999999999999,"15min":1.718},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":138.68600000000001,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.30199999999999999},{"interface":"front","average":{"1min":8.7729999999999997,"5min":3.415,"15min":1.788},"min":{"1min":0.34300000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":138.49600000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.35699999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:15:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.494,"15min":1.1160000000000001},"min":{"1min":0.35099999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.349,"5min":11.076000000000001,"15min":15.196},"last":0.436},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.3859999999999999,"15min":1.012},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30299999999999999},"max":{"1min":1.367,"5min":14.965999999999999,"15min":14.965999999999999},"last":0.38100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.3380000000000001,"15min":1.288},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.28599999999999998},"max":{"1min":8.2200000000000006,"5min":9.6560000000000006,"15min":15.301},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.141,"5min":1.3700000000000001,"15min":1.252},"min":{"1min":0.39900000000000002,"5min":0.311,"15min":0.311},"max":{"1min":5.6559999999999997,"5min":15.762,"15min":15.762},"last":0.40400000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":187174912,"data_stored":187170840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":327815168,"data_stored":327811096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":447283200,"data_stored":447283200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":513212416,"data_stored":513212416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":236584960,"data_stored":236584960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":232001536,"data_stored":231997464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":384565248,"data_stored":384565248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:16:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:09 smithi027 ceph-mon[195233]: pgmap v596: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:09.658 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:09.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:09.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:09 smithi017 ceph-mon[215646]: pgmap v596: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:09.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:09 smithi017 ceph-mon[222878]: pgmap v596: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 730 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 21834/34929 objects degraded (62.510%); 0 B/s, 29 objects/s recovering 2023-09-23T15:16:09.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:09.779 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:09.808 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:09.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:09.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:09.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:09.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:10.007 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:10.069 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:10.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:10.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:10.174 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:10.207 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:10.270 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:10.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:10.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:10.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:10.445 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:10.547 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:10.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:10.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:10.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:10.678 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:11.015 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:11.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:11.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:11.081 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:11.128 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:11.156 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:11.209 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:11.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:11.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:11.346 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:11.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:11.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:11.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:11.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:11.544 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:11.604 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:11.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:11.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:11.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:11.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:11 smithi017 ceph-mon[215646]: pgmap v597: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:11.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:11 smithi017 ceph-mon[222878]: pgmap v597: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:11.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:16:11.794 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:11.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:11 smithi027 ceph-mon[195233]: pgmap v597: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.4 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:12.142 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:12.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:12.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:12.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:12.280 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:12.355 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:16:12.456 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:16:12.456 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:16:12.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.456 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:193466 6.8 6:1feb014d:::benchmark_data_smithi017_350392_object93075:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:12.157417+0000", 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.128095496, 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12813017199999999, 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:16:12.457 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193466 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157417+0000", 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157417+0000", 2023-09-23T15:16:12.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157416+0000", 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157423+0000", 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6590000000000007e-06 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157437+0000", 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3811e-05 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.157445+0000", 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4500000000000004e-06 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.166177+0000", 2023-09-23T15:16:12.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087320390000000005 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.166336+0000", 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158932 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.166483+0000", 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000146599 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.184626+0000", 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018142980999999999 2023-09-23T15:16:12.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.187047+0000", 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024205120000000001 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:16:12.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:193461 6.8 6:1afc7d91:::benchmark_data_smithi017_350392_object93070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:12.135564+0000", 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.149948373, 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15004405300000001, 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:16:12.466 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 193461 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.467 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135564+0000", 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135564+0000", 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135564+0000", 2023-09-23T15:16:12.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135567+0000", 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4829999999999999e-06 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135574+0000", 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6309999999999997e-06 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:16:12.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.135578+0000", 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6349999999999997e-06 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.160378+0000", 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024799249999999998 2023-09-23T15:16:12.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.160526+0000", 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014832499999999999 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.160662+0000", 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135709 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:12.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.183564+0000", 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022902367999999999 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:12.184621+0000", 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001056562 2023-09-23T15:16:12.475 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2023-09-23T15:16:12.476 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:16:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21674/26640 objects degraded (81.359%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21674/26640 objects degraded (81.359%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:12.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21674/26640 objects degraded (81.359%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:12.851 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:12.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:12.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:12.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:13.058 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:13.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:16:13.547 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:13.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:13.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:13.686 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:13.742 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:13.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:13.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:13.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:13.986 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:14.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:14.047 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:14.156 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:14.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:14.264 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:14.293 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:14.363 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:14.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:14.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[215646]: pgmap v598: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:14.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:14.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:14.411 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[222878]: pgmap v598: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:14.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:14.412 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:14.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:14.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:14 smithi027 ceph-mon[195233]: pgmap v598: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 558 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 551 op/s; 21674/26640 objects degraded (81.359%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:14.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:14.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:14.501 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:14.558 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:14.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:14.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:14.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:14.759 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:15.178 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:16:15.396 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:15.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1130768098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:15.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:15.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:15.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:15.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1130768098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:15.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:15.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:15.570 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1130768098' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:15.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: pgmap v599: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 482 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 21674/23010 objects degraded (94.194%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[215646]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: pgmap v599: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 482 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 21674/23010 objects degraded (94.194%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:16.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:16.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:16.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T15:16:16.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:16 smithi017 ceph-mon[222878]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: pgmap v599: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 482 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 604 op/s; 21674/23010 objects degraded (94.194%); 0 B/s, 30 objects/s recovering 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: osdmap e215: 8 total, 8 up, 7 in 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:16 smithi027 ceph-mon[195233]: failed to encode map e215 with expected crc 2023-09-23T15:16:16.605 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:16.759 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:16.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:16.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:16.875 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:16.909 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:16.963 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:16.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:16.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:17.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:17.179 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:17.193 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21674/23010 objects degraded (94.194%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:17.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21674/23010 objects degraded (94.194%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:17.515 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21674/23010 objects degraded (94.194%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:17.753 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:17.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:17.758 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:17.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:17.892 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:17.952 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:17.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:17.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:18.059 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:18.084 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:18.093 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:18.120 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:18.166 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:18.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:18.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:18.214 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:18.265 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:18.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:18.316 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:18.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:18.361 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:18] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[222878]: pgmap v601: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/725611795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[215646]: pgmap v601: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:18.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:16:18.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/725611795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:18.374 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:18.422 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:18.467 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:18.478 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.495 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.497 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:16:18.542 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:18.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:18 smithi027 ceph-mon[195233]: pgmap v601: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.9 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:16:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/725611795' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:18.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:18.679 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:19.189 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:19.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:19.302 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:19.332 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:16:19.348 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:19.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:19 smithi017 ceph-mon[222878]: from='client.45389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:19.417 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:19 smithi017 ceph-mon[222878]: pgmap v602: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:19.418 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:19 smithi017 ceph-mon[215646]: from='client.45389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:19.419 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:19 smithi017 ceph-mon[215646]: pgmap v602: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:19.517 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:19.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:19.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:19 smithi027 ceph-mon[195233]: from='client.45389 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:19.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:19 smithi027 ceph-mon[195233]: pgmap v602: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 389 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 546 op/s; 21523/18540 objects degraded (116.090%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:19.622 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:19.733 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:19.769 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-09-23T15:16:16.435015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":407935196,"num_objects":6180,"num_object_clones":0,"num_object_copies":18540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21523,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6180,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":195653,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8239,"num_bytes_recovered":481400999,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124538,"ondisk_log_size":124538,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3042268,"kb_used_data":1326052,"kb_used_omap":30,"kb_used_meta":1716129,"kb_avail":653173796,"statfs":{"total":671965249536,"available":668849967104,"internally_reserved":0,"allocated":1357877248,"data_stored":1351967269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1757316696},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,4,2,2,6,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-358023168,"num_objects":-5463,"num_object_clones":0,"num_object_copies":-16389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-311,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5463,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":311,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003314"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":208,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943643+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:15:53.943643+0000","last_peered":"2023-09-23T15:15:53.943643+0000","last_clean":"2023-09-23T15:15:53.943643+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:15:53.943643+0000","last_undegraded":"2023-09-23T15:15:53.943643+0000","last_fullsized":"2023-09-23T15:15:53.943643+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":143,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.149354+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:16:04.149354+0000","last_peered":"2023-09-23T15:16:04.149354+0000","last_clean":"2023-09-23T15:16:04.149354+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:16:04.149354+0000","last_undegraded":"2023-09-23T15:16:04.149354+0000","last_fullsized":"2023-09-23T15:16:04.149354+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964568+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:15:53.964568+0000","last_peered":"2023-09-23T15:15:53.964568+0000","last_clean":"2023-09-23T15:15:53.964568+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:15:53.964568+0000","last_undegraded":"2023-09-23T15:15:53.964568+0000","last_fullsized":"2023-09-23T15:15:53.964568+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":179,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960612+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:15:53.960612+0000","last_peered":"2023-09-23T15:15:53.960612+0000","last_clean":"2023-09-23T15:15:53.960612+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:15:53.960612+0000","last_undegraded":"2023-09-23T15:15:53.960612+0000","last_fullsized":"2023-09-23T15:15:53.960612+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":212,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.750886+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:15:54.750886+0000","last_peered":"2023-09-23T15:15:54.750886+0000","last_clean":"2023-09-23T15:15:54.750886+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:15:54.750886+0000","last_undegraded":"2023-09-23T15:15:54.750886+0000","last_fullsized":"2023-09-23T15:15:54.750886+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":188,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964217+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:15:53.964217+0000","last_peered":"2023-09-23T15:15:53.964217+0000","last_clean":"2023-09-23T15:15:53.964217+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:15:53.964217+0000","last_undegraded":"2023-09-23T15:15:53.964217+0000","last_fullsized":"2023-09-23T15:15:53.964217+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961153+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:15:53.961153+0000","last_peered":"2023-09-23T15:15:53.961153+0000","last_clean":"2023-09-23T15:15:53.961153+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:15:53.961153+0000","last_undegraded":"2023-09-23T15:15:53.961153+0000","last_fullsized":"2023-09-23T15:15:53.961153+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":194,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950489+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:15:53.950489+0000","last_peered":"2023-09-23T15:15:53.950489+0000","last_clean":"2023-09-23T15:15:53.950489+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:15:53.950489+0000","last_undegraded":"2023-09-23T15:15:53.950489+0000","last_fullsized":"2023-09-23T15:15:53.950489+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.150682+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:16:04.150682+0000","last_peered":"2023-09-23T15:16:04.150682+0000","last_clean":"2023-09-23T15:16:04.150682+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:16:04.150682+0000","last_undegraded":"2023-09-23T15:16:04.150682+0000","last_fullsized":"2023-09-23T15:16:04.150682+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":204,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959464+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:15:53.959464+0000","last_peered":"2023-09-23T15:15:53.959464+0000","last_clean":"2023-09-23T15:15:53.959464+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:15:53.959464+0000","last_undegraded":"2023-09-23T15:15:53.959464+0000","last_fullsized":"2023-09-23T15:15:53.959464+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":137,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950222+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:15:53.950222+0000","last_peered":"2023-09-23T15:15:53.950222+0000","last_clean":"2023-09-23T15:15:53.950222+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:15:53.950222+0000","last_undegraded":"2023-09-23T15:15:53.950222+0000","last_fullsized":"2023-09-23T15:15:53.950222+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":76,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950612+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:15:53.950612+0000","last_peered":"2023-09-23T15:15:53.950612+0000","last_clean":"2023-09-23T15:15:53.950612+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:15:53.950612+0000","last_undegraded":"2023-09-23T15:15:53.950612+0000","last_fullsized":"2023-09-23T15:15:53.950612+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":86,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964439+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:15:53.964439+0000","last_peered":"2023-09-23T15:15:53.964439+0000","last_clean":"2023-09-23T15:15:53.964439+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:15:53.964439+0000","last_undegraded":"2023-09-23T15:15:53.964439+0000","last_fullsized":"2023-09-23T15:15:53.964439+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":198,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943599+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:15:53.943599+0000","last_peered":"2023-09-23T15:15:53.943599+0000","last_clean":"2023-09-23T15:15:53.943599+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:15:53.943599+0000","last_undegraded":"2023-09-23T15:15:53.943599+0000","last_fullsized":"2023-09-23T15:15:53.943599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943616+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:15:53.943616+0000","last_peered":"2023-09-23T15:15:53.943616+0000","last_clean":"2023-09-23T15:15:53.943616+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:15:53.943616+0000","last_undegraded":"2023-09-23T15:15:53.943616+0000","last_fullsized":"2023-09-23T15:15:53.943616+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":503,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.151920+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:16:04.151920+0000","last_peered":"2023-09-23T15:16:04.151920+0000","last_clean":"2023-09-23T15:16:04.151920+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:16:04.151920+0000","last_undegraded":"2023-09-23T15:16:04.151920+0000","last_fullsized":"2023-09-23T15:16:04.151920+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":76,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943677+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:15:53.943677+0000","last_peered":"2023-09-23T15:15:53.943677+0000","last_clean":"2023-09-23T15:15:53.943677+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:15:53.943677+0000","last_undegraded":"2023-09-23T15:15:53.943677+0000","last_fullsized":"2023-09-23T15:15:53.943677+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.955897+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:15:53.955897+0000","last_peered":"2023-09-23T15:15:53.955897+0000","last_clean":"2023-09-23T15:15:53.955897+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:15:53.955897+0000","last_undegraded":"2023-09-23T15:15:53.955897+0000","last_fullsized":"2023-09-23T15:15:53.955897+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":503,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.152194+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:16:04.152194+0000","last_peered":"2023-09-23T15:16:04.152194+0000","last_clean":"2023-09-23T15:16:04.152194+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:16:04.152194+0000","last_undegraded":"2023-09-23T15:16:04.152194+0000","last_fullsized":"2023-09-23T15:16:04.152194+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":178,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960663+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:15:53.960663+0000","last_peered":"2023-09-23T15:15:53.960663+0000","last_clean":"2023-09-23T15:15:53.960663+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:15:53.960663+0000","last_undegraded":"2023-09-23T15:15:53.960663+0000","last_fullsized":"2023-09-23T15:15:53.960663+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959599+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:15:53.959599+0000","last_peered":"2023-09-23T15:15:53.959599+0000","last_clean":"2023-09-23T15:15:53.959599+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:15:53.959599+0000","last_undegraded":"2023-09-23T15:15:53.959599+0000","last_fullsized":"2023-09-23T15:15:53.959599+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.149847+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:16:04.149847+0000","last_peered":"2023-09-23T15:16:04.149847+0000","last_clean":"2023-09-23T15:16:04.149847+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:16:04.149847+0000","last_undegraded":"2023-09-23T15:16:04.149847+0000","last_fullsized":"2023-09-23T15:16:04.149847+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943558+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:15:53.943558+0000","last_peered":"2023-09-23T15:15:53.943558+0000","last_clean":"2023-09-23T15:15:53.943558+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:15:53.943558+0000","last_undegraded":"2023-09-23T15:15:53.943558+0000","last_fullsized":"2023-09-23T15:15:53.943558+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":167,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960691+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:15:53.960691+0000","last_peered":"2023-09-23T15:15:53.960691+0000","last_clean":"2023-09-23T15:15:53.960691+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:15:53.960691+0000","last_undegraded":"2023-09-23T15:15:53.960691+0000","last_fullsized":"2023-09-23T15:15:53.960691+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":503,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.154030+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:16:04.154030+0000","last_peered":"2023-09-23T15:16:04.154030+0000","last_clean":"2023-09-23T15:16:04.154030+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:16:04.154030+0000","last_undegraded":"2023-09-23T15:16:04.154030+0000","last_fullsized":"2023-09-23T15:16:04.154030+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":180,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963149+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:15:53.963149+0000","last_peered":"2023-09-23T15:15:53.963149+0000","last_clean":"2023-09-23T15:15:53.963149+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:15:53.963149+0000","last_undegraded":"2023-09-23T15:15:53.963149+0000","last_fullsized":"2023-09-23T15:15:53.963149+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":205,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.751052+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:15:54.751052+0000","last_peered":"2023-09-23T15:15:54.751052+0000","last_clean":"2023-09-23T15:15:54.751052+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:15:54.751052+0000","last_undegraded":"2023-09-23T15:15:54.751052+0000","last_fullsized":"2023-09-23T15:15:54.751052+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"214'12044","reported_seq":12095,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.719900+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:16:14.719900+0000","last_peered":"2023-09-23T15:16:14.719900+0000","last_clean":"2023-09-23T15:16:14.719900+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:16:14.719900+0000","last_undegraded":"2023-09-23T15:16:14.719900+0000","last_fullsized":"2023-09-23T15:16:14.719900+0000","mapping_epoch":171,"log_start":"176'2000","ondisk_log_start":"176'2000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22937600,"num_objects":350,"num_object_clones":0,"num_object_copies":1050,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":350,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12044,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.152749+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:16:04.152749+0000","last_peered":"2023-09-23T15:16:04.152749+0000","last_clean":"2023-09-23T15:16:04.152749+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:16:04.152749+0000","last_undegraded":"2023-09-23T15:16:04.152749+0000","last_fullsized":"2023-09-23T15:16:04.152749+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":88,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.944029+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:15:53.944029+0000","last_peered":"2023-09-23T15:15:53.944029+0000","last_clean":"2023-09-23T15:15:53.944029+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:15:53.944029+0000","last_undegraded":"2023-09-23T15:15:53.944029+0000","last_fullsized":"2023-09-23T15:15:53.944029+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":165,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959174+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:15:53.959174+0000","last_peered":"2023-09-23T15:15:53.959174+0000","last_clean":"2023-09-23T15:15:53.959174+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:15:53.959174+0000","last_undegraded":"2023-09-23T15:15:53.959174+0000","last_fullsized":"2023-09-23T15:15:53.959174+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"214'12299","reported_seq":18919,"reported_epoch":214,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:14.775709+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:16:14.775709+0000","last_peered":"2023-09-23T15:16:14.775709+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:16:14.775709+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"193'5900","ondisk_log_start":"193'5900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6399,"log_dups_size":0,"ondisk_log_size":6399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1718,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12299,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3280,"num_bytes_recovered":154927104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-1351}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":482,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:04.150470+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:16:04.150470+0000","last_peered":"2023-09-23T15:16:04.150470+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:16:04.150470+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":88,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943997+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:15:53.943997+0000","last_peered":"2023-09-23T15:15:53.943997+0000","last_clean":"2023-09-23T15:15:53.943997+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:15:53.943997+0000","last_undegraded":"2023-09-23T15:15:53.943997+0000","last_fullsized":"2023-09-23T15:15:53.943997+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943456+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:15:53.943456+0000","last_peered":"2023-09-23T15:15:53.943456+0000","last_clean":"2023-09-23T15:15:53.943456+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:15:53.943456+0000","last_undegraded":"2023-09-23T15:15:53.943456+0000","last_fullsized":"2023-09-23T15:15:53.943456+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"214'12037","reported_seq":12091,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:13.675014+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:16:13.675014+0000","last_peered":"2023-09-23T15:16:13.675014+0000","last_clean":"2023-09-23T15:16:13.675014+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:16:13.675014+0000","last_undegraded":"2023-09-23T15:16:13.675014+0000","last_fullsized":"2023-09-23T15:16:13.675014+0000","mapping_epoch":171,"log_start":"177'2900","ondisk_log_start":"177'2900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9137,"log_dups_size":0,"ondisk_log_size":9137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26542080,"num_objects":405,"num_object_clones":0,"num_object_copies":1215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12037,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.151332+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:16:04.151332+0000","last_peered":"2023-09-23T15:16:04.151332+0000","last_clean":"2023-09-23T15:16:04.151332+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:16:04.151332+0000","last_undegraded":"2023-09-23T15:16:04.151332+0000","last_fullsized":"2023-09-23T15:16:04.151332+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":74,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959577+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:15:53.959577+0000","last_peered":"2023-09-23T15:15:53.959577+0000","last_clean":"2023-09-23T15:15:53.959577+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:15:53.959577+0000","last_undegraded":"2023-09-23T15:15:53.959577+0000","last_fullsized":"2023-09-23T15:15:53.959577+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963880+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:15:53.963880+0000","last_peered":"2023-09-23T15:15:53.963880+0000","last_clean":"2023-09-23T15:15:53.963880+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:15:53.963880+0000","last_undegraded":"2023-09-23T15:15:53.963880+0000","last_fullsized":"2023-09-23T15:15:53.963880+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"214'12244","reported_seq":12298,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.466856+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:16:14.466856+0000","last_peered":"2023-09-23T15:16:14.466856+0000","last_clean":"2023-09-23T15:16:14.466856+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:16:14.466856+0000","last_undegraded":"2023-09-23T15:16:14.466856+0000","last_fullsized":"2023-09-23T15:16:14.466856+0000","mapping_epoch":171,"log_start":"197'6600","ondisk_log_start":"197'6600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5644,"log_dups_size":0,"ondisk_log_size":5644,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27525120,"num_objects":420,"num_object_clones":0,"num_object_copies":1260,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12244,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.150884+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:16:04.150884+0000","last_peered":"2023-09-23T15:16:04.150884+0000","last_clean":"2023-09-23T15:16:04.150884+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:16:04.150884+0000","last_undegraded":"2023-09-23T15:16:04.150884+0000","last_fullsized":"2023-09-23T15:16:04.150884+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959513+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:53.959513+0000","last_peered":"2023-09-23T15:15:53.959513+0000","last_clean":"2023-09-23T15:15:53.959513+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:15:53.959513+0000","last_undegraded":"2023-09-23T15:15:53.959513+0000","last_fullsized":"2023-09-23T15:15:53.959513+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":174,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465941+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:15:54.465941+0000","last_peered":"2023-09-23T15:15:54.465941+0000","last_clean":"2023-09-23T15:15:54.465941+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:15:54.465941+0000","last_undegraded":"2023-09-23T15:15:54.465941+0000","last_fullsized":"2023-09-23T15:15:54.465941+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"214'12503","reported_seq":12560,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:14.277610+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:16:14.277610+0000","last_peered":"2023-09-23T15:16:14.277610+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:16:14.277610+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"182'5000","ondisk_log_start":"182'5000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7503,"log_dups_size":0,"ondisk_log_size":7503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26804224,"num_objects":409,"num_object_clones":0,"num_object_copies":1227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12503,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-4601}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465823+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:15:54.465823+0000","last_peered":"2023-09-23T15:15:54.465823+0000","last_clean":"2023-09-23T15:15:54.465823+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:15:54.465823+0000","last_undegraded":"2023-09-23T15:15:54.465823+0000","last_fullsized":"2023-09-23T15:15:54.465823+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.953257+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:15:53.953257+0000","last_peered":"2023-09-23T15:15:53.953257+0000","last_clean":"2023-09-23T15:15:53.953257+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:15:53.953257+0000","last_undegraded":"2023-09-23T15:15:53.953257+0000","last_fullsized":"2023-09-23T15:15:53.953257+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":172,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943637+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:15:53.943637+0000","last_peered":"2023-09-23T15:15:53.943637+0000","last_clean":"2023-09-23T15:15:53.943637+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:15:53.943637+0000","last_undegraded":"2023-09-23T15:15:53.943637+0000","last_fullsized":"2023-09-23T15:15:53.943637+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"214'12150","reported_seq":12204,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:13.671941+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:16:13.671941+0000","last_peered":"2023-09-23T15:16:13.671941+0000","last_clean":"2023-09-23T15:16:13.671941+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:16:13.671941+0000","last_undegraded":"2023-09-23T15:16:13.671941+0000","last_fullsized":"2023-09-23T15:16:13.671941+0000","mapping_epoch":171,"log_start":"177'3000","ondisk_log_start":"177'3000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9150,"log_dups_size":0,"ondisk_log_size":9150,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28049408,"num_objects":428,"num_object_clones":0,"num_object_copies":1284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12150,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":508,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465780+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:15:54.465780+0000","last_peered":"2023-09-23T15:15:54.465780+0000","last_clean":"2023-09-23T15:15:54.465780+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:15:54.465780+0000","last_undegraded":"2023-09-23T15:15:54.465780+0000","last_fullsized":"2023-09-23T15:15:54.465780+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963417+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:15:53.963417+0000","last_peered":"2023-09-23T15:15:53.963417+0000","last_clean":"2023-09-23T15:15:53.963417+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:15:53.963417+0000","last_undegraded":"2023-09-23T15:15:53.963417+0000","last_fullsized":"2023-09-23T15:15:53.963417+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943731+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:15:53.943731+0000","last_peered":"2023-09-23T15:15:53.943731+0000","last_clean":"2023-09-23T15:15:53.943731+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:15:53.943731+0000","last_undegraded":"2023-09-23T15:15:53.943731+0000","last_fullsized":"2023-09-23T15:15:53.943731+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"214'12200","reported_seq":12254,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.790253+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:16:14.790253+0000","last_peered":"2023-09-23T15:16:14.790253+0000","last_clean":"2023-09-23T15:16:14.790253+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:16:14.790253+0000","last_undegraded":"2023-09-23T15:16:14.790253+0000","last_fullsized":"2023-09-23T15:16:14.790253+0000","mapping_epoch":171,"log_start":"193'5800","ondisk_log_start":"193'5800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24772608,"num_objects":378,"num_object_clones":0,"num_object_copies":1134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12200,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":491,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.149936+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:16:04.149936+0000","last_peered":"2023-09-23T15:16:04.149936+0000","last_clean":"2023-09-23T15:16:04.149936+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:16:04.149936+0000","last_undegraded":"2023-09-23T15:16:04.149936+0000","last_fullsized":"2023-09-23T15:16:04.149936+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":197,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943370+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:15:53.943370+0000","last_peered":"2023-09-23T15:15:53.943370+0000","last_clean":"2023-09-23T15:15:53.943370+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:15:53.943370+0000","last_undegraded":"2023-09-23T15:15:53.943370+0000","last_fullsized":"2023-09-23T15:15:53.943370+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.944061+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:15:53.944061+0000","last_peered":"2023-09-23T15:15:53.944061+0000","last_clean":"2023-09-23T15:15:53.944061+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:15:53.944061+0000","last_undegraded":"2023-09-23T15:15:53.944061+0000","last_fullsized":"2023-09-23T15:15:53.944061+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"214'12156","reported_seq":22028,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.719898+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:16:14.719898+0000","last_peered":"2023-09-23T15:16:14.719898+0000","last_clean":"2023-09-23T15:16:14.719898+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:16:14.719898+0000","last_undegraded":"2023-09-23T15:16:14.719898+0000","last_fullsized":"2023-09-23T15:16:14.719898+0000","mapping_epoch":203,"log_start":"176'2100","ondisk_log_start":"176'2100","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10056,"log_dups_size":0,"ondisk_log_size":10056,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24576024,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12156,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.465887+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:15:54.465887+0000","last_peered":"2023-09-23T15:15:54.465887+0000","last_clean":"2023-09-23T15:15:54.465887+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:15:54.465887+0000","last_undegraded":"2023-09-23T15:15:54.465887+0000","last_fullsized":"2023-09-23T15:15:54.465887+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.950295+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:15:53.950295+0000","last_peered":"2023-09-23T15:15:53.950295+0000","last_clean":"2023-09-23T15:15:53.950295+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:15:53.950295+0000","last_undegraded":"2023-09-23T15:15:53.950295+0000","last_fullsized":"2023-09-23T15:15:53.950295+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":91,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963694+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:15:53.963694+0000","last_peered":"2023-09-23T15:15:53.963694+0000","last_clean":"2023-09-23T15:15:53.963694+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:15:53.963694+0000","last_undegraded":"2023-09-23T15:15:53.963694+0000","last_fullsized":"2023-09-23T15:15:53.963694+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"214'12206","reported_seq":12260,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.466625+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:16:14.466625+0000","last_peered":"2023-09-23T15:16:14.466625+0000","last_clean":"2023-09-23T15:16:14.466625+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:16:14.466625+0000","last_undegraded":"2023-09-23T15:16:14.466625+0000","last_fullsized":"2023-09-23T15:16:14.466625+0000","mapping_epoch":171,"log_start":"197'6600","ondisk_log_start":"197'6600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5606,"log_dups_size":0,"ondisk_log_size":5606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23724032,"num_objects":362,"num_object_clones":0,"num_object_copies":1086,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":362,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12206,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.466116+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:15:54.466116+0000","last_peered":"2023-09-23T15:15:54.466116+0000","last_clean":"2023-09-23T15:15:54.466116+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:15:54.466116+0000","last_undegraded":"2023-09-23T15:15:54.466116+0000","last_fullsized":"2023-09-23T15:15:54.466116+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":390,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.965587+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:15:53.965587+0000","last_peered":"2023-09-23T15:15:53.965587+0000","last_clean":"2023-09-23T15:15:53.965587+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:15:53.965587+0000","last_undegraded":"2023-09-23T15:15:53.965587+0000","last_fullsized":"2023-09-23T15:15:53.965587+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959515+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:15:53.959515+0000","last_peered":"2023-09-23T15:15:53.959515+0000","last_clean":"2023-09-23T15:15:53.959515+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:15:53.959515+0000","last_undegraded":"2023-09-23T15:15:53.959515+0000","last_fullsized":"2023-09-23T15:15:53.959515+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.157226+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:16:04.157226+0000","last_peered":"2023-09-23T15:16:04.157226+0000","last_clean":"2023-09-23T15:16:04.157226+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:16:04.157226+0000","last_undegraded":"2023-09-23T15:16:04.157226+0000","last_fullsized":"2023-09-23T15:16:04.157226+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"214'12003","reported_seq":12057,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.752732+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:16:14.752732+0000","last_peered":"2023-09-23T15:16:14.752732+0000","last_clean":"2023-09-23T15:16:14.752732+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:16:14.752732+0000","last_undegraded":"2023-09-23T15:16:14.752732+0000","last_fullsized":"2023-09-23T15:16:14.752732+0000","mapping_epoch":171,"log_start":"192'5600","ondisk_log_start":"192'5600","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6403,"log_dups_size":0,"ondisk_log_size":6403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23003136,"num_objects":351,"num_object_clones":0,"num_object_copies":1053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12003,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964131+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:15:53.964131+0000","last_peered":"2023-09-23T15:15:53.964131+0000","last_clean":"2023-09-23T15:15:53.964131+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:15:53.964131+0000","last_undegraded":"2023-09-23T15:15:53.964131+0000","last_fullsized":"2023-09-23T15:15:53.964131+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":732,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959346+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:15:53.959346+0000","last_peered":"2023-09-23T15:15:53.959346+0000","last_clean":"2023-09-23T15:15:53.959346+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:15:53.959346+0000","last_undegraded":"2023-09-23T15:15:53.959346+0000","last_fullsized":"2023-09-23T15:15:53.959346+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":173,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943609+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:15:53.943609+0000","last_peered":"2023-09-23T15:15:53.943609+0000","last_clean":"2023-09-23T15:15:53.943609+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:15:53.943609+0000","last_undegraded":"2023-09-23T15:15:53.943609+0000","last_fullsized":"2023-09-23T15:15:53.943609+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":106,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.965336+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:15:53.965336+0000","last_peered":"2023-09-23T15:15:53.965336+0000","last_clean":"2023-09-23T15:15:53.965336+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:15:53.965336+0000","last_undegraded":"2023-09-23T15:15:53.965336+0000","last_fullsized":"2023-09-23T15:15:53.965336+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"214'12237","reported_seq":12288,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.220700+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:16:14.220700+0000","last_peered":"2023-09-23T15:16:14.220700+0000","last_clean":"2023-09-23T15:16:14.220700+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:16:14.220700+0000","last_undegraded":"2023-09-23T15:16:14.220700+0000","last_fullsized":"2023-09-23T15:16:14.220700+0000","mapping_epoch":171,"log_start":"198'6900","ondisk_log_start":"198'6900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5337,"log_dups_size":0,"ondisk_log_size":5337,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25493504,"num_objects":389,"num_object_clones":0,"num_object_copies":1167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12237,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960751+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:15:53.960751+0000","last_peered":"2023-09-23T15:15:53.960751+0000","last_clean":"2023-09-23T15:15:53.960751+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:15:53.960751+0000","last_undegraded":"2023-09-23T15:15:53.960751+0000","last_fullsized":"2023-09-23T15:15:53.960751+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":827,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.162671+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:16:04.162671+0000","last_peered":"2023-09-23T15:16:04.162671+0000","last_clean":"2023-09-23T15:16:04.162671+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:16:04.162671+0000","last_undegraded":"2023-09-23T15:16:04.162671+0000","last_fullsized":"2023-09-23T15:16:04.162671+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":86,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964028+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:15:53.964028+0000","last_peered":"2023-09-23T15:15:53.964028+0000","last_clean":"2023-09-23T15:15:53.964028+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:15:53.964028+0000","last_undegraded":"2023-09-23T15:15:53.964028+0000","last_fullsized":"2023-09-23T15:15:53.964028+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":110,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.965085+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:15:53.965085+0000","last_peered":"2023-09-23T15:15:53.965085+0000","last_clean":"2023-09-23T15:15:53.965085+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:15:53.965085+0000","last_undegraded":"2023-09-23T15:15:53.965085+0000","last_fullsized":"2023-09-23T15:15:53.965085+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"214'11982","reported_seq":12033,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.676971+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:16:14.676971+0000","last_peered":"2023-09-23T15:16:14.676971+0000","last_clean":"2023-09-23T15:16:14.676971+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:16:14.676971+0000","last_undegraded":"2023-09-23T15:16:14.676971+0000","last_fullsized":"2023-09-23T15:16:14.676971+0000","mapping_epoch":171,"log_start":"175'1900","ondisk_log_start":"175'1900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10082,"log_dups_size":0,"ondisk_log_size":10082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26083328,"num_objects":398,"num_object_clones":0,"num_object_copies":1194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11982,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":501,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959356+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:15:53.959356+0000","last_peered":"2023-09-23T15:15:53.959356+0000","last_clean":"2023-09-23T15:15:53.959356+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:15:53.959356+0000","last_undegraded":"2023-09-23T15:15:53.959356+0000","last_fullsized":"2023-09-23T15:15:53.959356+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":503,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961421+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:15:53.961421+0000","last_peered":"2023-09-23T15:15:53.961421+0000","last_clean":"2023-09-23T15:15:53.961421+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:15:53.961421+0000","last_undegraded":"2023-09-23T15:15:53.961421+0000","last_fullsized":"2023-09-23T15:15:53.961421+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":128,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.951319+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:15:53.951319+0000","last_peered":"2023-09-23T15:15:53.951319+0000","last_clean":"2023-09-23T15:15:53.951319+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:15:53.951319+0000","last_undegraded":"2023-09-23T15:15:53.951319+0000","last_fullsized":"2023-09-23T15:15:53.951319+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964346+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:15:53.964346+0000","last_peered":"2023-09-23T15:15:53.964346+0000","last_clean":"2023-09-23T15:15:53.964346+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:15:53.964346+0000","last_undegraded":"2023-09-23T15:15:53.964346+0000","last_fullsized":"2023-09-23T15:15:53.964346+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"214'12283","reported_seq":12337,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:14.745164+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:16:14.745164+0000","last_peered":"2023-09-23T15:16:14.745164+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:16:14.745164+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"193'5900","ondisk_log_start":"193'5900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6383,"log_dups_size":0,"ondisk_log_size":6383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23658496,"num_objects":361,"num_object_clones":0,"num_object_copies":1083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12283,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-4591}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959592+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:15:53.959592+0000","last_peered":"2023-09-23T15:15:53.959592+0000","last_clean":"2023-09-23T15:15:53.959592+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:15:53.959592+0000","last_undegraded":"2023-09-23T15:15:53.959592+0000","last_fullsized":"2023-09-23T15:15:53.959592+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":297,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963907+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:15:53.963907+0000","last_peered":"2023-09-23T15:15:53.963907+0000","last_clean":"2023-09-23T15:15:53.963907+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:15:53.963907+0000","last_undegraded":"2023-09-23T15:15:53.963907+0000","last_fullsized":"2023-09-23T15:15:53.963907+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":159,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.963945+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:15:53.963945+0000","last_peered":"2023-09-23T15:15:53.963945+0000","last_clean":"2023-09-23T15:15:53.963945+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:15:53.963945+0000","last_undegraded":"2023-09-23T15:15:53.963945+0000","last_fullsized":"2023-09-23T15:15:53.963945+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960446+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:15:53.960446+0000","last_peered":"2023-09-23T15:15:53.960446+0000","last_clean":"2023-09-23T15:15:53.960446+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:15:53.960446+0000","last_undegraded":"2023-09-23T15:15:53.960446+0000","last_fullsized":"2023-09-23T15:15:53.960446+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"214'12097","reported_seq":12148,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:13.664898+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:16:13.664898+0000","last_peered":"2023-09-23T15:16:13.664898+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:16:13.664898+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"177'3000","ondisk_log_start":"177'3000","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9097,"log_dups_size":0,"ondisk_log_size":9097,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26935296,"num_objects":411,"num_object_clones":0,"num_object_copies":1233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12097,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-4463}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.466088+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:15:54.466088+0000","last_peered":"2023-09-23T15:15:54.466088+0000","last_clean":"2023-09-23T15:15:54.466088+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:15:54.466088+0000","last_undegraded":"2023-09-23T15:15:54.466088+0000","last_fullsized":"2023-09-23T15:15:54.466088+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":820,"reported_epoch":213,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:15:53.961575+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:15:53.961575+0000","last_peered":"2023-09-23T15:15:53.961575+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:15:53.961575+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":86,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.965494+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:15:53.965494+0000","last_peered":"2023-09-23T15:15:53.965494+0000","last_clean":"2023-09-23T15:15:53.965494+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:15:53.965494+0000","last_undegraded":"2023-09-23T15:15:53.965494+0000","last_fullsized":"2023-09-23T15:15:53.965494+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":164,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959894+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:15:53.959894+0000","last_peered":"2023-09-23T15:15:53.959894+0000","last_clean":"2023-09-23T15:15:53.959894+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:15:53.959894+0000","last_undegraded":"2023-09-23T15:15:53.959894+0000","last_fullsized":"2023-09-23T15:15:53.959894+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"214'11926","reported_seq":11977,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:14.318515+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:16:14.318515+0000","last_peered":"2023-09-23T15:16:14.318515+0000","last_clean":"2023-09-23T15:16:14.318515+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:16:14.318515+0000","last_undegraded":"2023-09-23T15:16:14.318515+0000","last_fullsized":"2023-09-23T15:16:14.318515+0000","mapping_epoch":171,"log_start":"180'4400","ondisk_log_start":"180'4400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7526,"log_dups_size":0,"ondisk_log_size":7526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24641536,"num_objects":376,"num_object_clones":0,"num_object_copies":1128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11926,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961729+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:15:53.961729+0000","last_peered":"2023-09-23T15:15:53.961729+0000","last_clean":"2023-09-23T15:15:53.961729+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:15:53.961729+0000","last_undegraded":"2023-09-23T15:15:53.961729+0000","last_fullsized":"2023-09-23T15:15:53.961729+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":190,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960200+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:15:53.960200+0000","last_peered":"2023-09-23T15:15:53.960200+0000","last_clean":"2023-09-23T15:15:53.960200+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:15:53.960200+0000","last_undegraded":"2023-09-23T15:15:53.960200+0000","last_fullsized":"2023-09-23T15:15:53.960200+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":674,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.969724+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:15:53.969724+0000","last_peered":"2023-09-23T15:15:53.969724+0000","last_clean":"2023-09-23T15:15:53.969724+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:15:53.969724+0000","last_undegraded":"2023-09-23T15:15:53.969724+0000","last_fullsized":"2023-09-23T15:15:53.969724+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":195,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.150713+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:16:04.150713+0000","last_peered":"2023-09-23T15:16:04.150713+0000","last_clean":"2023-09-23T15:16:04.150713+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:16:04.150713+0000","last_undegraded":"2023-09-23T15:16:04.150713+0000","last_fullsized":"2023-09-23T15:16:04.150713+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"214'12258","reported_seq":12316,"reported_epoch":214,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:14.194543+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:16:14.194543+0000","last_peered":"2023-09-23T15:16:14.194543+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:16:14.194543+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"198'6900","ondisk_log_start":"198'6900","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5358,"log_dups_size":0,"ondisk_log_size":5358,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24379392,"num_objects":372,"num_object_clones":0,"num_object_copies":1116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12258,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-4594}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":491,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960194+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:15:53.960194+0000","last_peered":"2023-09-23T15:15:53.960194+0000","last_clean":"2023-09-23T15:15:53.960194+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:15:53.960194+0000","last_undegraded":"2023-09-23T15:15:53.960194+0000","last_fullsized":"2023-09-23T15:15:53.960194+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":690,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.964069+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:15:53.964069+0000","last_peered":"2023-09-23T15:15:53.964069+0000","last_clean":"2023-09-23T15:15:53.964069+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:15:53.964069+0000","last_undegraded":"2023-09-23T15:15:53.964069+0000","last_fullsized":"2023-09-23T15:15:53.964069+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":177,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.960530+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:15:53.960530+0000","last_peered":"2023-09-23T15:15:53.960530+0000","last_clean":"2023-09-23T15:15:53.960530+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:15:53.960530+0000","last_undegraded":"2023-09-23T15:15:53.960530+0000","last_fullsized":"2023-09-23T15:15:53.960530+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943446+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:15:53.943446+0000","last_peered":"2023-09-23T15:15:53.943446+0000","last_clean":"2023-09-23T15:15:53.943446+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:15:53.943446+0000","last_undegraded":"2023-09-23T15:15:53.943446+0000","last_fullsized":"2023-09-23T15:15:53.943446+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":166,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959823+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:15:53.959823+0000","last_peered":"2023-09-23T15:15:53.959823+0000","last_clean":"2023-09-23T15:15:53.959823+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:15:53.959823+0000","last_undegraded":"2023-09-23T15:15:53.959823+0000","last_fullsized":"2023-09-23T15:15:53.959823+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":214,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.957710+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:15:53.957710+0000","last_peered":"2023-09-23T15:15:53.957710+0000","last_clean":"2023-09-23T15:15:53.957710+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:15:53.957710+0000","last_undegraded":"2023-09-23T15:15:53.957710+0000","last_fullsized":"2023-09-23T15:15:53.957710+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":192,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959675+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:15:53.959675+0000","last_peered":"2023-09-23T15:15:53.959675+0000","last_clean":"2023-09-23T15:15:53.959675+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:15:53.959675+0000","last_undegraded":"2023-09-23T15:15:53.959675+0000","last_fullsized":"2023-09-23T15:15:53.959675+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":206,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943701+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:15:53.943701+0000","last_peered":"2023-09-23T15:15:53.943701+0000","last_clean":"2023-09-23T15:15:53.943701+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:15:53.943701+0000","last_undegraded":"2023-09-23T15:15:53.943701+0000","last_fullsized":"2023-09-23T15:15:53.943701+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":158,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.959469+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:15:53.959469+0000","last_peered":"2023-09-23T15:15:53.959469+0000","last_clean":"2023-09-23T15:15:53.959469+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:15:53.959469+0000","last_undegraded":"2023-09-23T15:15:53.959469+0000","last_fullsized":"2023-09-23T15:15:53.959469+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":189,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943491+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:15:53.943491+0000","last_peered":"2023-09-23T15:15:53.943491+0000","last_clean":"2023-09-23T15:15:53.943491+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:15:53.943491+0000","last_undegraded":"2023-09-23T15:15:53.943491+0000","last_fullsized":"2023-09-23T15:15:53.943491+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":186,"reported_epoch":214,"state":"active+clean","last_fresh":"2023-09-23T15:16:04.149158+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:16:04.149158+0000","last_peered":"2023-09-23T15:16:04.149158+0000","last_clean":"2023-09-23T15:16:04.149158+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:16:04.149158+0000","last_undegraded":"2023-09-23T15:16:04.149158+0000","last_fullsized":"2023-09-23T15:16:04.149158+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":200,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:54.750923+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:15:54.750923+0000","last_peered":"2023-09-23T15:15:54.750923+0000","last_clean":"2023-09-23T15:15:54.750923+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:15:54.750923+0000","last_undegraded":"2023-09-23T15:15:54.750923+0000","last_fullsized":"2023-09-23T15:15:54.750923+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":162,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.961242+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:15:53.961242+0000","last_peered":"2023-09-23T15:15:53.961242+0000","last_clean":"2023-09-23T15:15:53.961242+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:15:53.961242+0000","last_undegraded":"2023-09-23T15:15:53.961242+0000","last_fullsized":"2023-09-23T15:15:53.961242+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":182,"reported_epoch":213,"state":"active+clean","last_fresh":"2023-09-23T15:15:53.943305+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:15:53.943305+0000","last_peered":"2023-09-23T15:15:53.943305+0000","last_clean":"2023-09-23T15:15:53.943305+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:15:53.943305+0000","last_undegraded":"2023-09-23T15:15:53.943305+0000","last_fullsized":"2023-09-23T15:15:53.943305+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":403177496,"num_objects":6153,"num_object_clones":0,"num_object_copies":18459,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21520,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6153,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":194625,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8180,"num_bytes_recovered":476053504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1251749888,"data_stored":1251737672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120125,"ondisk_log_size":120125,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201538,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463174,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":385004,"kb_used_data":214116,"kb_used_omap":0,"kb_used_meta":170880,"kb_avail":93360148,"statfs":{"total":95995035648,"available":95600791552,"internally_reserved":0,"allocated":219254784,"data_stored":218538363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":174981120},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.1469999999999998,"15min":2.7120000000000002},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.2519999999999998,"5min":41.491,"15min":108.39400000000001},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.4649999999999999,"15min":2.9289999999999998},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.1029999999999998,"5min":43.151000000000003,"15min":108.452},"last":8.2949999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":3.395,"15min":3.379},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.266,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.40000000000000002},{"interface":"front","average":{"1min":3.2589999999999999,"5min":4.274,"15min":3.8290000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.4060000000000006,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.96699999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5129999999999999,"5min":5.7489999999999997,"15min":6.0949999999999998},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":68.828000000000003,"5min":112.352,"15min":180.374},"last":7.694},{"interface":"front","average":{"1min":5.681,"5min":5.8140000000000001,"15min":6.0709999999999997},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":68.730999999999995,"5min":117.667,"15min":181.685},"last":2.3330000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":15.903,"5min":7.0270000000000001,"15min":3.012},"min":{"1min":0.627,"5min":0.249,"15min":0.249},"max":{"1min":102.254,"5min":102.254,"15min":102.254},"last":1.9399999999999999},{"interface":"front","average":{"1min":17.141999999999999,"5min":7.4619999999999997,"15min":3.0840000000000001},"min":{"1min":0.46100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":98.941999999999993,"5min":98.941999999999993,"15min":98.941999999999993},"last":1.859}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0349999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.7150000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.624,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0019999999999998,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.0099999999999998,"15min":1.423},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":3.1789999999999998,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.0249999999999999,"15min":1.4590000000000001},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0510000000000002,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.2689999999999999,"15min":1.032},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.125,"5min":7.5910000000000002,"15min":17},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1659999999999999,"15min":0.995},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":2.524,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.59999999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015211,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":365960,"kb_used_data":203392,"kb_used_omap":6,"kb_used_meta":162553,"kb_avail":93379192,"statfs":{"total":95995035648,"available":95620292608,"internally_reserved":0,"allocated":208273408,"data_stored":207245038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":166454347},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":3.4049999999999998,"15min":3.6709999999999998},"min":{"1min":0.40200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.338000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":5.6859999999999999,"5min":4.6150000000000002,"15min":4.6109999999999998},"min":{"1min":0.29599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.414999999999999,"5min":16.414999999999999,"15min":16.414999999999999},"last":0.97499999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":4.1429999999999998,"15min":4.133},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.315,"5min":13.843,"15min":13.843},"last":1.766},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.0709999999999997,"15min":4.1719999999999997},"min":{"1min":0.434,"5min":0.253,"15min":0.253},"max":{"1min":16.364999999999998,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.41399999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":8.3599999999999994,"15min":9.984},"min":{"1min":0.40899999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.617999999999999,"5min":123.426,"15min":123.426},"last":0.47799999999999998},{"interface":"front","average":{"1min":5.3920000000000003,"5min":9.3279999999999994,"15min":11.486000000000001},"min":{"1min":0.68100000000000005,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.451000000000001,"5min":123.32899999999999,"15min":123.32899999999999},"last":0.871}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8409999999999993,"5min":9.8409999999999993,"15min":9.8409999999999993},"last":0.63700000000000001},{"interface":"front","average":{"1min":2.823,"5min":3.3250000000000002,"15min":3.48},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.228000000000002,"5min":16.779,"15min":16.779},"last":0.83299999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":3.4340000000000002,"15min":3.6640000000000001},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.898999999999999,"5min":16.166,"15min":16.166},"last":0.90200000000000002},{"interface":"front","average":{"1min":3.5179999999999998,"5min":3.0449999999999999,"15min":2.9460000000000002},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8680000000000003,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":3.4209999999999998,"15min":3.6800000000000002},"min":{"1min":0.67500000000000004,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":14.867000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":0.94599999999999995},{"interface":"front","average":{"1min":3.7240000000000002,"5min":3,"15min":2.8319999999999999},"min":{"1min":0.438,"5min":0.377,"15min":0.377},"max":{"1min":12.448,"5min":12.448,"15min":12.448},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":3.1379999999999999,"15min":3.1499999999999999},"min":{"1min":0.47199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.66900000000000004},{"interface":"front","average":{"1min":3.5169999999999999,"5min":3.323,"15min":3.343},"min":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.280999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.79500000000000004}]}]},{"osd":0,"up_from":95,"seq":408021893261,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":434908,"kb_used_data":105236,"kb_used_omap":8,"kb_used_meta":329655,"kb_avail":93310244,"statfs":{"total":95995035648,"available":95549689856,"internally_reserved":0,"allocated":107761664,"data_stored":106830532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":337567316},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":3.298,"15min":3.96},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4109999999999996,"5min":31.510999999999999,"15min":31.510999999999999},"last":15.462},{"interface":"front","average":{"1min":1.9770000000000001,"5min":3.7280000000000002,"15min":4.3819999999999997},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5590000000000002,"5min":31.378,"15min":31.378},"last":12.099}]},{"osd":2,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":7.133,"15min":3.5539999999999998},"min":{"1min":0.26200000000000001,"5min":0.189,"15min":0.17000000000000001},"max":{"1min":13.593,"5min":103.81,"15min":122.20099999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":3.4279999999999999,"5min":6.077,"15min":3.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.18099999999999999},"max":{"1min":9.7479999999999993,"5min":108.19499999999999,"15min":122.071},"last":18.960999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":8.2420000000000009,"5min":6.8710000000000004,"15min":2.8610000000000002},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":72.137,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":13.818,"5min":6.9850000000000003,"15min":3.2029999999999998},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":95.795000000000002,"5min":95.795000000000002,"15min":95.795000000000002},"last":15.507999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.5230000000000001,"15min":1.649},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.9950000000000001,"5min":31.341999999999999,"15min":31.341999999999999},"last":4.343},{"interface":"front","average":{"1min":2.4710000000000001,"5min":3.282,"15min":1.72},"min":{"1min":0.29699999999999999,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":13.566000000000001,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.42599999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.992,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29699999999999999},"max":{"1min":9.6020000000000003,"5min":109.369,"15min":109.369},"last":5.3470000000000004},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.2170000000000001,"15min":1.857},"min":{"1min":0.42999999999999999,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":4.4219999999999997,"5min":109.387,"15min":109.387},"last":4.2800000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.996,"15min":1.752},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":13.529999999999999,"5min":99.114999999999995,"15min":99.114999999999995},"last":5.6909999999999998},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.8650000000000002,"15min":1.677},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":9.6460000000000008,"5min":98.921999999999997,"15min":98.921999999999997},"last":5.3849999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":3.181,"15min":1.802},"min":{"1min":0.44600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.6690000000000005,"5min":99.549999999999997,"15min":99.549999999999997},"last":5.4080000000000004},{"interface":"front","average":{"1min":1.702,"5min":2.2770000000000001,"15min":1.5549999999999999},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9720000000000004,"5min":31.225999999999999,"15min":31.225999999999999},"last":4.3140000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566208,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":582232,"kb_used_data":249488,"kb_used_omap":11,"kb_used_meta":332724,"kb_avail":93162920,"statfs":{"total":95995035648,"available":95398830080,"internally_reserved":0,"allocated":255475712,"data_stored":254552071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":340710160},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":4.9969999999999999,"15min":2.2229999999999999},"min":{"1min":0.27400000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.085000000000001,"5min":87.132999999999996,"15min":87.132999999999996},"last":0.23699999999999999},{"interface":"front","average":{"1min":3.931,"5min":3.8490000000000002,"15min":1.831},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.833,"5min":84.820999999999998,"15min":84.820999999999998},"last":16.748999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":5.492,"15min":5.1900000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.058999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":9.1850000000000005},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.782,"15min":2.96},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.962999999999999,"5min":84.751999999999995,"15min":84.751999999999995},"last":16.954000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:15:29 2023","interfaces":[{"interface":"back","average":{"1min":14.097,"5min":6.1619999999999999,"15min":2.661},"min":{"1min":0.70399999999999996,"5min":0.219,"15min":0.219},"max":{"1min":86.332999999999998,"5min":87.108000000000004,"15min":87.108000000000004},"last":11.723000000000001},{"interface":"front","average":{"1min":12.151999999999999,"5min":5.577,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":7.3440000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.129,"15min":1.831},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.824,"5min":84.608000000000004,"15min":84.608000000000004},"last":16.831},{"interface":"front","average":{"1min":2.8849999999999998,"5min":4.016,"15min":2.4020000000000001},"min":{"1min":0.42399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.988,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.52700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":3.266,"15min":2.0579999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.193999999999999,"5min":84.498000000000005,"15min":84.498000000000005},"last":7.3700000000000001},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.8690000000000002,"15min":1.9179999999999999},"min":{"1min":0.441,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.865,"5min":84.664000000000001,"15min":84.664000000000001},"last":16.898}]},{"osd":6,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.1200000000000001,"15min":2.0369999999999999},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.891999999999999,"5min":84.715999999999994,"15min":84.715999999999994},"last":16.872},{"interface":"front","average":{"1min":2.9729999999999999,"5min":3.5750000000000002,"15min":2.0939999999999999},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.949999999999999,"5min":87.055000000000007,"15min":87.055000000000007},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":3.3519999999999999,"15min":1.915},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":16.239000000000001,"5min":84.596000000000004,"15min":84.596000000000004},"last":7.3959999999999999},{"interface":"front","average":{"1min":1.966,"5min":3.0630000000000002,"15min":1.8049999999999999},"min":{"1min":0.48899999999999999,"5min":0.38600000000000001,"15min":0.34200000000000003},"max":{"1min":10.916,"5min":84.783000000000001,"15min":84.783000000000001},"last":16.920999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402682,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":605676,"kb_used_data":292132,"kb_used_omap":3,"kb_used_meta":313532,"kb_avail":93139476,"statfs":{"total":95995035648,"available":95374823424,"internally_reserved":0,"allocated":299143168,"data_stored":298254466,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":321056937},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":3.3100000000000001,"15min":2.0110000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":13.064,"5min":61.744,"15min":79.611000000000004},"last":6.0640000000000001},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.2130000000000001,"15min":2.0249999999999999},"min":{"1min":0.40200000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":12.348000000000001,"5min":61.686,"15min":79.656000000000006},"last":14.295}]},{"osd":1,"last update":"Sat Sep 23 15:15:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":4.702,"15min":4.6239999999999997},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.566000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":0.499},{"interface":"front","average":{"1min":3.5339999999999998,"5min":4.5609999999999999,"15min":4.7510000000000003},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.539000000000001,"5min":21.539000000000001,"15min":21.539000000000001},"last":6.093}]},{"osd":2,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8600000000000003,"5min":4.6020000000000003,"15min":3.6909999999999998},"min":{"1min":0.432,"5min":0.22,"15min":0.22},"max":{"1min":61.732999999999997,"5min":63.009999999999998,"15min":109.077},"last":6.5270000000000001},{"interface":"front","average":{"1min":8.202,"5min":5.101,"15min":3.9460000000000002},"min":{"1min":0.47999999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":60.305999999999997,"5min":69.715999999999994,"15min":119.72799999999999},"last":12.478}]},{"osd":4,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.6139999999999999,"15min":1.4319999999999999},"min":{"1min":0.36199999999999999,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":54.051000000000002,"5min":54.051000000000002,"15min":54.051000000000002},"last":12.321},{"interface":"front","average":{"1min":5.2939999999999996,"5min":2.7029999999999998,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":52.228999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.55000000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":2.399,"15min":1.5740000000000001},"min":{"1min":0.40899999999999997,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":8.0459999999999994,"5min":8.9450000000000003,"15min":56.960999999999999},"last":12.359999999999999},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.552,"15min":1.806},"min":{"1min":0.49299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.8730000000000002,"5min":10.738,"15min":56.981999999999999},"last":0.65900000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":3.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.42899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":78.102000000000004,"5min":78.102000000000004,"15min":78.102000000000004},"last":12.451000000000001},{"interface":"front","average":{"1min":6.5919999999999996,"5min":3.327,"15min":1.962},"min":{"1min":0.69299999999999995,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":78.028000000000006,"5min":78.028000000000006,"15min":78.028000000000006},"last":1.288}]},{"osd":7,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":2.5110000000000001,"15min":1.71},"min":{"1min":0.52200000000000002,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":5.923,"5min":10.603,"15min":14.885999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":3.016,"5min":2.2160000000000002,"15min":1.363},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.109,"5min":14.351000000000001,"15min":14.351000000000001},"last":6.1230000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986447,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":331368,"kb_used_data":136928,"kb_used_omap":0,"kb_used_meta":194432,"kb_avail":93413784,"statfs":{"total":95995035648,"available":95655714816,"internally_reserved":0,"allocated":140214272,"data_stored":139501426,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":199098368},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":2.5830000000000002,"15min":2.6200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.8250000000000002,"5min":65.988,"15min":122.69199999999999},"last":4.2460000000000004},{"interface":"front","average":{"1min":1.552,"5min":2.5219999999999998,"15min":2.698},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":5.7149999999999999,"5min":65.396000000000001,"15min":122.745},"last":5.3540000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":3.6920000000000002,"15min":4.2629999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7539999999999996,"5min":53.832000000000001,"15min":53.832000000000001},"last":1.679},{"interface":"front","average":{"1min":2.8860000000000001,"5min":4.3369999999999997,"15min":4.8300000000000001},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.114000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.89400000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":4.1260000000000003,"15min":3.1080000000000001},"min":{"1min":0.37,"5min":0.255,"15min":0.255},"max":{"1min":12.595000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":2.153},{"interface":"front","average":{"1min":3.2869999999999999,"5min":4.383,"15min":3.3530000000000002},"min":{"1min":0.34300000000000003,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":7.173,"5min":147.88900000000001,"15min":147.88900000000001},"last":1.8180000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":9.3309999999999995,"15min":3.7869999999999999},"min":{"1min":0.36799999999999999,"5min":0.252,"15min":0.252},"max":{"1min":96.447000000000003,"5min":163.97499999999999,"15min":163.97499999999999},"last":1.4339999999999999},{"interface":"front","average":{"1min":7.0019999999999998,"5min":9.1080000000000005,"15min":3.5750000000000002},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":95.944999999999993,"5min":164.447,"15min":164.447},"last":3.5}]},{"osd":5,"last update":"Sat Sep 23 15:15:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.966,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":87.912999999999997,"5min":87.912999999999997,"15min":87.912999999999997},"last":1.9930000000000001},{"interface":"front","average":{"1min":5.4960000000000004,"5min":1.9450000000000001,"15min":1.395},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":88.602999999999994,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.86199999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:15:22 2023","interfaces":[{"interface":"back","average":{"1min":5.2169999999999996,"5min":4.2329999999999997,"15min":2.0449999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":86.091999999999999,"5min":210.44300000000001,"15min":210.44300000000001},"last":1.1739999999999999},{"interface":"front","average":{"1min":5.4509999999999996,"5min":3.984,"15min":1.9319999999999999},"min":{"1min":0.33900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":86.048000000000002,"5min":210.78899999999999,"15min":210.78899999999999},"last":1.0309999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.91800000000000004,"15min":1.129},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":0.75900000000000001,"5min":7.7389999999999999,"15min":18.23},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.079,"5min":1.3400000000000001,"15min":1.179},"min":{"1min":0.375,"5min":0.314,"15min":0.314},"max":{"1min":6.1399999999999997,"5min":14.044,"15min":14.653},"last":3.7789999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724810,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":337120,"kb_used_data":124760,"kb_used_omap":0,"kb_used_meta":212352,"kb_avail":93408032,"statfs":{"total":95995035648,"available":95649824768,"internally_reserved":0,"allocated":127754240,"data_stored":127045373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":217448448},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.9430000000000001,"15min":1.4079999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":7.8319999999999999,"5min":16.981999999999999,"15min":24.962},"last":0.34499999999999997},{"interface":"front","average":{"1min":3.972,"5min":2.1589999999999998,"15min":1.4550000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":12.531000000000001,"5min":18.814,"15min":25.317},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.3730000000000002,"15min":1.9059999999999999},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.6110000000000007,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.29899999999999999},{"interface":"front","average":{"1min":4.1109999999999998,"5min":2.7730000000000001,"15min":2.4369999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.423,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.266,"15min":2.4609999999999999},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3889999999999993,"5min":52.414000000000001,"15min":97.087999999999994},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.5070000000000001,"5min":3.3570000000000002,"15min":2.4209999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":52.375999999999998,"15min":97.305999999999997},"last":0.41999999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":10.388,"5min":5.0510000000000002,"15min":2.488},"min":{"1min":0.54900000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.570999999999998,"5min":80.570999999999998,"15min":80.570999999999998},"last":118.208},{"interface":"front","average":{"1min":9.8190000000000008,"5min":5.1440000000000001,"15min":2.3460000000000001},"min":{"1min":0.44400000000000001,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":80.551000000000002,"5min":80.551000000000002,"15min":80.551000000000002},"last":118.42}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":8.7479999999999993,"5min":3.2589999999999999,"15min":1.718},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":138.68600000000001,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":8.7729999999999997,"5min":3.415,"15min":1.788},"min":{"1min":0.34300000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":138.49600000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.36299999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88900000000000001,"15min":1.1339999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.5550000000000002,"5min":5.8760000000000003,"15min":15.196},"last":0.436},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.86499999999999999,"15min":1.0229999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9990000000000001,"5min":4.8819999999999997,"15min":14.965999999999999},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.3380000000000001,"15min":1.288},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.28599999999999998},"max":{"1min":8.2200000000000006,"5min":9.6560000000000006,"15min":15.301},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.141,"5min":1.3700000000000001,"15min":1.252},"min":{"1min":0.39900000000000002,"5min":0.311,"15min":0.311},"max":{"1min":5.6559999999999997,"5min":15.762,"15min":15.762},"last":0.46700000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":99749888,"data_stored":99745816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":188944384,"data_stored":188940312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":241500160,"data_stored":241500160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":269287424,"data_stored":269287424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":128843776,"data_stored":128843776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":122097664,"data_stored":122093592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":201326592,"data_stored":201326592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:16:19.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:16:19.890 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:19.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:20.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:20.038 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:20.396 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:20.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:20.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:20.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:20.635 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:20.898 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:20.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:20.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:20.941 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:20.987 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:16:20.987 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:16:21.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:21.055 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:21.114 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:21.124 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:21.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:21.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:21.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:21.345 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:21.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:21 smithi017 ceph-mon[222878]: pgmap v603: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:21.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:21 smithi017 ceph-mon[215646]: pgmap v603: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:21.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:21 smithi027 ceph-mon[195233]: pgmap v603: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:22.251 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:22.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:22.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:16:22.366 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:22.513 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:22.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21333/10155 objects degraded (210.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21333/10155 objects degraded (210.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.844 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:22.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21333/10155 objects degraded (210.074%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:22.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:22.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:22.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:23.008 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:23.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:23.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:23.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:23.176 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:23.221 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:23.670 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:23.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:23.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:23.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:23.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:23 smithi017 ceph-mon[215646]: pgmap v604: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:23.807 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:23 smithi017 ceph-mon[222878]: pgmap v604: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:23.820 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:23 smithi027 ceph-mon[195233]: pgmap v604: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 214 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 549 op/s; 21333/10155 objects degraded (210.074%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:23.823 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:23.883 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:23.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:23.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:23.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:24.089 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:24.374 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:24.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:24.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:24.662 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:24.837 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:24.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:24.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:24.987 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:25.041 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:25.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:25.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:25.059 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:25.067 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:25.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:25.175 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:25.231 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:25.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:25.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:25.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:25.368 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:25.419 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:25.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:25.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:25.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:25.709 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:25.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:25 smithi017 ceph-mon[215646]: pgmap v605: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 143 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21333/6714 objects degraded (317.739%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:25.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:25 smithi017 ceph-mon[222878]: pgmap v605: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 143 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21333/6714 objects degraded (317.739%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:25 smithi027 ceph-mon[195233]: pgmap v605: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 143 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21333/6714 objects degraded (317.739%); 0 B/s, 34 objects/s recovering 2023-09-23T15:16:25.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 100388 objects 2023-09-23T15:16:25.903 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :185.749 2023-09-23T15:16:25.981 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:25.981 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 21333/6714 objects degraded (317.739%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":793,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":215,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":102},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":2238,"data_bytes":149592284,"bytes_used":2264449024,"bytes_avail":669700800512,"bytes_total":671965249536,"degraded_objects":21333,"degraded_total":6714,"degraded_ratio":3.1773905272564789,"recovering_objects_per_sec":34,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":341,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":542},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T15:13:42.319777+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (7m)\n [==========================..] (remaining: 30s)","progress":0.93577980995178223,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2726517743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3438807787' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2726517743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.558 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3438807787' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:26.566 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:16:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2726517743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:26.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3438807787' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: pgmap v606: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: failed to encode map e216 with expected crc 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T15:16:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21168/2244 objects degraded (943.316%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:27.887 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:27.889 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: pgmap v606: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:27.889 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:27.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: failed to encode map e216 with expected crc 2023-09-23T15:16:27.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:27.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T15:16:27.890 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21168/2244 objects degraded (943.316%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:27.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: pgmap v606: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.9 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 481 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 31 objects/s recovering 2023-09-23T15:16:27.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:27.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: failed to encode map e216 with expected crc 2023-09-23T15:16:27.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:27.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: osdmap e216: 8 total, 8 up, 7 in 2023-09-23T15:16:27.891 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21168/2244 objects degraded (943.316%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:27.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:27.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:27.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:28.024 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:28.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:28] "GET /metrics HTTP/1.1" 200 36304 "" "Prometheus/2.43.0" 2023-09-23T15:16:28.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[215646]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[215646]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[215646]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[222878]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[222878]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[222878]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:28.825 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:28.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:28 smithi027 ceph-mon[195233]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:28 smithi027 ceph-mon[195233]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:28 smithi027 ceph-mon[195233]: failed to encode map e216 with expected crc 2023-09-23T15:16:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:28.928 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:28.963 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:29.016 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:29.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:29.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:29.123 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:29.154 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:29.203 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:29.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:29.209 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:29.310 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:29.345 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:29.417 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:16:29.417 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:16:29.608 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:16:29.608 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:16:29.963 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:29.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:29.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:30.066 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:30.093 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:30.430 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:30.432 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:30 smithi017 ceph-mon[215646]: pgmap v608: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 35 objects/s recovering 2023-09-23T15:16:30.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:30 smithi017 ceph-mon[222878]: pgmap v608: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 35 objects/s recovering 2023-09-23T15:16:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:30.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:30.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:30.558 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:30 smithi027 ceph-mon[195233]: pgmap v608: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 50 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 542 op/s; 21168/2244 objects degraded (943.316%); 0 B/s, 35 objects/s recovering 2023-09-23T15:16:30.561 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:30.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:16:30.608 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:30.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:30.633 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T15:16:30.633 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T15:16:30.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:30.748 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:30.824 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:30.843 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:30.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:30.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:30.861 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:30.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:30.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:30.940 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:30.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:30.963 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:30.976 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:30.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:31.008 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:31.023 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:31.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:31.028 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:31.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.030 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.031 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:16:31.093 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:16:31.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:31.172 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:31 smithi027 ceph-mon[195233]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:31 smithi027 ceph-mon[195233]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:31 smithi027 ceph-mon[195233]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T15:16:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:31 smithi027 ceph-mon[195233]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4250277577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[215646]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[215646]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[215646]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T15:16:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[215646]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4250277577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[222878]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[222878]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[222878]: osdmap e217: 8 total, 8 up, 7 in 2023-09-23T15:16:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[222878]: failed to encode map e217 with expected crc 2023-09-23T15:16:31.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4250277577' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:31.836 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:31.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:31.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:31.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:31.971 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:32.024 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:32.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:32.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:32.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:32.147 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:32.164 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:32.374 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:32.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[215646]: pgmap v610: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 183 objects/s recovering 2023-09-23T15:16:32.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[215646]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[215646]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[215646]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T15:16:32.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 19868/81 objects degraded (24528.395%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:32.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[222878]: pgmap v610: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 183 objects/s recovering 2023-09-23T15:16:32.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[222878]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[222878]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[222878]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T15:16:32.377 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 19868/81 objects degraded (24528.395%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:32.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:32.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:32.507 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:32.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:32 smithi027 ceph-mon[195233]: pgmap v610: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 419 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 183 objects/s recovering 2023-09-23T15:16:32.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:32 smithi027 ceph-mon[195233]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:32 smithi027 ceph-mon[195233]: failed to encode map e218 with expected crc 2023-09-23T15:16:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:32 smithi027 ceph-mon[195233]: osdmap e218: 8 total, 8 up, 7 in 2023-09-23T15:16:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 19868/81 objects degraded (24528.395%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:32.707 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:32.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:32.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:32.839 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:33.000 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:33.002 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:33.196 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:33.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:33.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:33.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:33.379 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:33.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:33.386 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:33.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:33.516 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:33.812 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:33.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:33.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:33.945 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:34.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:16:34.179 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:34.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:34.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:34.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:34.310 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:34.358 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:34.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:34.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:34.435 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:34.438 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:16:34.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[215646]: pgmap v612: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 216 objects/s recovering 2023-09-23T15:16:34.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[222878]: pgmap v612: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 216 objects/s recovering 2023-09-23T15:16:34.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:34.496 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:34.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:34 smithi027 ceph-mon[195233]: pgmap v612: 109 pgs: 102 active+clean, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 120 op/s; 19868/81 objects degraded (24528.395%); 0 B/s, 216 objects/s recovering 2023-09-23T15:16:34.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:34.700 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:34.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:34.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:34.803 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:34.830 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:34.875 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:34.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:34.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:34.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:35.007 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:35.053 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:35.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:35.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:35.159 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:35.190 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:16:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:35.462 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:16:35.462 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:16:35.462 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:16:35.462 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:16:35.488 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:35.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:35.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:35.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:16:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:16:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:35.621 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:35.667 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:35.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:35.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:35.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:35.801 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:36.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:16:36.158 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":612,"stamp":"2023-09-23T15:16:32.440374+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4757700,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6238,"num_read_kb":13669,"num_write":201806,"num_write_kb":6438768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9894,"num_bytes_recovered":481400999,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":124691,"ondisk_log_size":124691,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1774052,"kb_used_data":29932,"kb_used_omap":30,"kb_used_meta":1744033,"kb_avail":654442012,"statfs":{"total":671965249536,"available":670148620288,"internally_reserved":0,"allocated":30650368,"data_stored":24669490,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1785890392},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-47185944,"num_objects":-721,"num_object_clones":0,"num_object_copies":-2163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":721,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1300,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001455"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":210,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189206+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:16:15.189206+0000","last_peered":"2023-09-23T15:16:15.189206+0000","last_clean":"2023-09-23T15:16:15.189206+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:16:15.189206+0000","last_undegraded":"2023-09-23T15:16:15.189206+0000","last_fullsized":"2023-09-23T15:16:15.189206+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199027+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:16:15.199027+0000","last_peered":"2023-09-23T15:16:15.199027+0000","last_clean":"2023-09-23T15:16:15.199027+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:16:15.199027+0000","last_undegraded":"2023-09-23T15:16:15.199027+0000","last_fullsized":"2023-09-23T15:16:15.199027+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":95,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151403+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:16:30.151403+0000","last_peered":"2023-09-23T15:16:30.151403+0000","last_clean":"2023-09-23T15:16:30.151403+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:16:30.151403+0000","last_undegraded":"2023-09-23T15:16:30.151403+0000","last_fullsized":"2023-09-23T15:16:30.151403+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":181,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179337+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:16:15.179337+0000","last_peered":"2023-09-23T15:16:15.179337+0000","last_clean":"2023-09-23T15:16:15.179337+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:16:15.179337+0000","last_undegraded":"2023-09-23T15:16:15.179337+0000","last_fullsized":"2023-09-23T15:16:15.179337+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":214,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198137+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:16:15.198137+0000","last_peered":"2023-09-23T15:16:15.198137+0000","last_clean":"2023-09-23T15:16:15.198137+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:16:15.198137+0000","last_undegraded":"2023-09-23T15:16:15.198137+0000","last_fullsized":"2023-09-23T15:16:15.198137+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":190,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199471+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:16:15.199471+0000","last_peered":"2023-09-23T15:16:15.199471+0000","last_clean":"2023-09-23T15:16:15.199471+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:16:15.199471+0000","last_undegraded":"2023-09-23T15:16:15.199471+0000","last_fullsized":"2023-09-23T15:16:15.199471+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":160,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176922+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:16:15.176922+0000","last_peered":"2023-09-23T15:16:15.176922+0000","last_clean":"2023-09-23T15:16:15.176922+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:16:15.176922+0000","last_undegraded":"2023-09-23T15:16:15.176922+0000","last_fullsized":"2023-09-23T15:16:15.176922+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":196,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198168+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:16:15.198168+0000","last_peered":"2023-09-23T15:16:15.198168+0000","last_clean":"2023-09-23T15:16:15.198168+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:16:15.198168+0000","last_undegraded":"2023-09-23T15:16:15.198168+0000","last_fullsized":"2023-09-23T15:16:15.198168+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176586+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:16:15.176586+0000","last_peered":"2023-09-23T15:16:15.176586+0000","last_clean":"2023-09-23T15:16:15.176586+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:16:15.176586+0000","last_undegraded":"2023-09-23T15:16:15.176586+0000","last_fullsized":"2023-09-23T15:16:15.176586+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.182985+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:16:15.182985+0000","last_peered":"2023-09-23T15:16:15.182985+0000","last_clean":"2023-09-23T15:16:15.182985+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:16:15.182985+0000","last_undegraded":"2023-09-23T15:16:15.182985+0000","last_fullsized":"2023-09-23T15:16:15.182985+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":139,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.197055+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:16:15.197055+0000","last_peered":"2023-09-23T15:16:15.197055+0000","last_clean":"2023-09-23T15:16:15.197055+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:16:15.197055+0000","last_undegraded":"2023-09-23T15:16:15.197055+0000","last_fullsized":"2023-09-23T15:16:15.197055+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":79,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:24.814938+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:16:24.814938+0000","last_peered":"2023-09-23T15:16:24.814938+0000","last_clean":"2023-09-23T15:16:24.814938+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:16:24.814938+0000","last_undegraded":"2023-09-23T15:16:24.814938+0000","last_fullsized":"2023-09-23T15:16:24.814938+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151581+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:16:30.151581+0000","last_peered":"2023-09-23T15:16:30.151581+0000","last_clean":"2023-09-23T15:16:30.151581+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:16:30.151581+0000","last_undegraded":"2023-09-23T15:16:30.151581+0000","last_fullsized":"2023-09-23T15:16:30.151581+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":200,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189198+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:16:15.189198+0000","last_peered":"2023-09-23T15:16:15.189198+0000","last_clean":"2023-09-23T15:16:15.189198+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:16:15.189198+0000","last_undegraded":"2023-09-23T15:16:15.189198+0000","last_fullsized":"2023-09-23T15:16:15.189198+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":191,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189241+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:16:15.189241+0000","last_peered":"2023-09-23T15:16:15.189241+0000","last_clean":"2023-09-23T15:16:15.189241+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:16:15.189241+0000","last_undegraded":"2023-09-23T15:16:15.189241+0000","last_fullsized":"2023-09-23T15:16:15.189241+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176330+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:16:15.176330+0000","last_peered":"2023-09-23T15:16:15.176330+0000","last_clean":"2023-09-23T15:16:15.176330+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:16:15.176330+0000","last_undegraded":"2023-09-23T15:16:15.176330+0000","last_fullsized":"2023-09-23T15:16:15.176330+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189324+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:16:15.189324+0000","last_peered":"2023-09-23T15:16:15.189324+0000","last_clean":"2023-09-23T15:16:15.189324+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:16:15.189324+0000","last_undegraded":"2023-09-23T15:16:15.189324+0000","last_fullsized":"2023-09-23T15:16:15.189324+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":207,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.200811+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:16:15.200811+0000","last_peered":"2023-09-23T15:16:15.200811+0000","last_clean":"2023-09-23T15:16:15.200811+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:16:15.200811+0000","last_undegraded":"2023-09-23T15:16:15.200811+0000","last_fullsized":"2023-09-23T15:16:15.200811+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175554+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:16:15.175554+0000","last_peered":"2023-09-23T15:16:15.175554+0000","last_clean":"2023-09-23T15:16:15.175554+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:16:15.175554+0000","last_undegraded":"2023-09-23T15:16:15.175554+0000","last_fullsized":"2023-09-23T15:16:15.175554+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":181,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:19.497117+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:16:19.497117+0000","last_peered":"2023-09-23T15:16:19.497117+0000","last_clean":"2023-09-23T15:16:19.497117+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:16:19.497117+0000","last_undegraded":"2023-09-23T15:16:19.497117+0000","last_fullsized":"2023-09-23T15:16:19.497117+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":184,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183170+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:16:15.183170+0000","last_peered":"2023-09-23T15:16:15.183170+0000","last_clean":"2023-09-23T15:16:15.183170+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:16:15.183170+0000","last_undegraded":"2023-09-23T15:16:15.183170+0000","last_fullsized":"2023-09-23T15:16:15.183170+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175800+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:16:15.175800+0000","last_peered":"2023-09-23T15:16:15.175800+0000","last_clean":"2023-09-23T15:16:15.175800+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:16:15.175800+0000","last_undegraded":"2023-09-23T15:16:15.175800+0000","last_fullsized":"2023-09-23T15:16:15.175800+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":62,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189367+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:16:15.189367+0000","last_peered":"2023-09-23T15:16:15.189367+0000","last_clean":"2023-09-23T15:16:15.189367+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:16:15.189367+0000","last_undegraded":"2023-09-23T15:16:15.189367+0000","last_fullsized":"2023-09-23T15:16:15.189367+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":169,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179413+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:16:15.179413+0000","last_peered":"2023-09-23T15:16:15.179413+0000","last_clean":"2023-09-23T15:16:15.179413+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:16:15.179413+0000","last_undegraded":"2023-09-23T15:16:15.179413+0000","last_fullsized":"2023-09-23T15:16:15.179413+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176867+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:16:15.176867+0000","last_peered":"2023-09-23T15:16:15.176867+0000","last_clean":"2023-09-23T15:16:15.176867+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:16:15.176867+0000","last_undegraded":"2023-09-23T15:16:15.176867+0000","last_fullsized":"2023-09-23T15:16:15.176867+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":182,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198538+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:16:15.198538+0000","last_peered":"2023-09-23T15:16:15.198538+0000","last_clean":"2023-09-23T15:16:15.198538+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:16:15.198538+0000","last_undegraded":"2023-09-23T15:16:15.198538+0000","last_fullsized":"2023-09-23T15:16:15.198538+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":207,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199710+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:16:15.199710+0000","last_peered":"2023-09-23T15:16:15.199710+0000","last_clean":"2023-09-23T15:16:15.199710+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:16:15.199710+0000","last_undegraded":"2023-09-23T15:16:15.199710+0000","last_fullsized":"2023-09-23T15:16:15.199710+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"215'12394","reported_seq":12447,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.854800+0000","last_change":"2023-09-23T15:11:42.375338+0000","last_active":"2023-09-23T15:16:25.854800+0000","last_peered":"2023-09-23T15:16:25.854800+0000","last_clean":"2023-09-23T15:16:25.854800+0000","last_became_active":"2023-09-23T15:11:42.375028+0000","last_became_peered":"2023-09-23T15:11:42.375028+0000","last_unstale":"2023-09-23T15:16:25.854800+0000","last_undegraded":"2023-09-23T15:16:25.854800+0000","last_fullsized":"2023-09-23T15:16:25.854800+0000","mapping_epoch":171,"log_start":"176'2300","ondisk_log_start":"176'2300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10094,"log_dups_size":0,"ondisk_log_size":10094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:02.871967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12394,"num_write_kb":396608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176676+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:16:15.176676+0000","last_peered":"2023-09-23T15:16:15.176676+0000","last_clean":"2023-09-23T15:16:15.176676+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:16:15.176676+0000","last_undegraded":"2023-09-23T15:16:15.176676+0000","last_fullsized":"2023-09-23T15:16:15.176676+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":90,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.192059+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:16:15.192059+0000","last_peered":"2023-09-23T15:16:15.192059+0000","last_clean":"2023-09-23T15:16:15.192059+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:16:15.192059+0000","last_undegraded":"2023-09-23T15:16:15.192059+0000","last_fullsized":"2023-09-23T15:16:15.192059+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":167,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.182738+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:16:15.182738+0000","last_peered":"2023-09-23T15:16:15.182738+0000","last_clean":"2023-09-23T15:16:15.182738+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:16:15.182738+0000","last_undegraded":"2023-09-23T15:16:15.182738+0000","last_fullsized":"2023-09-23T15:16:15.182738+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"215'12666","reported_seq":22599,"reported_epoch":216,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:29.814213+0000","last_change":"2023-09-23T15:14:17.267663+0000","last_active":"2023-09-23T15:16:29.814213+0000","last_peered":"2023-09-23T15:16:29.814213+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.299028+0000","last_became_peered":"2023-09-23T15:12:43.299028+0000","last_unstale":"2023-09-23T15:16:29.814213+0000","last_undegraded":"2023-09-23T15:12:43.028174+0000","last_fullsized":"2023-09-23T15:12:42.856971+0000","mapping_epoch":184,"log_start":"195'6200","ondisk_log_start":"195'6200","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6466,"log_dups_size":0,"ondisk_log_size":6466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:29.401957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":63,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12666,"num_write_kb":405312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4935,"num_bytes_recovered":154927104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,6],"avail_no_missing":["3","6"],"object_location_counts":[{"shards":"3,6","objects":-63}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":484,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:15.176300+0000","last_change":"2023-09-23T15:12:42.908684+0000","last_active":"2023-09-23T15:16:15.176300+0000","last_peered":"2023-09-23T15:16:15.176300+0000","last_clean":"2023-09-23T15:12:40.545249+0000","last_became_active":"2023-09-23T15:12:42.908327+0000","last_became_peered":"2023-09-23T15:12:42.908327+0000","last_unstale":"2023-09-23T15:16:15.176300+0000","last_undegraded":"2023-09-23T15:12:42.848945+0000","last_fullsized":"2023-09-23T15:12:42.848396+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6],"avail_no_missing":["0","6","7"],"object_location_counts":[{"shards":"0,6","objects":-1},{"shards":"0,6,7","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":90,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.190048+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:16:15.190048+0000","last_peered":"2023-09-23T15:16:15.190048+0000","last_clean":"2023-09-23T15:16:15.190048+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:16:15.190048+0000","last_undegraded":"2023-09-23T15:16:15.190048+0000","last_fullsized":"2023-09-23T15:16:15.190048+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":208,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.188927+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:16:15.188927+0000","last_peered":"2023-09-23T15:16:15.188927+0000","last_clean":"2023-09-23T15:16:15.188927+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:16:15.188927+0000","last_undegraded":"2023-09-23T15:16:15.188927+0000","last_fullsized":"2023-09-23T15:16:15.188927+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"215'12442","reported_seq":12498,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.902760+0000","last_change":"2023-09-23T15:11:42.383451+0000","last_active":"2023-09-23T15:16:25.902760+0000","last_peered":"2023-09-23T15:16:25.902760+0000","last_clean":"2023-09-23T15:16:25.902760+0000","last_became_active":"2023-09-23T15:11:42.383047+0000","last_became_peered":"2023-09-23T15:11:42.383047+0000","last_unstale":"2023-09-23T15:16:25.902760+0000","last_undegraded":"2023-09-23T15:16:25.902760+0000","last_fullsized":"2023-09-23T15:16:25.902760+0000","mapping_epoch":171,"log_start":"178'3300","ondisk_log_start":"178'3300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9142,"log_dups_size":0,"ondisk_log_size":9142,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:05.140856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12442,"num_write_kb":398144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176637+0000","last_change":"2023-09-23T15:12:45.085282+0000","last_active":"2023-09-23T15:16:15.176637+0000","last_peered":"2023-09-23T15:16:15.176637+0000","last_clean":"2023-09-23T15:16:15.176637+0000","last_became_active":"2023-09-23T15:12:45.084840+0000","last_became_peered":"2023-09-23T15:12:45.084840+0000","last_unstale":"2023-09-23T15:16:15.176637+0000","last_undegraded":"2023-09-23T15:16:15.176637+0000","last_fullsized":"2023-09-23T15:16:15.176637+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":76,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183179+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:16:15.183179+0000","last_peered":"2023-09-23T15:16:15.183179+0000","last_clean":"2023-09-23T15:16:15.183179+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:16:15.183179+0000","last_undegraded":"2023-09-23T15:16:15.183179+0000","last_fullsized":"2023-09-23T15:16:15.183179+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":204,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198942+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:16:15.198942+0000","last_peered":"2023-09-23T15:16:15.198942+0000","last_clean":"2023-09-23T15:16:15.198942+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:16:15.198942+0000","last_undegraded":"2023-09-23T15:16:15.198942+0000","last_fullsized":"2023-09-23T15:16:15.198942+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"215'12664","reported_seq":12720,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.866800+0000","last_change":"2023-09-23T15:11:42.378532+0000","last_active":"2023-09-23T15:16:25.866800+0000","last_peered":"2023-09-23T15:16:25.866800+0000","last_clean":"2023-09-23T15:16:25.866800+0000","last_became_active":"2023-09-23T15:11:42.378131+0000","last_became_peered":"2023-09-23T15:11:42.378131+0000","last_unstale":"2023-09-23T15:16:25.866800+0000","last_undegraded":"2023-09-23T15:16:25.866800+0000","last_fullsized":"2023-09-23T15:16:25.866800+0000","mapping_epoch":171,"log_start":"199'7100","ondisk_log_start":"199'7100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5564,"log_dups_size":0,"ondisk_log_size":5564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:11.587231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12664,"num_write_kb":405248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179110+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:16:15.179110+0000","last_peered":"2023-09-23T15:16:15.179110+0000","last_clean":"2023-09-23T15:16:15.179110+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:16:15.179110+0000","last_undegraded":"2023-09-23T15:16:15.179110+0000","last_fullsized":"2023-09-23T15:16:15.179110+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":175,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183382+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:15.183382+0000","last_peered":"2023-09-23T15:16:15.183382+0000","last_clean":"2023-09-23T15:16:15.183382+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:16:15.183382+0000","last_undegraded":"2023-09-23T15:16:15.183382+0000","last_fullsized":"2023-09-23T15:16:15.183382+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":176,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175939+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:16:15.175939+0000","last_peered":"2023-09-23T15:16:15.175939+0000","last_clean":"2023-09-23T15:16:15.175939+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:16:15.175939+0000","last_undegraded":"2023-09-23T15:16:15.175939+0000","last_fullsized":"2023-09-23T15:16:15.175939+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"215'12912","reported_seq":12971,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:25.889035+0000","last_change":"2023-09-23T15:12:43.197879+0000","last_active":"2023-09-23T15:16:25.889035+0000","last_peered":"2023-09-23T15:16:25.889035+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.188286+0000","last_became_peered":"2023-09-23T15:12:43.188286+0000","last_unstale":"2023-09-23T15:16:25.889035+0000","last_undegraded":"2023-09-23T15:12:42.882454+0000","last_fullsized":"2023-09-23T15:12:42.847316+0000","mapping_epoch":184,"log_start":"189'5400","ondisk_log_start":"189'5400","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7512,"log_dups_size":0,"ondisk_log_size":7512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:00.632874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5010,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12912,"num_write_kb":413184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":-5010}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176245+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:16:15.176245+0000","last_peered":"2023-09-23T15:16:15.176245+0000","last_clean":"2023-09-23T15:16:15.176245+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:16:15.176245+0000","last_undegraded":"2023-09-23T15:16:15.176245+0000","last_fullsized":"2023-09-23T15:16:15.176245+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":179,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198412+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:16:15.198412+0000","last_peered":"2023-09-23T15:16:15.198412+0000","last_clean":"2023-09-23T15:16:15.198412+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:16:15.198412+0000","last_undegraded":"2023-09-23T15:16:15.198412+0000","last_fullsized":"2023-09-23T15:16:15.198412+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":174,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189276+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:16:15.189276+0000","last_peered":"2023-09-23T15:16:15.189276+0000","last_clean":"2023-09-23T15:16:15.189276+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:16:15.189276+0000","last_undegraded":"2023-09-23T15:16:15.189276+0000","last_fullsized":"2023-09-23T15:16:15.189276+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"215'12578","reported_seq":12634,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.881804+0000","last_change":"2023-09-23T15:11:42.385366+0000","last_active":"2023-09-23T15:16:25.881804+0000","last_peered":"2023-09-23T15:16:25.881804+0000","last_clean":"2023-09-23T15:16:25.881804+0000","last_became_active":"2023-09-23T15:11:42.385092+0000","last_became_peered":"2023-09-23T15:11:42.385092+0000","last_unstale":"2023-09-23T15:16:25.881804+0000","last_undegraded":"2023-09-23T15:16:25.881804+0000","last_fullsized":"2023-09-23T15:16:25.881804+0000","mapping_epoch":171,"log_start":"178'3400","ondisk_log_start":"178'3400","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9178,"log_dups_size":0,"ondisk_log_size":9178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:22.250967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12578,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":510,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175579+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:16:15.175579+0000","last_peered":"2023-09-23T15:16:15.175579+0000","last_clean":"2023-09-23T15:16:15.175579+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:16:15.175579+0000","last_undegraded":"2023-09-23T15:16:15.175579+0000","last_fullsized":"2023-09-23T15:16:15.175579+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":191,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198814+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:16:15.198814+0000","last_peered":"2023-09-23T15:16:15.198814+0000","last_clean":"2023-09-23T15:16:15.198814+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:16:15.198814+0000","last_undegraded":"2023-09-23T15:16:15.198814+0000","last_fullsized":"2023-09-23T15:16:15.198814+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":208,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189121+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:16:15.189121+0000","last_peered":"2023-09-23T15:16:15.189121+0000","last_clean":"2023-09-23T15:16:15.189121+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:16:15.189121+0000","last_undegraded":"2023-09-23T15:16:15.189121+0000","last_fullsized":"2023-09-23T15:16:15.189121+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"215'12578","reported_seq":12634,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.900016+0000","last_change":"2023-09-23T15:11:42.375566+0000","last_active":"2023-09-23T15:16:25.900016+0000","last_peered":"2023-09-23T15:16:25.900016+0000","last_clean":"2023-09-23T15:16:25.900016+0000","last_became_active":"2023-09-23T15:11:42.375246+0000","last_became_peered":"2023-09-23T15:11:42.375246+0000","last_unstale":"2023-09-23T15:16:25.900016+0000","last_undegraded":"2023-09-23T15:16:25.900016+0000","last_fullsized":"2023-09-23T15:16:25.900016+0000","mapping_epoch":171,"log_start":"194'6100","ondisk_log_start":"194'6100","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:26.431981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12578,"num_write_kb":402496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175852+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:16:15.175852+0000","last_peered":"2023-09-23T15:16:15.175852+0000","last_clean":"2023-09-23T15:16:15.175852+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:16:15.175852+0000","last_undegraded":"2023-09-23T15:16:15.175852+0000","last_fullsized":"2023-09-23T15:16:15.175852+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":199,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.188703+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:16:15.188703+0000","last_peered":"2023-09-23T15:16:15.188703+0000","last_clean":"2023-09-23T15:16:15.188703+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:16:15.188703+0000","last_undegraded":"2023-09-23T15:16:15.188703+0000","last_fullsized":"2023-09-23T15:16:15.188703+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":191,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.190144+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:16:15.190144+0000","last_peered":"2023-09-23T15:16:15.190144+0000","last_clean":"2023-09-23T15:16:15.190144+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:16:15.190144+0000","last_undegraded":"2023-09-23T15:16:15.190144+0000","last_fullsized":"2023-09-23T15:16:15.190144+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"215'12532","reported_seq":22406,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.911458+0000","last_change":"2023-09-23T15:14:18.960738+0000","last_active":"2023-09-23T15:16:25.911458+0000","last_peered":"2023-09-23T15:16:25.911458+0000","last_clean":"2023-09-23T15:16:25.911458+0000","last_became_active":"2023-09-23T15:14:18.912854+0000","last_became_peered":"2023-09-23T15:14:18.912854+0000","last_unstale":"2023-09-23T15:16:25.911458+0000","last_undegraded":"2023-09-23T15:16:25.911458+0000","last_fullsized":"2023-09-23T15:16:25.911458+0000","mapping_epoch":203,"log_start":"177'2500","ondisk_log_start":"177'2500","created":171,"last_epoch_clean":204,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10032,"log_dups_size":0,"ondisk_log_size":10032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:20:27.662749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12532,"num_write_kb":400961,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4900,"num_bytes_recovered":321126400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176782+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:16:15.176782+0000","last_peered":"2023-09-23T15:16:15.176782+0000","last_clean":"2023-09-23T15:16:15.176782+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:16:15.176782+0000","last_undegraded":"2023-09-23T15:16:15.176782+0000","last_fullsized":"2023-09-23T15:16:15.176782+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":208,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.197941+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:16:15.197941+0000","last_peered":"2023-09-23T15:16:15.197941+0000","last_clean":"2023-09-23T15:16:15.197941+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:16:15.197941+0000","last_undegraded":"2023-09-23T15:16:15.197941+0000","last_fullsized":"2023-09-23T15:16:15.197941+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":95,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.152115+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:16:30.152115+0000","last_peered":"2023-09-23T15:16:30.152115+0000","last_clean":"2023-09-23T15:16:30.152115+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:16:30.152115+0000","last_undegraded":"2023-09-23T15:16:30.152115+0000","last_fullsized":"2023-09-23T15:16:30.152115+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.0","version":"215'12568","reported_seq":12624,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.842577+0000","last_change":"2023-09-23T15:11:42.376957+0000","last_active":"2023-09-23T15:16:25.842577+0000","last_peered":"2023-09-23T15:16:25.842577+0000","last_clean":"2023-09-23T15:16:25.842577+0000","last_became_active":"2023-09-23T15:11:42.376695+0000","last_became_peered":"2023-09-23T15:11:42.376695+0000","last_unstale":"2023-09-23T15:16:25.842577+0000","last_undegraded":"2023-09-23T15:16:25.842577+0000","last_fullsized":"2023-09-23T15:16:25.842577+0000","mapping_epoch":171,"log_start":"198'7000","ondisk_log_start":"198'7000","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5568,"log_dups_size":0,"ondisk_log_size":5568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:03.122265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12568,"num_write_kb":402176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176725+0000","last_change":"2023-09-23T15:12:45.085860+0000","last_active":"2023-09-23T15:16:15.176725+0000","last_peered":"2023-09-23T15:16:15.176725+0000","last_clean":"2023-09-23T15:16:15.176725+0000","last_became_active":"2023-09-23T15:12:45.085526+0000","last_became_peered":"2023-09-23T15:12:45.085526+0000","last_unstale":"2023-09-23T15:16:15.176725+0000","last_undegraded":"2023-09-23T15:16:15.176725+0000","last_fullsized":"2023-09-23T15:16:15.176725+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":394,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151908+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:16:30.151908+0000","last_peered":"2023-09-23T15:16:30.151908+0000","last_clean":"2023-09-23T15:16:30.151908+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:16:30.151908+0000","last_undegraded":"2023-09-23T15:16:30.151908+0000","last_fullsized":"2023-09-23T15:16:30.151908+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":192,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183093+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:15.183093+0000","last_peered":"2023-09-23T15:16:15.183093+0000","last_clean":"2023-09-23T15:16:15.183093+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:16:15.183093+0000","last_undegraded":"2023-09-23T15:16:15.183093+0000","last_fullsized":"2023-09-23T15:16:15.183093+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":138,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.202560+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:16:15.202560+0000","last_peered":"2023-09-23T15:16:15.202560+0000","last_clean":"2023-09-23T15:16:15.202560+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:16:15.202560+0000","last_undegraded":"2023-09-23T15:16:15.202560+0000","last_fullsized":"2023-09-23T15:16:15.202560+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.6","version":"215'12354","reported_seq":12410,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.899934+0000","last_change":"2023-09-23T15:11:42.376005+0000","last_active":"2023-09-23T15:16:25.899934+0000","last_peered":"2023-09-23T15:16:25.899934+0000","last_clean":"2023-09-23T15:16:25.899934+0000","last_became_active":"2023-09-23T15:11:42.375702+0000","last_became_peered":"2023-09-23T15:11:42.375702+0000","last_unstale":"2023-09-23T15:16:25.899934+0000","last_undegraded":"2023-09-23T15:16:25.899934+0000","last_fullsized":"2023-09-23T15:16:25.899934+0000","mapping_epoch":171,"log_start":"193'5900","ondisk_log_start":"193'5900","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6454,"log_dups_size":0,"ondisk_log_size":6454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:28:42.714128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12354,"num_write_kb":395328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176376+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:16:15.176376+0000","last_peered":"2023-09-23T15:16:15.176376+0000","last_clean":"2023-09-23T15:16:15.176376+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:16:15.176376+0000","last_undegraded":"2023-09-23T15:16:15.176376+0000","last_fullsized":"2023-09-23T15:16:15.176376+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":734,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.182952+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:16:15.182952+0000","last_peered":"2023-09-23T15:16:15.182952+0000","last_clean":"2023-09-23T15:16:15.182952+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:16:15.182952+0000","last_undegraded":"2023-09-23T15:16:15.182952+0000","last_fullsized":"2023-09-23T15:16:15.182952+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":175,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189166+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:16:15.189166+0000","last_peered":"2023-09-23T15:16:15.189166+0000","last_clean":"2023-09-23T15:16:15.189166+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:16:15.189166+0000","last_undegraded":"2023-09-23T15:16:15.189166+0000","last_fullsized":"2023-09-23T15:16:15.189166+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151664+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:16:30.151664+0000","last_peered":"2023-09-23T15:16:30.151664+0000","last_clean":"2023-09-23T15:16:30.151664+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:16:30.151664+0000","last_undegraded":"2023-09-23T15:16:30.151664+0000","last_fullsized":"2023-09-23T15:16:30.151664+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.5","version":"215'12626","reported_seq":12679,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.843288+0000","last_change":"2023-09-23T15:11:42.383792+0000","last_active":"2023-09-23T15:16:25.843288+0000","last_peered":"2023-09-23T15:16:25.843288+0000","last_clean":"2023-09-23T15:16:25.843288+0000","last_became_active":"2023-09-23T15:11:42.383358+0000","last_became_peered":"2023-09-23T15:11:42.383358+0000","last_unstale":"2023-09-23T15:16:25.843288+0000","last_undegraded":"2023-09-23T15:16:25.843288+0000","last_fullsized":"2023-09-23T15:16:25.843288+0000","mapping_epoch":171,"log_start":"200'7300","ondisk_log_start":"200'7300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5326,"log_dups_size":0,"ondisk_log_size":5326,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:43.136315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12626,"num_write_kb":404032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176636+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:16:15.176636+0000","last_peered":"2023-09-23T15:16:15.176636+0000","last_clean":"2023-09-23T15:16:15.176636+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:16:15.176636+0000","last_undegraded":"2023-09-23T15:16:15.176636+0000","last_fullsized":"2023-09-23T15:16:15.176636+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":829,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.203403+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:16:15.203403+0000","last_peered":"2023-09-23T15:16:15.203403+0000","last_clean":"2023-09-23T15:16:15.203403+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:16:15.203403+0000","last_undegraded":"2023-09-23T15:16:15.203403+0000","last_fullsized":"2023-09-23T15:16:15.203403+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":90,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151306+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:16:30.151306+0000","last_peered":"2023-09-23T15:16:30.151306+0000","last_clean":"2023-09-23T15:16:30.151306+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:16:30.151306+0000","last_undegraded":"2023-09-23T15:16:30.151306+0000","last_fullsized":"2023-09-23T15:16:30.151306+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.152041+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:16:30.152041+0000","last_peered":"2023-09-23T15:16:30.152041+0000","last_clean":"2023-09-23T15:16:30.152041+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:16:30.152041+0000","last_undegraded":"2023-09-23T15:16:30.152041+0000","last_fullsized":"2023-09-23T15:16:30.152041+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"6.7","version":"215'12380","reported_seq":12433,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.880004+0000","last_change":"2023-09-23T15:11:42.376809+0000","last_active":"2023-09-23T15:16:25.880004+0000","last_peered":"2023-09-23T15:16:25.880004+0000","last_clean":"2023-09-23T15:16:25.880004+0000","last_became_active":"2023-09-23T15:11:42.376506+0000","last_became_peered":"2023-09-23T15:11:42.376506+0000","last_unstale":"2023-09-23T15:16:25.880004+0000","last_undegraded":"2023-09-23T15:16:25.880004+0000","last_fullsized":"2023-09-23T15:16:25.880004+0000","mapping_epoch":171,"log_start":"176'2300","ondisk_log_start":"176'2300","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:59.723853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12380,"num_write_kb":396160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":503,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.182891+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:16:15.182891+0000","last_peered":"2023-09-23T15:16:15.182891+0000","last_clean":"2023-09-23T15:16:15.182891+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:16:15.182891+0000","last_undegraded":"2023-09-23T15:16:15.182891+0000","last_fullsized":"2023-09-23T15:16:15.182891+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":505,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175673+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:16:15.175673+0000","last_peered":"2023-09-23T15:16:15.175673+0000","last_clean":"2023-09-23T15:16:15.175673+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:16:15.175673+0000","last_undegraded":"2023-09-23T15:16:15.175673+0000","last_fullsized":"2023-09-23T15:16:15.175673+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":130,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199043+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:16:15.199043+0000","last_peered":"2023-09-23T15:16:15.199043+0000","last_clean":"2023-09-23T15:16:15.199043+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:16:15.199043+0000","last_undegraded":"2023-09-23T15:16:15.199043+0000","last_fullsized":"2023-09-23T15:16:15.199043+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":166,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199329+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:16:15.199329+0000","last_peered":"2023-09-23T15:16:15.199329+0000","last_clean":"2023-09-23T15:16:15.199329+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:16:15.199329+0000","last_undegraded":"2023-09-23T15:16:15.199329+0000","last_fullsized":"2023-09-23T15:16:15.199329+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"215'12644","reported_seq":12700,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:25.900153+0000","last_change":"2023-09-23T15:12:43.210582+0000","last_active":"2023-09-23T15:16:25.900153+0000","last_peered":"2023-09-23T15:16:25.900153+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.195338+0000","last_became_peered":"2023-09-23T15:12:43.195338+0000","last_unstale":"2023-09-23T15:16:25.900153+0000","last_undegraded":"2023-09-23T15:12:42.886003+0000","last_fullsized":"2023-09-23T15:12:42.846591+0000","mapping_epoch":184,"log_start":"195'6200","ondisk_log_start":"195'6200","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":6444,"log_dups_size":0,"ondisk_log_size":6444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:28.633892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4952,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12644,"num_write_kb":404608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[{"shards":"3,5","objects":-4952}],"blocked_by":[],"up_primary":1,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.175898+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:16:15.175898+0000","last_peered":"2023-09-23T15:16:15.175898+0000","last_clean":"2023-09-23T15:16:15.175898+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:16:15.175898+0000","last_undegraded":"2023-09-23T15:16:15.175898+0000","last_fullsized":"2023-09-23T15:16:15.175898+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":301,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151171+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:16:30.151171+0000","last_peered":"2023-09-23T15:16:30.151171+0000","last_clean":"2023-09-23T15:16:30.151171+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:16:30.151171+0000","last_undegraded":"2023-09-23T15:16:30.151171+0000","last_fullsized":"2023-09-23T15:16:30.151171+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176287+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:16:15.176287+0000","last_peered":"2023-09-23T15:16:15.176287+0000","last_clean":"2023-09-23T15:16:15.176287+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:16:15.176287+0000","last_undegraded":"2023-09-23T15:16:15.176287+0000","last_fullsized":"2023-09-23T15:16:15.176287+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":191,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179188+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:16:15.179188+0000","last_peered":"2023-09-23T15:16:15.179188+0000","last_clean":"2023-09-23T15:16:15.179188+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:16:15.179188+0000","last_undegraded":"2023-09-23T15:16:15.179188+0000","last_fullsized":"2023-09-23T15:16:15.179188+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"215'12508","reported_seq":12563,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:25.903814+0000","last_change":"2023-09-23T15:12:43.195975+0000","last_active":"2023-09-23T15:16:25.903814+0000","last_peered":"2023-09-23T15:16:25.903814+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.186580+0000","last_became_peered":"2023-09-23T15:12:43.186580+0000","last_unstale":"2023-09-23T15:16:25.903814+0000","last_undegraded":"2023-09-23T15:12:42.892180+0000","last_fullsized":"2023-09-23T15:12:42.847045+0000","mapping_epoch":184,"log_start":"178'3400","ondisk_log_start":"178'3400","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":9108,"log_dups_size":0,"ondisk_log_size":9108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:07.010431+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4874,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12508,"num_write_kb":400256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[2,6],"avail_no_missing":["2","6"],"object_location_counts":[{"shards":"2,6","objects":-4874}],"blocked_by":[],"up_primary":1,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179488+0000","last_change":"2023-09-23T15:12:45.088691+0000","last_active":"2023-09-23T15:16:15.179488+0000","last_peered":"2023-09-23T15:16:15.179488+0000","last_clean":"2023-09-23T15:16:15.179488+0000","last_became_active":"2023-09-23T15:12:45.088371+0000","last_became_peered":"2023-09-23T15:12:45.088371+0000","last_unstale":"2023-09-23T15:16:15.179488+0000","last_undegraded":"2023-09-23T15:16:15.179488+0000","last_fullsized":"2023-09-23T15:16:15.179488+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":822,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:15.175038+0000","last_change":"2023-09-23T15:12:43.230434+0000","last_active":"2023-09-23T15:16:15.175038+0000","last_peered":"2023-09-23T15:16:15.175038+0000","last_clean":"2023-09-23T15:12:40.548041+0000","last_became_active":"2023-09-23T15:12:42.887879+0000","last_became_peered":"2023-09-23T15:12:42.887879+0000","last_unstale":"2023-09-23T15:16:15.175038+0000","last_undegraded":"2023-09-23T15:12:43.230249+0000","last_fullsized":"2023-09-23T15:12:42.844878+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":168,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:42.306003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":90,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.151829+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:16:30.151829+0000","last_peered":"2023-09-23T15:16:30.151829+0000","last_clean":"2023-09-23T15:16:30.151829+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:16:30.151829+0000","last_undegraded":"2023-09-23T15:16:30.151829+0000","last_fullsized":"2023-09-23T15:16:30.151829+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":166,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176062+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:16:15.176062+0000","last_peered":"2023-09-23T15:16:15.176062+0000","last_clean":"2023-09-23T15:16:15.176062+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:16:15.176062+0000","last_undegraded":"2023-09-23T15:16:15.176062+0000","last_fullsized":"2023-09-23T15:16:15.176062+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"215'12302","reported_seq":12355,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:25.889933+0000","last_change":"2023-09-23T15:11:42.378237+0000","last_active":"2023-09-23T15:16:25.889933+0000","last_peered":"2023-09-23T15:16:25.889933+0000","last_clean":"2023-09-23T15:16:25.889933+0000","last_became_active":"2023-09-23T15:11:42.378021+0000","last_became_peered":"2023-09-23T15:11:42.378021+0000","last_unstale":"2023-09-23T15:16:25.889933+0000","last_undegraded":"2023-09-23T15:16:25.889933+0000","last_fullsized":"2023-09-23T15:16:25.889933+0000","mapping_epoch":171,"log_start":"185'4800","ondisk_log_start":"185'4800","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":7502,"log_dups_size":0,"ondisk_log_size":7502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:37:38.124429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12302,"num_write_kb":393664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176586+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:16:15.176586+0000","last_peered":"2023-09-23T15:16:15.176586+0000","last_clean":"2023-09-23T15:16:15.176586+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:16:15.176586+0000","last_undegraded":"2023-09-23T15:16:15.176586+0000","last_fullsized":"2023-09-23T15:16:15.176586+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":192,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199646+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:16:15.199646+0000","last_peered":"2023-09-23T15:16:15.199646+0000","last_clean":"2023-09-23T15:16:15.199646+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:16:15.199646+0000","last_undegraded":"2023-09-23T15:16:15.199646+0000","last_fullsized":"2023-09-23T15:16:15.199646+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":678,"reported_epoch":217,"state":"active+clean","last_fresh":"2023-09-23T15:16:30.152054+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:16:30.152054+0000","last_peered":"2023-09-23T15:16:30.152054+0000","last_clean":"2023-09-23T15:16:30.152054+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:16:30.152054+0000","last_undegraded":"2023-09-23T15:16:30.152054+0000","last_fullsized":"2023-09-23T15:16:30.152054+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":197,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.198216+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:16:15.198216+0000","last_peered":"2023-09-23T15:16:15.198216+0000","last_clean":"2023-09-23T15:16:15.198216+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:16:15.198216+0000","last_undegraded":"2023-09-23T15:16:15.198216+0000","last_fullsized":"2023-09-23T15:16:15.198216+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"215'12630","reported_seq":12690,"reported_epoch":215,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:16:25.854282+0000","last_change":"2023-09-23T15:12:43.177601+0000","last_active":"2023-09-23T15:16:25.854282+0000","last_peered":"2023-09-23T15:16:25.854282+0000","last_clean":"2023-09-23T15:11:41.347017+0000","last_became_active":"2023-09-23T15:12:43.166546+0000","last_became_peered":"2023-09-23T15:12:43.166546+0000","last_unstale":"2023-09-23T15:16:25.854282+0000","last_undegraded":"2023-09-23T15:12:42.883114+0000","last_fullsized":"2023-09-23T15:12:42.851081+0000","mapping_epoch":184,"log_start":"200'7300","ondisk_log_start":"200'7300","created":171,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:11:41.347017+0000","last_clean_scrub_stamp":"2023-09-23T15:11:41.347017+0000","objects_scrubbed":0,"log_size":5330,"log_dups_size":0,"ondisk_log_size":5330,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:41.951577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12630,"num_write_kb":404160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":-4966}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":493,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176493+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:16:15.176493+0000","last_peered":"2023-09-23T15:16:15.176493+0000","last_clean":"2023-09-23T15:16:15.176493+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:16:15.176493+0000","last_undegraded":"2023-09-23T15:16:15.176493+0000","last_fullsized":"2023-09-23T15:16:15.176493+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":692,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199299+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:16:15.199299+0000","last_peered":"2023-09-23T15:16:15.199299+0000","last_clean":"2023-09-23T15:16:15.199299+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:16:15.199299+0000","last_undegraded":"2023-09-23T15:16:15.199299+0000","last_fullsized":"2023-09-23T15:16:15.199299+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":179,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.179265+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:16:15.179265+0000","last_peered":"2023-09-23T15:16:15.179265+0000","last_clean":"2023-09-23T15:16:15.179265+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:16:15.179265+0000","last_undegraded":"2023-09-23T15:16:15.179265+0000","last_fullsized":"2023-09-23T15:16:15.179265+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":157,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.188871+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:16:15.188871+0000","last_peered":"2023-09-23T15:16:15.188871+0000","last_clean":"2023-09-23T15:16:15.188871+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:16:15.188871+0000","last_undegraded":"2023-09-23T15:16:15.188871+0000","last_fullsized":"2023-09-23T15:16:15.188871+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":168,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176022+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:16:15.176022+0000","last_peered":"2023-09-23T15:16:15.176022+0000","last_clean":"2023-09-23T15:16:15.176022+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:16:15.176022+0000","last_undegraded":"2023-09-23T15:16:15.176022+0000","last_fullsized":"2023-09-23T15:16:15.176022+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":216,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.202890+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:16:15.202890+0000","last_peered":"2023-09-23T15:16:15.202890+0000","last_clean":"2023-09-23T15:16:15.202890+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:16:15.202890+0000","last_undegraded":"2023-09-23T15:16:15.202890+0000","last_fullsized":"2023-09-23T15:16:15.202890+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":194,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183486+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:16:15.183486+0000","last_peered":"2023-09-23T15:16:15.183486+0000","last_clean":"2023-09-23T15:16:15.183486+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:16:15.183486+0000","last_undegraded":"2023-09-23T15:16:15.183486+0000","last_fullsized":"2023-09-23T15:16:15.183486+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189396+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:16:15.189396+0000","last_peered":"2023-09-23T15:16:15.189396+0000","last_clean":"2023-09-23T15:16:15.189396+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:16:15.189396+0000","last_undegraded":"2023-09-23T15:16:15.189396+0000","last_fullsized":"2023-09-23T15:16:15.189396+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":160,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.183059+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:16:15.183059+0000","last_peered":"2023-09-23T15:16:15.183059+0000","last_clean":"2023-09-23T15:16:15.183059+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:16:15.183059+0000","last_undegraded":"2023-09-23T15:16:15.183059+0000","last_fullsized":"2023-09-23T15:16:15.183059+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":191,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.189020+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:16:15.189020+0000","last_peered":"2023-09-23T15:16:15.189020+0000","last_clean":"2023-09-23T15:16:15.189020+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:16:15.189020+0000","last_undegraded":"2023-09-23T15:16:15.189020+0000","last_fullsized":"2023-09-23T15:16:15.189020+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":188,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.199954+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:16:15.199954+0000","last_peered":"2023-09-23T15:16:15.199954+0000","last_clean":"2023-09-23T15:16:15.199954+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:16:15.199954+0000","last_undegraded":"2023-09-23T15:16:15.199954+0000","last_fullsized":"2023-09-23T15:16:15.199954+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":202,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.196987+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:16:15.196987+0000","last_peered":"2023-09-23T15:16:15.196987+0000","last_clean":"2023-09-23T15:16:15.196987+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:16:15.196987+0000","last_undegraded":"2023-09-23T15:16:15.196987+0000","last_fullsized":"2023-09-23T15:16:15.196987+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":164,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.176995+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:16:15.176995+0000","last_peered":"2023-09-23T15:16:15.176995+0000","last_clean":"2023-09-23T15:16:15.176995+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:16:15.176995+0000","last_undegraded":"2023-09-23T15:16:15.176995+0000","last_fullsized":"2023-09-23T15:16:15.176995+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":184,"reported_epoch":215,"state":"active+clean","last_fresh":"2023-09-23T15:16:15.188706+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:16:15.188706+0000","last_peered":"2023-09-23T15:16:15.188706+0000","last_clean":"2023-09-23T15:16:15.188706+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:16:15.188706+0000","last_undegraded":"2023-09-23T15:16:15.188706+0000","last_fullsized":"2023-09-23T15:16:15.188706+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19865,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":200778,"num_write_kb":6424833,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9835,"num_bytes_recovered":476053504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":120278,"ondisk_log_size":120278,"up":48,"acting":43,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1148241,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":62,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201541,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463177,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":172292,"kb_used_data":2748,"kb_used_omap":0,"kb_used_meta":169536,"kb_avail":93572860,"statfs":{"total":95995035648,"available":95818608640,"internally_reserved":0,"allocated":2813952,"data_stored":2089501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":173604864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.1469999999999998,"15min":2.7120000000000002},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.2519999999999998,"5min":41.491,"15min":108.39400000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.4649999999999999,"15min":2.9289999999999998},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.1029999999999998,"5min":43.151000000000003,"15min":108.452},"last":0.38500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":3.395,"15min":3.379},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.266,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.45200000000000001},{"interface":"front","average":{"1min":3.2589999999999999,"5min":4.274,"15min":3.8290000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.4060000000000006,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.53000000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5129999999999999,"5min":5.7489999999999997,"15min":6.0949999999999998},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":68.828000000000003,"5min":112.352,"15min":180.374},"last":0.63600000000000001},{"interface":"front","average":{"1min":5.681,"5min":5.8140000000000001,"15min":6.0709999999999997},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":68.730999999999995,"5min":117.667,"15min":181.685},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":15.903,"5min":7.0270000000000001,"15min":3.012},"min":{"1min":0.627,"5min":0.249,"15min":0.249},"max":{"1min":102.254,"5min":102.254,"15min":102.254},"last":0.51100000000000001},{"interface":"front","average":{"1min":17.141999999999999,"5min":7.4619999999999997,"15min":3.0840000000000001},"min":{"1min":0.46100000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":98.941999999999993,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0349999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.7150000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.624,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0019999999999998,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.0099999999999998,"15min":1.423},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":3.1789999999999998,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.67700000000000005},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.0249999999999999,"15min":1.4590000000000001},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0510000000000002,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:15:36 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.2689999999999999,"15min":1.032},"min":{"1min":0.32600000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.125,"5min":7.5910000000000002,"15min":17},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.83799999999999997,"5min":1.1659999999999999,"15min":0.995},"min":{"1min":0.35199999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":2.524,"5min":7.7809999999999997,"15min":7.7809999999999997},"last":0.621}]}]},{"osd":1,"up_from":183,"seq":785979015215,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":181756,"kb_used_data":6900,"kb_used_omap":6,"kb_used_meta":174841,"kb_avail":93563396,"statfs":{"total":95995035648,"available":95808917504,"internally_reserved":0,"allocated":7065600,"data_stored":6017662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":179037259},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":3.4049999999999998,"15min":3.6709999999999998},"min":{"1min":0.40200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.338000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":5.6859999999999999,"5min":4.6150000000000002,"15min":4.6109999999999998},"min":{"1min":0.29599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.414999999999999,"5min":16.414999999999999,"15min":16.414999999999999},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":4.1429999999999998,"15min":4.133},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.315,"5min":13.843,"15min":13.843},"last":0.26700000000000002},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.0709999999999997,"15min":4.1719999999999997},"min":{"1min":0.434,"5min":0.253,"15min":0.253},"max":{"1min":16.364999999999998,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":8.3599999999999994,"15min":9.984},"min":{"1min":0.40899999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.617999999999999,"5min":123.426,"15min":123.426},"last":3.3450000000000002},{"interface":"front","average":{"1min":5.3920000000000003,"5min":9.3279999999999994,"15min":11.486000000000001},"min":{"1min":0.68100000000000005,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.451000000000001,"5min":123.32899999999999,"15min":123.32899999999999},"last":3.0310000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8409999999999993,"5min":9.8409999999999993,"15min":9.8409999999999993},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.823,"5min":3.3250000000000002,"15min":3.48},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.228000000000002,"5min":16.779,"15min":16.779},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":3.4340000000000002,"15min":3.6640000000000001},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.898999999999999,"5min":16.166,"15min":16.166},"last":0.53500000000000003},{"interface":"front","average":{"1min":3.5179999999999998,"5min":3.0449999999999999,"15min":2.9460000000000002},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8680000000000003,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":3.4209999999999998,"15min":3.6800000000000002},"min":{"1min":0.67500000000000004,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":14.867000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":0.61599999999999999},{"interface":"front","average":{"1min":3.7240000000000002,"5min":3,"15min":2.8319999999999999},"min":{"1min":0.438,"5min":0.377,"15min":0.377},"max":{"1min":12.448,"5min":12.448,"15min":12.448},"last":0.65800000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":3.1379999999999999,"15min":3.1499999999999999},"min":{"1min":0.47199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.56399999999999995},{"interface":"front","average":{"1min":3.5169999999999999,"5min":3.323,"15min":3.343},"min":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.280999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.63900000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893264,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324468,"kb_used_data":7084,"kb_used_omap":8,"kb_used_meta":317367,"kb_avail":93420684,"statfs":{"total":95995035648,"available":95662780416,"internally_reserved":0,"allocated":7254016,"data_stored":6315161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":324984404},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":3.298,"15min":3.96},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4109999999999996,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.9770000000000001,"5min":3.7280000000000002,"15min":4.3819999999999997},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5590000000000002,"5min":31.378,"15min":31.378},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":7.133,"15min":3.5539999999999998},"min":{"1min":0.26200000000000001,"5min":0.189,"15min":0.17000000000000001},"max":{"1min":13.593,"5min":103.81,"15min":122.20099999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":3.4279999999999999,"5min":6.077,"15min":3.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.18099999999999999},"max":{"1min":9.7479999999999993,"5min":108.19499999999999,"15min":122.071},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":8.2420000000000009,"5min":6.8710000000000004,"15min":2.8610000000000002},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20000000000000001},"max":{"1min":72.137,"5min":99.204999999999998,"15min":99.204999999999998},"last":0.46000000000000002},{"interface":"front","average":{"1min":13.818,"5min":6.9850000000000003,"15min":3.2029999999999998},"min":{"1min":0.32200000000000001,"5min":0.25700000000000001,"15min":0.252},"max":{"1min":95.795000000000002,"5min":95.795000000000002,"15min":95.795000000000002},"last":0.48999999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.5230000000000001,"15min":1.649},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.9950000000000001,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.4710000000000001,"5min":3.282,"15min":1.72},"min":{"1min":0.29699999999999999,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":13.566000000000001,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.992,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29699999999999999},"max":{"1min":9.6020000000000003,"5min":109.369,"15min":109.369},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.2170000000000001,"15min":1.857},"min":{"1min":0.42999999999999999,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":4.4219999999999997,"5min":109.387,"15min":109.387},"last":0.61099999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.996,"15min":1.752},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":13.529999999999999,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.59799999999999998},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.8650000000000002,"15min":1.677},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":9.6460000000000008,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.66500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":3.181,"15min":1.802},"min":{"1min":0.44600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.6690000000000005,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.702,"5min":2.2770000000000001,"15min":1.5549999999999999},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9720000000000004,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.69399999999999995}]}]},{"osd":2,"up_from":105,"seq":450971566211,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347444,"kb_used_data":2412,"kb_used_omap":11,"kb_used_meta":345012,"kb_avail":93397708,"statfs":{"total":95995035648,"available":95639252992,"internally_reserved":0,"allocated":2469888,"data_stored":1534452,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":353293072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":4.9969999999999999,"15min":2.2229999999999999},"min":{"1min":0.27400000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.085000000000001,"5min":87.132999999999996,"15min":87.132999999999996},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.931,"5min":3.8490000000000002,"15min":1.831},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.833,"5min":84.820999999999998,"15min":84.820999999999998},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":5.492,"15min":5.1900000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.058999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.30099999999999999},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.782,"15min":2.96},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.962999999999999,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.28499999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:15:29 2023","interfaces":[{"interface":"back","average":{"1min":14.097,"5min":6.1619999999999999,"15min":2.661},"min":{"1min":0.70399999999999996,"5min":0.219,"15min":0.219},"max":{"1min":86.332999999999998,"5min":87.108000000000004,"15min":87.108000000000004},"last":0.252},{"interface":"front","average":{"1min":12.151999999999999,"5min":5.577,"15min":2.5369999999999999},"min":{"1min":0.252,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.129,"15min":1.831},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.824,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":2.8849999999999998,"5min":4.016,"15min":2.4020000000000001},"min":{"1min":0.42399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.988,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":3.266,"15min":2.0579999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.193999999999999,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.8690000000000002,"15min":1.9179999999999999},"min":{"1min":0.441,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.865,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.51800000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.1200000000000001,"15min":2.0369999999999999},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.891999999999999,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.9729999999999999,"5min":3.5750000000000002,"15min":2.0939999999999999},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.949999999999999,"5min":87.055000000000007,"15min":87.055000000000007},"last":0.48499999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":3.3519999999999999,"15min":1.915},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":16.239000000000001,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.966,"5min":3.0630000000000002,"15min":1.8049999999999999},"min":{"1min":0.48899999999999999,"5min":0.38600000000000001,"15min":0.34200000000000003},"max":{"1min":10.916,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.56200000000000006}]}]},{"osd":3,"up_from":110,"seq":472446402685,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":333960,"kb_used_data":6464,"kb_used_omap":3,"kb_used_meta":327484,"kb_avail":93411192,"statfs":{"total":95995035648,"available":95653060608,"internally_reserved":0,"allocated":6619136,"data_stored":5718639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":335343785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":4.0780000000000003,"5min":3.3100000000000001,"15min":2.0110000000000001},"min":{"1min":0.20999999999999999,"5min":0.191,"15min":0.17899999999999999},"max":{"1min":13.064,"5min":61.744,"15min":79.611000000000004},"last":0.27100000000000002},{"interface":"front","average":{"1min":3.9380000000000002,"5min":3.2130000000000001,"15min":2.0249999999999999},"min":{"1min":0.40200000000000002,"5min":0.17599999999999999,"15min":0.17599999999999999},"max":{"1min":12.348000000000001,"5min":61.686,"15min":79.656000000000006},"last":0.34100000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:15:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":4.702,"15min":4.6239999999999997},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.566000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":0.30299999999999999},{"interface":"front","average":{"1min":3.5339999999999998,"5min":4.5609999999999999,"15min":4.7510000000000003},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.539000000000001,"5min":21.539000000000001,"15min":21.539000000000001},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.8600000000000003,"5min":4.6020000000000003,"15min":3.6909999999999998},"min":{"1min":0.432,"5min":0.22,"15min":0.22},"max":{"1min":61.732999999999997,"5min":63.009999999999998,"15min":109.077},"last":0.40500000000000003},{"interface":"front","average":{"1min":8.202,"5min":5.101,"15min":3.9460000000000002},"min":{"1min":0.47999999999999998,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":60.305999999999997,"5min":69.715999999999994,"15min":119.72799999999999},"last":0.49099999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.6139999999999999,"15min":1.4319999999999999},"min":{"1min":0.36199999999999999,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":54.051000000000002,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.77200000000000002},{"interface":"front","average":{"1min":5.2939999999999996,"5min":2.7029999999999998,"15min":1.6359999999999999},"min":{"1min":0.45400000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":52.228999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.72499999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.351,"5min":2.399,"15min":1.5740000000000001},"min":{"1min":0.40899999999999997,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":8.0459999999999994,"5min":8.9450000000000003,"15min":56.960999999999999},"last":0.81299999999999994},{"interface":"front","average":{"1min":2.7679999999999998,"5min":2.552,"15min":1.806},"min":{"1min":0.49299999999999999,"5min":0.315,"15min":0.315},"max":{"1min":5.8730000000000002,"5min":10.738,"15min":56.981999999999999},"last":0.69599999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":6.2629999999999999,"5min":3.1920000000000002,"15min":1.6990000000000001},"min":{"1min":0.42899999999999999,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":78.102000000000004,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.876},{"interface":"front","average":{"1min":6.5919999999999996,"5min":3.327,"15min":1.962},"min":{"1min":0.69299999999999995,"5min":0.371,"15min":0.32300000000000001},"max":{"1min":78.028000000000006,"5min":78.028000000000006,"15min":78.028000000000006},"last":0.85099999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:15:30 2023","interfaces":[{"interface":"back","average":{"1min":2.6779999999999999,"5min":2.5110000000000001,"15min":1.71},"min":{"1min":0.52200000000000002,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":5.923,"5min":10.603,"15min":14.885999999999999},"last":0.78800000000000003},{"interface":"front","average":{"1min":3.016,"5min":2.2160000000000002,"15min":1.363},"min":{"1min":0.376,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":13.109,"5min":14.351000000000001,"15min":14.351000000000001},"last":0.70899999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986450,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206400,"kb_used_data":2168,"kb_used_omap":0,"kb_used_meta":204224,"kb_avail":93538752,"statfs":{"total":95995035648,"available":95783682048,"internally_reserved":0,"allocated":2220032,"data_stored":1499156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":209125376},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":2.5830000000000002,"15min":2.6200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.8250000000000002,"5min":65.988,"15min":122.69199999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.552,"5min":2.5219999999999998,"15min":2.698},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":5.7149999999999999,"5min":65.396000000000001,"15min":122.745},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":3.6920000000000002,"15min":4.2629999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7539999999999996,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":2.8860000000000001,"5min":4.3369999999999997,"15min":4.8300000000000001},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.114000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":4.1260000000000003,"15min":3.1080000000000001},"min":{"1min":0.37,"5min":0.255,"15min":0.255},"max":{"1min":12.595000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.625},{"interface":"front","average":{"1min":3.2869999999999999,"5min":4.383,"15min":3.3530000000000002},"min":{"1min":0.34300000000000003,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":7.173,"5min":147.88900000000001,"15min":147.88900000000001},"last":0.69799999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":7.7069999999999999,"5min":9.3309999999999995,"15min":3.7869999999999999},"min":{"1min":0.36799999999999999,"5min":0.252,"15min":0.252},"max":{"1min":96.447000000000003,"5min":163.97499999999999,"15min":163.97499999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":7.0019999999999998,"5min":9.1080000000000005,"15min":3.5750000000000002},"min":{"1min":0.38300000000000001,"5min":0.315,"15min":0.27800000000000002},"max":{"1min":95.944999999999993,"5min":164.447,"15min":164.447},"last":0.57199999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:15:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.966,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":87.912999999999997,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.60199999999999998},{"interface":"front","average":{"1min":5.4960000000000004,"5min":1.9450000000000001,"15min":1.395},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":88.602999999999994,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.71599999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:16:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":3.6160000000000001,"15min":2.0510000000000002},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.3820000000000001,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.72899999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":3.7080000000000002,"15min":1.956},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.7749999999999999,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:15:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.91800000000000004,"15min":1.129},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.309},"max":{"1min":0.75900000000000001,"5min":7.7389999999999999,"15min":18.23},"last":0.745},{"interface":"front","average":{"1min":1.079,"5min":1.3400000000000001,"15min":1.179},"min":{"1min":0.375,"5min":0.314,"15min":0.314},"max":{"1min":6.1399999999999997,"5min":14.044,"15min":14.653},"last":0.67600000000000005}]}]},{"osd":5,"up_from":39,"seq":167503724813,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":207732,"kb_used_data":2156,"kb_used_omap":0,"kb_used_meta":205568,"kb_avail":93537420,"statfs":{"total":95995035648,"available":95782318080,"internally_reserved":0,"allocated":2207744,"data_stored":1494919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":210501632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.9430000000000001,"15min":1.4079999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":7.8319999999999999,"5min":16.981999999999999,"15min":24.962},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.972,"5min":2.1589999999999998,"15min":1.4550000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":12.531000000000001,"5min":18.814,"15min":25.317},"last":0.46400000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.3730000000000002,"15min":1.9059999999999999},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.6110000000000007,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":4.1109999999999998,"5min":2.7730000000000001,"15min":2.4369999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.423,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.41399999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.266,"15min":2.4609999999999999},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3889999999999993,"5min":52.414000000000001,"15min":97.087999999999994},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.5070000000000001,"5min":3.3570000000000002,"15min":2.4209999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":52.375999999999998,"15min":97.305999999999997},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":10.388,"5min":5.0510000000000002,"15min":2.488},"min":{"1min":0.54900000000000004,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":80.570999999999998,"5min":80.570999999999998,"15min":80.570999999999998},"last":0.64000000000000001},{"interface":"front","average":{"1min":9.8190000000000008,"5min":5.1440000000000001,"15min":2.3460000000000001},"min":{"1min":0.44400000000000001,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":80.551000000000002,"5min":80.551000000000002,"15min":80.551000000000002},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":8.7479999999999993,"5min":3.2589999999999999,"15min":1.718},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":138.68600000000001,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.67300000000000004},{"interface":"front","average":{"1min":8.7729999999999997,"5min":3.415,"15min":1.788},"min":{"1min":0.34300000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":138.49600000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.59499999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88900000000000001,"15min":1.1339999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.5550000000000002,"5min":5.8760000000000003,"15min":15.196},"last":0.72299999999999998},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.86499999999999999,"15min":1.0229999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9990000000000001,"5min":4.8819999999999997,"15min":14.965999999999999},"last":0.74299999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.3380000000000001,"15min":1.288},"min":{"1min":0.35199999999999998,"5min":0.33300000000000002,"15min":0.28599999999999998},"max":{"1min":8.2200000000000006,"5min":9.6560000000000006,"15min":15.301},"last":0.70299999999999996},{"interface":"front","average":{"1min":1.141,"5min":1.3700000000000001,"15min":1.252},"min":{"1min":0.39900000000000002,"5min":0.311,"15min":0.311},"max":{"1min":5.6559999999999997,"5min":15.762,"15min":15.762},"last":0.76000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:16:36.228 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:36.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:36.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:36.336 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:36.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:36 smithi017 ceph-mon[215646]: from='client.35505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:36.370 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:36 smithi017 ceph-mon[215646]: pgmap v613: 109 pgs: 102 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 19356/81 objects degraded (23896.296%); 0 B/s, 229 objects/s recovering 2023-09-23T15:16:36.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:36 smithi017 ceph-mon[222878]: from='client.35505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:36.370 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:36 smithi017 ceph-mon[222878]: pgmap v613: 109 pgs: 102 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 19356/81 objects degraded (23896.296%); 0 B/s, 229 objects/s recovering 2023-09-23T15:16:36.372 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:36.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:36 smithi027 ceph-mon[195233]: from='client.35505 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:36.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:36 smithi027 ceph-mon[195233]: pgmap v613: 109 pgs: 102 active+clean, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 91 op/s; 19356/81 objects degraded (23896.296%); 0 B/s, 229 objects/s recovering 2023-09-23T15:16:36.947 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:36.964 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_1' removed 2023-09-23T15:16:36.973 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:36.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:37.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:37.112 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:37.157 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:37.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:37.214 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:16:37.214 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 19356/81 objects degraded (23896.296%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:37.214 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:16:37.214 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[215646]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T15:16:37.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:16:37.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 19356/81 objects degraded (23896.296%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:37.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:16:37.215 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:37 smithi017 ceph-mon[222878]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T15:16:37.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:37.290 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:37.333 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:37.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:37.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:37.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:37.469 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:37.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:16:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 19356/81 objects degraded (23896.296%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:16:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3843194910' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:16:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:37 smithi027 ceph-mon[195233]: osdmap e219: 8 total, 8 up, 7 in 2023-09-23T15:16:37.634 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:37.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:37.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:37.735 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:16:37.736 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:16:37.736 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T15:16:37.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T15:16:37.737 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:37.766 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:37.866 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:37.917 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:37.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:37.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:38.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:38.078 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:38.234 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:38.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:38.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:38.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:38.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[215646]: pgmap v614: 109 pgs: 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 19293/81 objects degraded (23818.519%); 0 B/s, 234 objects/s recovering 2023-09-23T15:16:38.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[215646]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[215646]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[215646]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T15:16:38.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[215646]: failed to encode map e220 with expected crc 2023-09-23T15:16:38.364 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:38] "GET /metrics HTTP/1.1" 200 36321 "" "Prometheus/2.43.0" 2023-09-23T15:16:38.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[222878]: pgmap v614: 109 pgs: 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 19293/81 objects degraded (23818.519%); 0 B/s, 234 objects/s recovering 2023-09-23T15:16:38.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[222878]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[222878]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[222878]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T15:16:38.365 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:38 smithi017 ceph-mon[222878]: failed to encode map e220 with expected crc 2023-09-23T15:16:38.366 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:38.411 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:38.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:38.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:38.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:38.545 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:38 smithi027 ceph-mon[195233]: pgmap v614: 109 pgs: 103 active+clean, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 0 B/s wr, 90 op/s; 19293/81 objects degraded (23818.519%); 0 B/s, 234 objects/s recovering 2023-09-23T15:16:38.545 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:38 smithi027 ceph-mon[195233]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.545 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:38 smithi027 ceph-mon[195233]: failed to encode map e219 with expected crc 2023-09-23T15:16:38.545 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:38 smithi027 ceph-mon[195233]: osdmap e220: 8 total, 8 up, 7 in 2023-09-23T15:16:38.545 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:38 smithi027 ceph-mon[195233]: failed to encode map e220 with expected crc 2023-09-23T15:16:38.548 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:38.598 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:38.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:38.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:38.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:38.734 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:38.781 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:38.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:38.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:38.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:38.922 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:38.975 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:16:38.985 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:38.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:38.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:39.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:39.117 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:39.161 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:39.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:39.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:39.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:16:39.267 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:39.293 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:39.338 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:39.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:39.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:39.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1598484052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: pgmap v617: 93 pgs: 91 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3/81 objects degraded (3.704%) 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[215646]: failed to encode map e221 with expected crc 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1598484052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: pgmap v617: 93 pgs: 91 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3/81 objects degraded (3.704%) 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T15:16:39.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:39 smithi017 ceph-mon[222878]: failed to encode map e221 with expected crc 2023-09-23T15:16:39.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:39.473 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: failed to encode map e220 with expected crc 2023-09-23T15:16:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1598484052' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: pgmap v617: 93 pgs: 91 active+clean, 2 active+recovery_wait+undersized+degraded+remapped; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 3/81 objects degraded (3.704%) 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: osdmap e221: 8 total, 8 up, 7 in 2023-09-23T15:16:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:39 smithi027 ceph-mon[195233]: failed to encode map e221 with expected crc 2023-09-23T15:16:39.621 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:39.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:39.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:39.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:39.757 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:39.791 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:39.807 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:39.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:39.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:39.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:39.941 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:40.390 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:40.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[215646]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[215646]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[215646]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.393 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[222878]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.393 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[222878]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.393 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:40 smithi017 ceph-mon[222878]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:40.394 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:40.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:40.529 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:40.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:40 smithi027 ceph-mon[195233]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:40 smithi027 ceph-mon[195233]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:40 smithi027 ceph-mon[195233]: failed to encode map e221 with expected crc 2023-09-23T15:16:40.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:40.724 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:40.762 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:40.784 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:40.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:40.824 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:40.844 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:40.844 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:40.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:40.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:40.866 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.868 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:16:40.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:40.982 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:41.032 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:41.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:41.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:41.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:41.167 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:41.214 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:41.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:41.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:41.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:41.347 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:41 smithi027 ceph-mon[195233]: pgmap v619: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:41.347 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T15:16:41.347 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3086045578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:41.349 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:41.393 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:41.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:41.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[215646]: pgmap v619: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T15:16:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3086045578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[222878]: pgmap v619: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]: dispatch 2023-09-23T15:16:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3086045578' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:41.500 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:41.526 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:41.569 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:41.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:41.675 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:41.707 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:41.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:16:41.914 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:41.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:41.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:41.952 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:42.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:42.052 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:42.171 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:16:42.171 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:16:42.210 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:42.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:42.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:42.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:42.344 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:42.391 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:42.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/81 objects degraded (3.704%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T15:16:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: 1.4 scrub starts 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[215646]: 1.4 scrub ok 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/81 objects degraded (3.704%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: 1.4 scrub starts 2023-09-23T15:16:42.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:42 smithi017 ceph-mon[222878]: 1.4 scrub ok 2023-09-23T15:16:42.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/81 objects degraded (3.704%), 2 pgs degraded, 2 pgs undersized) 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: failed to encode map e222 with expected crc 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: osdmap e222: 8 total, 8 up, 7 in 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: 1.4 scrub starts 2023-09-23T15:16:42.522 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:42 smithi027 ceph-mon[195233]: 1.4 scrub ok 2023-09-23T15:16:42.524 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:42.567 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:42.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:42.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:42.672 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:42.700 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:42.743 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:42.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:42.749 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:42.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:42.877 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:42.920 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:42.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:42.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:43.028 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:43.054 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:43.098 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:43.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:43.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:43.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:43.230 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.232 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:43.250 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' created 2023-09-23T15:16:43.280 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:43.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:43.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:43.412 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:43.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: pgmap v621: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:43.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T15:16:43.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:43.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:43.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: pgmap v621: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:43.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T15:16:43.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:43.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:43.457 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:43.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:43.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:43.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:16:43.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:43.593 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: pgmap v621: 93 pgs: 93 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: failed to encode map e222 with expected crc 2023-09-23T15:16:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T15:16:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:43.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:16:44.106 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:44.114 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:44.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:44.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:44.288 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:44.314 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T15:16:44.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[222878]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:44 smithi017 ceph-mon[215646]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.449 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:44.529 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:44.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:44.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4051934171' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: osdmap e223: 8 total, 8 up, 7 in 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:44 smithi027 ceph-mon[195233]: failed to encode map e223 with expected crc 2023-09-23T15:16:44.624 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:44.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:44.669 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:44.720 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:44.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:44.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:44.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:44.859 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:44.916 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:44.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:44.920 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:44.970 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:44.972 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:16:45.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:45.050 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: pgmap v624: 109 pgs: 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 9 unknown, 91 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:45.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:45 smithi027 ceph-mon[195233]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:45.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: pgmap v624: 109 pgs: 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 9 unknown, 91 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[215646]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: osdmap e224: 8 total, 8 up, 7 in 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: failed to encode map e224 with expected crc 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: pgmap v624: 109 pgs: 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 9 unknown, 91 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T15:16:45.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:45 smithi017 ceph-mon[222878]: from='client.45446 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:16:46.106 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:46.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:46.210 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:46.242 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:46.299 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":624,"stamp":"2023-09-23T15:16:44.443741+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4757700,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6237,"num_read_kb":13668,"num_write":1028,"num_write_kb":13935,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":5675175,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4413,"ondisk_log_size":4413,"up":300,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":290,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":1805640,"kb_used_data":30672,"kb_used_omap":30,"kb_used_meta":1774881,"kb_avail":654410424,"statfs":{"total":671965249536,"available":670116274176,"internally_reserved":0,"allocated":31408128,"data_stored":25305740,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1817478744},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":67,"apply_latency_ms":67,"commit_latency_ns":67000000,"apply_latency_ns":67000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.477186"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":218,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246867+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:16:43.246867+0000","last_peered":"2023-09-23T15:16:43.246867+0000","last_clean":"2023-09-23T15:16:43.246867+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:16:43.246867+0000","last_undegraded":"2023-09-23T15:16:43.246867+0000","last_fullsized":"2023-09-23T15:16:43.246867+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":150,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.109364+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:16:38.109364+0000","last_peered":"2023-09-23T15:16:38.109364+0000","last_clean":"2023-09-23T15:16:38.109364+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:16:38.109364+0000","last_undegraded":"2023-09-23T15:16:38.109364+0000","last_fullsized":"2023-09-23T15:16:38.109364+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":98,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965185+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:16:37.965185+0000","last_peered":"2023-09-23T15:16:37.965185+0000","last_clean":"2023-09-23T15:16:37.965185+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:16:37.965185+0000","last_undegraded":"2023-09-23T15:16:37.965185+0000","last_fullsized":"2023-09-23T15:16:37.965185+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":187,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.968852+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:16:38.968852+0000","last_peered":"2023-09-23T15:16:38.968852+0000","last_clean":"2023-09-23T15:16:38.968852+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:16:38.968852+0000","last_undegraded":"2023-09-23T15:16:38.968852+0000","last_fullsized":"2023-09-23T15:16:38.968852+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":219,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108704+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:16:38.108704+0000","last_peered":"2023-09-23T15:16:38.108704+0000","last_clean":"2023-09-23T15:16:38.108704+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:16:38.108704+0000","last_undegraded":"2023-09-23T15:16:38.108704+0000","last_fullsized":"2023-09-23T15:16:38.108704+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T15:16:42.483767+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:16:42.483767+0000","last_peered":"2023-09-23T15:16:42.483767+0000","last_clean":"2023-09-23T15:16:42.483767+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:16:42.483767+0000","last_undegraded":"2023-09-23T15:16:42.483767+0000","last_fullsized":"2023-09-23T15:16:42.483767+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":165,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967638+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:16:37.967638+0000","last_peered":"2023-09-23T15:16:37.967638+0000","last_clean":"2023-09-23T15:16:37.967638+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:16:37.967638+0000","last_undegraded":"2023-09-23T15:16:37.967638+0000","last_fullsized":"2023-09-23T15:16:37.967638+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":201,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108751+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:16:38.108751+0000","last_peered":"2023-09-23T15:16:38.108751+0000","last_clean":"2023-09-23T15:16:38.108751+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:16:38.108751+0000","last_undegraded":"2023-09-23T15:16:38.108751+0000","last_fullsized":"2023-09-23T15:16:38.108751+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.968025+0000","last_change":"2023-09-23T15:12:45.086275+0000","last_active":"2023-09-23T15:16:37.968025+0000","last_peered":"2023-09-23T15:16:37.968025+0000","last_clean":"2023-09-23T15:16:37.968025+0000","last_became_active":"2023-09-23T15:12:45.085952+0000","last_became_peered":"2023-09-23T15:12:45.085952+0000","last_unstale":"2023-09-23T15:16:37.968025+0000","last_undegraded":"2023-09-23T15:16:37.968025+0000","last_fullsized":"2023-09-23T15:16:37.968025+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":211,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108870+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:16:38.108870+0000","last_peered":"2023-09-23T15:16:38.108870+0000","last_clean":"2023-09-23T15:16:38.108870+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:16:38.108870+0000","last_undegraded":"2023-09-23T15:16:38.108870+0000","last_fullsized":"2023-09-23T15:16:38.108870+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":144,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.109016+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:16:38.109016+0000","last_peered":"2023-09-23T15:16:38.109016+0000","last_clean":"2023-09-23T15:16:38.109016+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:16:38.109016+0000","last_undegraded":"2023-09-23T15:16:38.109016+0000","last_fullsized":"2023-09-23T15:16:38.109016+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":84,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.109320+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:16:38.109320+0000","last_peered":"2023-09-23T15:16:38.109320+0000","last_clean":"2023-09-23T15:16:38.109320+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:16:38.109320+0000","last_undegraded":"2023-09-23T15:16:38.109320+0000","last_fullsized":"2023-09-23T15:16:38.109320+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":93,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965915+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:16:37.965915+0000","last_peered":"2023-09-23T15:16:37.965915+0000","last_clean":"2023-09-23T15:16:37.965915+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:16:37.965915+0000","last_undegraded":"2023-09-23T15:16:37.965915+0000","last_fullsized":"2023-09-23T15:16:37.965915+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":208,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247128+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:16:43.247128+0000","last_peered":"2023-09-23T15:16:43.247128+0000","last_clean":"2023-09-23T15:16:43.247128+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:16:43.247128+0000","last_undegraded":"2023-09-23T15:16:43.247128+0000","last_fullsized":"2023-09-23T15:16:43.247128+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":199,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246954+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:16:43.246954+0000","last_peered":"2023-09-23T15:16:43.246954+0000","last_clean":"2023-09-23T15:16:43.246954+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:16:43.246954+0000","last_undegraded":"2023-09-23T15:16:43.246954+0000","last_fullsized":"2023-09-23T15:16:43.246954+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967913+0000","last_change":"2023-09-23T15:12:45.090138+0000","last_active":"2023-09-23T15:16:37.967913+0000","last_peered":"2023-09-23T15:16:37.967913+0000","last_clean":"2023-09-23T15:16:37.967913+0000","last_became_active":"2023-09-23T15:12:45.089586+0000","last_became_peered":"2023-09-23T15:12:45.089586+0000","last_unstale":"2023-09-23T15:16:37.967913+0000","last_undegraded":"2023-09-23T15:16:37.967913+0000","last_fullsized":"2023-09-23T15:16:37.967913+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:27.961636+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":86,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247002+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:16:43.247002+0000","last_peered":"2023-09-23T15:16:43.247002+0000","last_clean":"2023-09-23T15:16:43.247002+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:16:43.247002+0000","last_undegraded":"2023-09-23T15:16:43.247002+0000","last_fullsized":"2023-09-23T15:16:43.247002+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":212,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.109145+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:16:38.109145+0000","last_peered":"2023-09-23T15:16:38.109145+0000","last_clean":"2023-09-23T15:16:38.109145+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:16:38.109145+0000","last_undegraded":"2023-09-23T15:16:38.109145+0000","last_fullsized":"2023-09-23T15:16:38.109145+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.966899+0000","last_change":"2023-09-23T15:12:45.080668+0000","last_active":"2023-09-23T15:16:37.966899+0000","last_peered":"2023-09-23T15:16:37.966899+0000","last_clean":"2023-09-23T15:16:37.966899+0000","last_became_active":"2023-09-23T15:12:45.080248+0000","last_became_peered":"2023-09-23T15:12:45.080248+0000","last_unstale":"2023-09-23T15:16:37.966899+0000","last_undegraded":"2023-09-23T15:16:37.966899+0000","last_fullsized":"2023-09-23T15:16:37.966899+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:49:12.687051+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":186,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967048+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:16:37.967048+0000","last_peered":"2023-09-23T15:16:37.967048+0000","last_clean":"2023-09-23T15:16:37.967048+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:16:37.967048+0000","last_undegraded":"2023-09-23T15:16:37.967048+0000","last_fullsized":"2023-09-23T15:16:37.967048+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":189,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108919+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:16:38.108919+0000","last_peered":"2023-09-23T15:16:38.108919+0000","last_clean":"2023-09-23T15:16:38.108919+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:16:38.108919+0000","last_undegraded":"2023-09-23T15:16:38.108919+0000","last_fullsized":"2023-09-23T15:16:38.108919+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967183+0000","last_change":"2023-09-23T15:12:45.086365+0000","last_active":"2023-09-23T15:16:37.967183+0000","last_peered":"2023-09-23T15:16:37.967183+0000","last_clean":"2023-09-23T15:16:37.967183+0000","last_became_active":"2023-09-23T15:12:45.086057+0000","last_became_peered":"2023-09-23T15:12:45.086057+0000","last_unstale":"2023-09-23T15:16:37.967183+0000","last_undegraded":"2023-09-23T15:16:37.967183+0000","last_fullsized":"2023-09-23T15:16:37.967183+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246815+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:16:43.246815+0000","last_peered":"2023-09-23T15:16:43.246815+0000","last_clean":"2023-09-23T15:16:43.246815+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:16:43.246815+0000","last_undegraded":"2023-09-23T15:16:43.246815+0000","last_fullsized":"2023-09-23T15:16:43.246815+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":174,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967951+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:16:37.967951+0000","last_peered":"2023-09-23T15:16:37.967951+0000","last_clean":"2023-09-23T15:16:37.967951+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:16:37.967951+0000","last_undegraded":"2023-09-23T15:16:37.967951+0000","last_fullsized":"2023-09-23T15:16:37.967951+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967476+0000","last_change":"2023-09-23T15:12:45.088119+0000","last_active":"2023-09-23T15:16:37.967476+0000","last_peered":"2023-09-23T15:16:37.967476+0000","last_clean":"2023-09-23T15:16:37.967476+0000","last_became_active":"2023-09-23T15:12:45.087414+0000","last_became_peered":"2023-09-23T15:12:45.087414+0000","last_unstale":"2023-09-23T15:16:37.967476+0000","last_undegraded":"2023-09-23T15:16:37.967476+0000","last_fullsized":"2023-09-23T15:16:37.967476+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:17.653076+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":190,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.239259+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:16:43.239259+0000","last_peered":"2023-09-23T15:16:43.239259+0000","last_clean":"2023-09-23T15:16:43.239259+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:16:43.239259+0000","last_undegraded":"2023-09-23T15:16:43.239259+0000","last_fullsized":"2023-09-23T15:16:43.239259+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":212,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.110375+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:16:38.110375+0000","last_peered":"2023-09-23T15:16:38.110375+0000","last_clean":"2023-09-23T15:16:38.110375+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:16:38.110375+0000","last_undegraded":"2023-09-23T15:16:38.110375+0000","last_fullsized":"2023-09-23T15:16:38.110375+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.968013+0000","last_change":"2023-09-23T15:12:45.083074+0000","last_active":"2023-09-23T15:16:37.968013+0000","last_peered":"2023-09-23T15:16:37.968013+0000","last_clean":"2023-09-23T15:16:37.968013+0000","last_became_active":"2023-09-23T15:12:45.082682+0000","last_became_peered":"2023-09-23T15:12:45.082682+0000","last_unstale":"2023-09-23T15:16:37.968013+0000","last_undegraded":"2023-09-23T15:16:37.968013+0000","last_fullsized":"2023-09-23T15:16:37.968013+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:26.187055+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:26.187055+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":98,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247187+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:16:43.247187+0000","last_peered":"2023-09-23T15:16:43.247187+0000","last_clean":"2023-09-23T15:16:43.247187+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:16:43.247187+0000","last_undegraded":"2023-09-23T15:16:43.247187+0000","last_fullsized":"2023-09-23T15:16:43.247187+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108502+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:16:38.108502+0000","last_peered":"2023-09-23T15:16:38.108502+0000","last_clean":"2023-09-23T15:16:38.108502+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:16:38.108502+0000","last_undegraded":"2023-09-23T15:16:38.108502+0000","last_fullsized":"2023-09-23T15:16:38.108502+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":502,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.974866+0000","last_change":"2023-09-23T15:16:38.974866+0000","last_active":"2023-09-23T15:16:38.974866+0000","last_peered":"2023-09-23T15:16:38.974866+0000","last_clean":"2023-09-23T15:16:38.974866+0000","last_became_active":"2023-09-23T15:16:38.974361+0000","last_became_peered":"2023-09-23T15:16:38.974361+0000","last_unstale":"2023-09-23T15:16:38.974866+0000","last_undegraded":"2023-09-23T15:16:38.974866+0000","last_fullsized":"2023-09-23T15:16:38.974866+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":381350,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":98,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247760+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:16:43.247760+0000","last_peered":"2023-09-23T15:16:43.247760+0000","last_clean":"2023-09-23T15:16:43.247760+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:16:43.247760+0000","last_undegraded":"2023-09-23T15:16:43.247760+0000","last_fullsized":"2023-09-23T15:16:43.247760+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":216,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246659+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:16:43.246659+0000","last_peered":"2023-09-23T15:16:43.246659+0000","last_clean":"2023-09-23T15:16:43.246659+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:16:43.246659+0000","last_undegraded":"2023-09-23T15:16:43.246659+0000","last_fullsized":"2023-09-23T15:16:43.246659+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":423,"reported_epoch":224,"state":"activating+undersized+remapped","last_fresh":"2023-09-23T15:16:44.243902+0000","last_change":"2023-09-23T15:16:44.242399+0000","last_active":"2023-09-23T15:16:42.486446+0000","last_peered":"2023-09-23T15:00:28.139526+0000","last_clean":"2023-09-23T15:00:28.139526+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T15:16:44.243902+0000","last_undegraded":"2023-09-23T15:16:44.243902+0000","last_fullsized":"2023-09-23T15:16:44.242124+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,4],"avail_no_missing":["6","0","1","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":81,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108879+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:16:38.108879+0000","last_peered":"2023-09-23T15:16:38.108879+0000","last_clean":"2023-09-23T15:16:38.108879+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:16:38.108879+0000","last_undegraded":"2023-09-23T15:16:38.108879+0000","last_fullsized":"2023-09-23T15:16:38.108879+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":212,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.241874+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:16:43.241874+0000","last_peered":"2023-09-23T15:16:43.241874+0000","last_clean":"2023-09-23T15:16:43.241874+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:16:43.241874+0000","last_undegraded":"2023-09-23T15:16:43.241874+0000","last_fullsized":"2023-09-23T15:16:43.241874+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":1,"reported_epoch":223,"state":"creating+peering","last_fresh":"2023-09-23T15:16:43.238167+0000","last_change":"2023-09-23T15:16:43.238167+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.238167+0000","last_undegraded":"2023-09-23T15:16:43.238167+0000","last_fullsized":"2023-09-23T15:16:43.238167+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:03.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967666+0000","last_change":"2023-09-23T15:12:45.088143+0000","last_active":"2023-09-23T15:16:37.967666+0000","last_peered":"2023-09-23T15:16:37.967666+0000","last_clean":"2023-09-23T15:16:37.967666+0000","last_became_active":"2023-09-23T15:12:45.086903+0000","last_became_peered":"2023-09-23T15:12:45.086903+0000","last_unstale":"2023-09-23T15:16:37.967666+0000","last_undegraded":"2023-09-23T15:16:37.967666+0000","last_fullsized":"2023-09-23T15:16:37.967666+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:31.275090+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:31.275090+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":180,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108740+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:38.108740+0000","last_peered":"2023-09-23T15:16:38.108740+0000","last_clean":"2023-09-23T15:16:38.108740+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:16:38.108740+0000","last_undegraded":"2023-09-23T15:16:38.108740+0000","last_fullsized":"2023-09-23T15:16:38.108740+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":181,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967495+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:16:37.967495+0000","last_peered":"2023-09-23T15:16:37.967495+0000","last_clean":"2023-09-23T15:16:37.967495+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:16:37.967495+0000","last_undegraded":"2023-09-23T15:16:37.967495+0000","last_fullsized":"2023-09-23T15:16:37.967495+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967642+0000","last_change":"2023-09-23T15:12:45.089951+0000","last_active":"2023-09-23T15:16:37.967642+0000","last_peered":"2023-09-23T15:16:37.967642+0000","last_clean":"2023-09-23T15:16:37.967642+0000","last_became_active":"2023-09-23T15:12:45.089209+0000","last_became_peered":"2023-09-23T15:12:45.089209+0000","last_unstale":"2023-09-23T15:16:37.967642+0000","last_undegraded":"2023-09-23T15:16:37.967642+0000","last_fullsized":"2023-09-23T15:16:37.967642+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:07:31.632511+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":184,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.109146+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:16:38.109146+0000","last_peered":"2023-09-23T15:16:38.109146+0000","last_clean":"2023-09-23T15:16:38.109146+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:16:38.109146+0000","last_undegraded":"2023-09-23T15:16:38.109146+0000","last_fullsized":"2023-09-23T15:16:38.109146+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":182,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247171+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:16:43.247171+0000","last_peered":"2023-09-23T15:16:43.247171+0000","last_clean":"2023-09-23T15:16:43.247171+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:16:43.247171+0000","last_undegraded":"2023-09-23T15:16:43.247171+0000","last_fullsized":"2023-09-23T15:16:43.247171+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":1,"reported_epoch":223,"state":"creating+peering","last_fresh":"2023-09-23T15:16:43.237442+0000","last_change":"2023-09-23T15:16:43.237442+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.237442+0000","last_undegraded":"2023-09-23T15:16:43.237442+0000","last_fullsized":"2023-09-23T15:16:43.237442+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":515,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.966903+0000","last_change":"2023-09-23T15:12:45.081143+0000","last_active":"2023-09-23T15:16:37.966903+0000","last_peered":"2023-09-23T15:16:37.966903+0000","last_clean":"2023-09-23T15:16:37.966903+0000","last_became_active":"2023-09-23T15:12:45.080731+0000","last_became_peered":"2023-09-23T15:12:45.080731+0000","last_unstale":"2023-09-23T15:16:37.966903+0000","last_undegraded":"2023-09-23T15:16:37.966903+0000","last_fullsized":"2023-09-23T15:16:37.966903+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:02.274295+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":199,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.241717+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:16:43.241717+0000","last_peered":"2023-09-23T15:16:43.241717+0000","last_clean":"2023-09-23T15:16:43.241717+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:16:43.241717+0000","last_undegraded":"2023-09-23T15:16:43.241717+0000","last_fullsized":"2023-09-23T15:16:43.241717+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":216,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247324+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:16:43.247324+0000","last_peered":"2023-09-23T15:16:43.247324+0000","last_clean":"2023-09-23T15:16:43.247324+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:16:43.247324+0000","last_undegraded":"2023-09-23T15:16:43.247324+0000","last_fullsized":"2023-09-23T15:16:43.247324+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":2,"reported_epoch":224,"state":"creating+activating","last_fresh":"2023-09-23T15:16:44.243491+0000","last_change":"2023-09-23T15:16:44.243709+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:44.243491+0000","last_undegraded":"2023-09-23T15:16:44.243491+0000","last_fullsized":"2023-09-23T15:16:44.243491+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":["6","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967272+0000","last_change":"2023-09-23T15:12:45.085799+0000","last_active":"2023-09-23T15:16:37.967272+0000","last_peered":"2023-09-23T15:16:37.967272+0000","last_clean":"2023-09-23T15:16:37.967272+0000","last_became_active":"2023-09-23T15:12:45.085155+0000","last_became_peered":"2023-09-23T15:12:45.085155+0000","last_unstale":"2023-09-23T15:16:37.967272+0000","last_undegraded":"2023-09-23T15:16:37.967272+0000","last_fullsized":"2023-09-23T15:16:37.967272+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":207,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246549+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:16:43.246549+0000","last_peered":"2023-09-23T15:16:43.246549+0000","last_clean":"2023-09-23T15:16:43.246549+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:16:43.246549+0000","last_undegraded":"2023-09-23T15:16:43.246549+0000","last_fullsized":"2023-09-23T15:16:43.246549+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":199,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247838+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:16:43.247838+0000","last_peered":"2023-09-23T15:16:43.247838+0000","last_clean":"2023-09-23T15:16:43.247838+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:16:43.247838+0000","last_undegraded":"2023-09-23T15:16:43.247838+0000","last_fullsized":"2023-09-23T15:16:43.247838+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":2,"reported_epoch":224,"state":"creating+activating","last_fresh":"2023-09-23T15:16:44.244090+0000","last_change":"2023-09-23T15:16:44.244324+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:44.244090+0000","last_undegraded":"2023-09-23T15:16:44.244090+0000","last_fullsized":"2023-09-23T15:16:44.244090+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":["6","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967342+0000","last_change":"2023-09-23T15:12:45.084048+0000","last_active":"2023-09-23T15:16:37.967342+0000","last_peered":"2023-09-23T15:16:37.967342+0000","last_clean":"2023-09-23T15:16:37.967342+0000","last_became_active":"2023-09-23T15:12:45.083715+0000","last_became_peered":"2023-09-23T15:12:45.083715+0000","last_unstale":"2023-09-23T15:16:37.967342+0000","last_undegraded":"2023-09-23T15:16:37.967342+0000","last_fullsized":"2023-09-23T15:16:37.967342+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:45.395893+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":213,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108665+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:16:38.108665+0000","last_peered":"2023-09-23T15:16:38.108665+0000","last_clean":"2023-09-23T15:16:38.108665+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:16:38.108665+0000","last_undegraded":"2023-09-23T15:16:38.108665+0000","last_fullsized":"2023-09-23T15:16:38.108665+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":98,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965673+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:16:37.965673+0000","last_peered":"2023-09-23T15:16:37.965673+0000","last_clean":"2023-09-23T15:16:37.965673+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:16:37.965673+0000","last_undegraded":"2023-09-23T15:16:37.965673+0000","last_fullsized":"2023-09-23T15:16:37.965673+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":9,"reported_epoch":224,"state":"active+clean","last_fresh":"2023-09-23T15:16:44.271607+0000","last_change":"2023-09-23T15:16:44.271607+0000","last_active":"2023-09-23T15:16:44.271607+0000","last_peered":"2023-09-23T15:16:44.271607+0000","last_clean":"2023-09-23T15:16:44.271607+0000","last_became_active":"2023-09-23T15:16:44.271334+0000","last_became_peered":"2023-09-23T15:16:44.271334+0000","last_unstale":"2023-09-23T15:16:44.271607+0000","last_undegraded":"2023-09-23T15:16:44.271607+0000","last_fullsized":"2023-09-23T15:16:44.271607+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:58.532783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":422,"reported_epoch":224,"state":"activating+undersized+remapped","last_fresh":"2023-09-23T15:16:44.241779+0000","last_change":"2023-09-23T15:16:44.241779+0000","last_active":"2023-09-23T15:16:42.487586+0000","last_peered":"2023-09-23T15:00:28.139526+0000","last_clean":"2023-09-23T15:00:28.139526+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T15:16:44.241779+0000","last_undegraded":"2023-09-23T15:16:44.241779+0000","last_fullsized":"2023-09-23T15:16:44.241524+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3],"avail_no_missing":["6","0","1","3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":397,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.966007+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:16:37.966007+0000","last_peered":"2023-09-23T15:16:37.966007+0000","last_clean":"2023-09-23T15:16:37.966007+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:16:37.966007+0000","last_undegraded":"2023-09-23T15:16:37.966007+0000","last_fullsized":"2023-09-23T15:16:37.966007+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":197,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108818+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:38.108818+0000","last_peered":"2023-09-23T15:16:38.108818+0000","last_clean":"2023-09-23T15:16:38.108818+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:16:38.108818+0000","last_undegraded":"2023-09-23T15:16:38.108818+0000","last_fullsized":"2023-09-23T15:16:38.108818+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":145,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T15:16:42.493367+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:16:42.493367+0000","last_peered":"2023-09-23T15:16:42.493367+0000","last_clean":"2023-09-23T15:16:42.493367+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:16:42.493367+0000","last_undegraded":"2023-09-23T15:16:42.493367+0000","last_fullsized":"2023-09-23T15:16:42.493367+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.241174+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:29.159730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.968013+0000","last_change":"2023-09-23T15:12:45.093015+0000","last_active":"2023-09-23T15:16:37.968013+0000","last_peered":"2023-09-23T15:16:37.968013+0000","last_clean":"2023-09-23T15:16:37.968013+0000","last_became_active":"2023-09-23T15:12:45.090958+0000","last_became_peered":"2023-09-23T15:12:45.090958+0000","last_unstale":"2023-09-23T15:16:37.968013+0000","last_undegraded":"2023-09-23T15:16:37.968013+0000","last_fullsized":"2023-09-23T15:16:37.968013+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:58.450807+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:58.450807+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":739,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108683+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:16:38.108683+0000","last_peered":"2023-09-23T15:16:38.108683+0000","last_clean":"2023-09-23T15:16:38.108683+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:16:38.108683+0000","last_undegraded":"2023-09-23T15:16:38.108683+0000","last_fullsized":"2023-09-23T15:16:38.108683+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":183,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246825+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:16:43.246825+0000","last_peered":"2023-09-23T15:16:43.246825+0000","last_clean":"2023-09-23T15:16:43.246825+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:16:43.246825+0000","last_undegraded":"2023-09-23T15:16:43.246825+0000","last_fullsized":"2023-09-23T15:16:43.246825+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":113,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.966426+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:16:37.966426+0000","last_peered":"2023-09-23T15:16:37.966426+0000","last_clean":"2023-09-23T15:16:37.966426+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:16:37.966426+0000","last_undegraded":"2023-09-23T15:16:37.966426+0000","last_fullsized":"2023-09-23T15:16:37.966426+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967181+0000","last_change":"2023-09-23T15:12:45.088297+0000","last_active":"2023-09-23T15:16:37.967181+0000","last_peered":"2023-09-23T15:16:37.967181+0000","last_clean":"2023-09-23T15:16:37.967181+0000","last_became_active":"2023-09-23T15:12:45.087756+0000","last_became_peered":"2023-09-23T15:12:45.087756+0000","last_unstale":"2023-09-23T15:16:37.967181+0000","last_undegraded":"2023-09-23T15:16:37.967181+0000","last_fullsized":"2023-09-23T15:16:37.967181+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:13.484371+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":836,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T15:16:42.492523+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:16:42.492523+0000","last_peered":"2023-09-23T15:16:42.492523+0000","last_clean":"2023-09-23T15:16:42.492523+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:16:42.492523+0000","last_undegraded":"2023-09-23T15:16:42.492523+0000","last_fullsized":"2023-09-23T15:16:42.492523+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":93,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965240+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:16:37.965240+0000","last_peered":"2023-09-23T15:16:37.965240+0000","last_clean":"2023-09-23T15:16:37.965240+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:16:37.965240+0000","last_undegraded":"2023-09-23T15:16:37.965240+0000","last_fullsized":"2023-09-23T15:16:37.965240+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":117,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965627+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:16:37.965627+0000","last_peered":"2023-09-23T15:16:37.965627+0000","last_clean":"2023-09-23T15:16:37.965627+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:16:37.965627+0000","last_undegraded":"2023-09-23T15:16:37.965627+0000","last_fullsized":"2023-09-23T15:16:37.965627+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":508,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108713+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:16:38.108713+0000","last_peered":"2023-09-23T15:16:38.108713+0000","last_clean":"2023-09-23T15:16:38.108713+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:16:38.108713+0000","last_undegraded":"2023-09-23T15:16:38.108713+0000","last_fullsized":"2023-09-23T15:16:38.108713+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":510,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967079+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:16:37.967079+0000","last_peered":"2023-09-23T15:16:37.967079+0000","last_clean":"2023-09-23T15:16:37.967079+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:16:37.967079+0000","last_undegraded":"2023-09-23T15:16:37.967079+0000","last_fullsized":"2023-09-23T15:16:37.967079+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":136,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T15:16:39.303604+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:16:39.303604+0000","last_peered":"2023-09-23T15:16:39.303604+0000","last_clean":"2023-09-23T15:16:39.303604+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:16:39.303604+0000","last_undegraded":"2023-09-23T15:16:39.303604+0000","last_fullsized":"2023-09-23T15:16:39.303604+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":174,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.240218+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:16:43.240218+0000","last_peered":"2023-09-23T15:16:43.240218+0000","last_clean":"2023-09-23T15:16:43.240218+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:16:43.240218+0000","last_undegraded":"2023-09-23T15:16:43.240218+0000","last_fullsized":"2023-09-23T15:16:43.240218+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":1,"reported_epoch":223,"state":"creating+peering","last_fresh":"2023-09-23T15:16:43.239292+0000","last_change":"2023-09-23T15:16:43.239292+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.239292+0000","last_undegraded":"2023-09-23T15:16:43.239292+0000","last_fullsized":"2023-09-23T15:16:43.239292+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967399+0000","last_change":"2023-09-23T15:12:45.085615+0000","last_active":"2023-09-23T15:16:37.967399+0000","last_peered":"2023-09-23T15:16:37.967399+0000","last_clean":"2023-09-23T15:16:37.967399+0000","last_became_active":"2023-09-23T15:12:45.084721+0000","last_became_peered":"2023-09-23T15:12:45.084721+0000","last_unstale":"2023-09-23T15:16:37.967399+0000","last_undegraded":"2023-09-23T15:16:37.967399+0000","last_fullsized":"2023-09-23T15:16:37.967399+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T14:53:07.968655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":304,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965054+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:16:37.965054+0000","last_peered":"2023-09-23T15:16:37.965054+0000","last_clean":"2023-09-23T15:16:37.965054+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:16:37.965054+0000","last_undegraded":"2023-09-23T15:16:37.965054+0000","last_fullsized":"2023-09-23T15:16:37.965054+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":166,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967822+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:16:37.967822+0000","last_peered":"2023-09-23T15:16:37.967822+0000","last_clean":"2023-09-23T15:16:37.967822+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:16:37.967822+0000","last_undegraded":"2023-09-23T15:16:37.967822+0000","last_fullsized":"2023-09-23T15:16:37.967822+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":196,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967749+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:16:37.967749+0000","last_peered":"2023-09-23T15:16:37.967749+0000","last_clean":"2023-09-23T15:16:37.967749+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:16:37.967749+0000","last_undegraded":"2023-09-23T15:16:37.967749+0000","last_fullsized":"2023-09-23T15:16:37.967749+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":421,"reported_epoch":224,"state":"activating+undersized+remapped","last_fresh":"2023-09-23T15:16:44.241497+0000","last_change":"2023-09-23T15:16:44.241722+0000","last_active":"2023-09-23T15:16:42.487061+0000","last_peered":"2023-09-23T15:00:28.139526+0000","last_clean":"2023-09-23T15:00:28.139526+0000","last_became_active":"2023-09-23T14:54:58.918256+0000","last_became_peered":"2023-09-23T14:54:58.918256+0000","last_unstale":"2023-09-23T15:16:44.241497+0000","last_undegraded":"2023-09-23T15:16:44.241497+0000","last_fullsized":"2023-09-23T15:16:44.241497+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:27.869946+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:27.869946+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0],"avail_no_missing":["6","0","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":842,"reported_epoch":221,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.973875+0000","last_change":"2023-09-23T15:16:38.973875+0000","last_active":"2023-09-23T15:16:38.973875+0000","last_peered":"2023-09-23T15:16:38.973875+0000","last_clean":"2023-09-23T15:16:38.973875+0000","last_became_active":"2023-09-23T15:16:38.973269+0000","last_became_peered":"2023-09-23T15:16:38.973269+0000","last_unstale":"2023-09-23T15:16:38.973875+0000","last_undegraded":"2023-09-23T15:16:38.973875+0000","last_fullsized":"2023-09-23T15:16:38.973875+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:17:00.522554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":93,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965915+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:16:37.965915+0000","last_peered":"2023-09-23T15:16:37.965915+0000","last_clean":"2023-09-23T15:16:37.965915+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:16:37.965915+0000","last_undegraded":"2023-09-23T15:16:37.965915+0000","last_fullsized":"2023-09-23T15:16:37.965915+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":171,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967705+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:16:37.967705+0000","last_peered":"2023-09-23T15:16:37.967705+0000","last_clean":"2023-09-23T15:16:37.967705+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:16:37.967705+0000","last_undegraded":"2023-09-23T15:16:37.967705+0000","last_fullsized":"2023-09-23T15:16:37.967705+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967955+0000","last_change":"2023-09-23T15:12:45.082645+0000","last_active":"2023-09-23T15:16:37.967955+0000","last_peered":"2023-09-23T15:16:37.967955+0000","last_clean":"2023-09-23T15:16:37.967955+0000","last_became_active":"2023-09-23T15:12:45.081460+0000","last_became_peered":"2023-09-23T15:12:45.081460+0000","last_unstale":"2023-09-23T15:16:37.967955+0000","last_undegraded":"2023-09-23T15:16:37.967955+0000","last_fullsized":"2023-09-23T15:16:37.967955+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:11:29.819901+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:11:29.819901+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":197,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.110363+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:16:38.110363+0000","last_peered":"2023-09-23T15:16:38.110363+0000","last_clean":"2023-09-23T15:16:38.110363+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:16:38.110363+0000","last_undegraded":"2023-09-23T15:16:38.110363+0000","last_fullsized":"2023-09-23T15:16:38.110363+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":681,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.965533+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:16:37.965533+0000","last_peered":"2023-09-23T15:16:37.965533+0000","last_clean":"2023-09-23T15:16:37.965533+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:16:37.965533+0000","last_undegraded":"2023-09-23T15:16:37.965533+0000","last_fullsized":"2023-09-23T15:16:37.965533+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":202,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108797+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:16:38.108797+0000","last_peered":"2023-09-23T15:16:38.108797+0000","last_clean":"2023-09-23T15:16:38.108797+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:16:38.108797+0000","last_undegraded":"2023-09-23T15:16:38.108797+0000","last_fullsized":"2023-09-23T15:16:38.108797+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T15:16:43.225650+0000","last_change":"2023-09-23T15:16:43.225650+0000","last_active":"2023-09-23T15:16:43.225650+0000","last_peered":"2023-09-23T15:16:43.225650+0000","last_clean":"2023-09-23T15:16:43.225650+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T15:16:43.225650+0000","last_undegraded":"2023-09-23T15:16:43.225650+0000","last_fullsized":"2023-09-23T15:16:43.225650+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":498,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967922+0000","last_change":"2023-09-23T15:12:45.082388+0000","last_active":"2023-09-23T15:16:37.967922+0000","last_peered":"2023-09-23T15:16:37.967922+0000","last_clean":"2023-09-23T15:16:37.967922+0000","last_became_active":"2023-09-23T15:12:45.082003+0000","last_became_peered":"2023-09-23T15:12:45.082003+0000","last_unstale":"2023-09-23T15:16:37.967922+0000","last_undegraded":"2023-09-23T15:16:37.967922+0000","last_fullsized":"2023-09-23T15:16:37.967922+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:10:29.173921+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:10:29.173921+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":699,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T15:16:42.487029+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:16:42.487029+0000","last_peered":"2023-09-23T15:16:42.487029+0000","last_clean":"2023-09-23T15:16:42.487029+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:16:42.487029+0000","last_undegraded":"2023-09-23T15:16:42.487029+0000","last_fullsized":"2023-09-23T15:16:42.487029+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":184,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967855+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:16:37.967855+0000","last_peered":"2023-09-23T15:16:37.967855+0000","last_clean":"2023-09-23T15:16:37.967855+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:16:37.967855+0000","last_undegraded":"2023-09-23T15:16:37.967855+0000","last_fullsized":"2023-09-23T15:16:37.967855+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":165,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246652+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:16:43.246652+0000","last_peered":"2023-09-23T15:16:43.246652+0000","last_clean":"2023-09-23T15:16:43.246652+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:16:43.246652+0000","last_undegraded":"2023-09-23T15:16:43.246652+0000","last_fullsized":"2023-09-23T15:16:43.246652+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":173,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967682+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:16:37.967682+0000","last_peered":"2023-09-23T15:16:37.967682+0000","last_clean":"2023-09-23T15:16:37.967682+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:16:37.967682+0000","last_undegraded":"2023-09-23T15:16:37.967682+0000","last_fullsized":"2023-09-23T15:16:37.967682+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":223,"reported_epoch":222,"state":"active+clean","last_fresh":"2023-09-23T15:16:42.494749+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:16:42.494749+0000","last_peered":"2023-09-23T15:16:42.494749+0000","last_clean":"2023-09-23T15:16:42.494749+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:16:42.494749+0000","last_undegraded":"2023-09-23T15:16:42.494749+0000","last_fullsized":"2023-09-23T15:16:42.494749+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":199,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108787+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:16:38.108787+0000","last_peered":"2023-09-23T15:16:38.108787+0000","last_clean":"2023-09-23T15:16:38.108787+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:16:38.108787+0000","last_undegraded":"2023-09-23T15:16:38.108787+0000","last_fullsized":"2023-09-23T15:16:38.108787+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.247047+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:16:43.247047+0000","last_peered":"2023-09-23T15:16:43.247047+0000","last_clean":"2023-09-23T15:16:43.247047+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:16:43.247047+0000","last_undegraded":"2023-09-23T15:16:43.247047+0000","last_fullsized":"2023-09-23T15:16:43.247047+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":165,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108787+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:16:38.108787+0000","last_peered":"2023-09-23T15:16:38.108787+0000","last_clean":"2023-09-23T15:16:38.108787+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:16:38.108787+0000","last_undegraded":"2023-09-23T15:16:38.108787+0000","last_fullsized":"2023-09-23T15:16:38.108787+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":199,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246782+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:16:43.246782+0000","last_peered":"2023-09-23T15:16:43.246782+0000","last_clean":"2023-09-23T15:16:43.246782+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:16:43.246782+0000","last_undegraded":"2023-09-23T15:16:43.246782+0000","last_fullsized":"2023-09-23T15:16:43.246782+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":193,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.110236+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:16:38.110236+0000","last_peered":"2023-09-23T15:16:38.110236+0000","last_clean":"2023-09-23T15:16:38.110236+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:16:38.110236+0000","last_undegraded":"2023-09-23T15:16:38.110236+0000","last_fullsized":"2023-09-23T15:16:38.110236+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":207,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:38.108795+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:16:38.108795+0000","last_peered":"2023-09-23T15:16:38.108795+0000","last_clean":"2023-09-23T15:16:38.108795+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:16:38.108795+0000","last_undegraded":"2023-09-23T15:16:38.108795+0000","last_fullsized":"2023-09-23T15:16:38.108795+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":169,"reported_epoch":220,"state":"active+clean","last_fresh":"2023-09-23T15:16:37.967853+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:16:37.967853+0000","last_peered":"2023-09-23T15:16:37.967853+0000","last_clean":"2023-09-23T15:16:37.967853+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:16:37.967853+0000","last_undegraded":"2023-09-23T15:16:37.967853+0000","last_fullsized":"2023-09-23T15:16:37.967853+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":192,"reported_epoch":223,"state":"active+clean","last_fresh":"2023-09-23T15:16:43.246175+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:16:43.246175+0000","last_peered":"2023-09-23T15:16:43.246175+0000","last_clean":"2023-09-23T15:16:43.246175+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:16:43.246175+0000","last_undegraded":"2023-09-23T15:16:43.246175+0000","last_fullsized":"2023-09-23T15:16:43.246175+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":21,"acting":21,"num_store_stats":3},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":557873,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":242,"num_read_kb":456,"num_write":121,"num_write_kb":1210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1475921,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":60,"num_store_stats":6},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201544,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463180,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":174988,"kb_used_data":2820,"kb_used_omap":0,"kb_used_meta":172160,"kb_avail":93570164,"statfs":{"total":95995035648,"available":95815847936,"internally_reserved":0,"allocated":2887680,"data_stored":2145893,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":176291840},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.1469999999999998,"15min":2.7120000000000002},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.2519999999999998,"5min":41.491,"15min":108.39400000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.4649999999999999,"15min":2.9289999999999998},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.1029999999999998,"5min":43.151000000000003,"15min":108.452},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3300000000000001,"5min":3.395,"15min":3.379},"min":{"1min":0.36699999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":9.266,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.60499999999999998},{"interface":"front","average":{"1min":3.2589999999999999,"5min":4.274,"15min":3.8290000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":8.4060000000000006,"5min":42.704000000000001,"15min":42.704000000000001},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5129999999999999,"5min":5.7489999999999997,"15min":6.0949999999999998},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":68.828000000000003,"5min":112.352,"15min":180.374},"last":0.44500000000000001},{"interface":"front","average":{"1min":5.681,"5min":5.8140000000000001,"15min":6.0709999999999997},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":68.730999999999995,"5min":117.667,"15min":181.685},"last":0.56499999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":7.0979999999999999,"15min":3.1539999999999999},"min":{"1min":0.42699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.1440000000000001,"5min":102.254,"15min":102.254},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.8109999999999999,"5min":7.234,"15min":3.1640000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7030000000000003,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.499}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0349999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.7150000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.624,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0019999999999998,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.67500000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.0099999999999998,"15min":1.423},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":3.1789999999999998,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.0249999999999999,"15min":1.4590000000000001},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0510000000000002,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.74099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.82499999999999996,"15min":1.0289999999999999},"min":{"1min":0.36599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":1.8700000000000001,"5min":5.125,"15min":17},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.77000000000000002,"15min":0.98999999999999999},"min":{"1min":0.438,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":1.8180000000000001,"5min":3.754,"15min":7.7809999999999997},"last":0.64000000000000001}]}]},{"osd":1,"up_from":183,"seq":785979015217,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":206688,"kb_used_data":7256,"kb_used_omap":6,"kb_used_meta":199417,"kb_avail":93538464,"statfs":{"total":95995035648,"available":95783387136,"internally_reserved":0,"allocated":7430144,"data_stored":6369600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":204203083},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.7919999999999998,"5min":3.4049999999999998,"15min":3.6709999999999998},"min":{"1min":0.40200000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":16.338000000000001,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.45600000000000002},{"interface":"front","average":{"1min":5.6859999999999999,"5min":4.6150000000000002,"15min":4.6109999999999998},"min":{"1min":0.29599999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":16.414999999999999,"5min":16.414999999999999,"15min":16.414999999999999},"last":0.38400000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":4.1429999999999998,"15min":4.133},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":10.315,"5min":13.843,"15min":13.843},"last":0.40699999999999997},{"interface":"front","average":{"1min":3.3479999999999999,"5min":4.0709999999999997,"15min":4.1719999999999997},"min":{"1min":0.434,"5min":0.253,"15min":0.253},"max":{"1min":16.364999999999998,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.48799999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":4.46,"5min":8.3599999999999994,"15min":9.984},"min":{"1min":0.40899999999999997,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":19.617999999999999,"5min":123.426,"15min":123.426},"last":0.52200000000000002},{"interface":"front","average":{"1min":5.3920000000000003,"5min":9.3279999999999994,"15min":11.486000000000001},"min":{"1min":0.68100000000000005,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":16.451000000000001,"5min":123.32899999999999,"15min":123.32899999999999},"last":0.442}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.6320000000000001,"5min":2.6960000000000002,"15min":2.6960000000000002},"min":{"1min":0.32600000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":9.8409999999999993,"5min":9.8409999999999993,"15min":9.8409999999999993},"last":0.73499999999999999},{"interface":"front","average":{"1min":2.823,"5min":3.3250000000000002,"15min":3.48},"min":{"1min":0.34000000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":16.228000000000002,"5min":16.779,"15min":16.779},"last":0.89200000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.855,"5min":3.4340000000000002,"15min":3.6640000000000001},"min":{"1min":0.48999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":14.898999999999999,"5min":16.166,"15min":16.166},"last":0.83699999999999997},{"interface":"front","average":{"1min":3.5179999999999998,"5min":3.0449999999999999,"15min":2.9460000000000002},"min":{"1min":0.36299999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.8680000000000003,"5min":9.8680000000000003,"15min":9.8680000000000003},"last":0.68200000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":3.4209999999999998,"15min":3.6800000000000002},"min":{"1min":0.67500000000000004,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":14.867000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":0.69899999999999995},{"interface":"front","average":{"1min":3.7240000000000002,"5min":3,"15min":2.8319999999999999},"min":{"1min":0.438,"5min":0.377,"15min":0.377},"max":{"1min":12.448,"5min":12.448,"15min":12.448},"last":0.94499999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.9700000000000002,"5min":3.1379999999999999,"15min":3.1499999999999999},"min":{"1min":0.47199999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":9.8919999999999995,"5min":9.8919999999999995,"15min":9.8919999999999995},"last":0.84799999999999998},{"interface":"front","average":{"1min":3.5169999999999999,"5min":3.323,"15min":3.343},"min":{"1min":0.40000000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":16.280999999999999,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.93200000000000005}]}]},{"osd":0,"up_from":95,"seq":408021893266,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":324520,"kb_used_data":7136,"kb_used_omap":8,"kb_used_meta":317367,"kb_avail":93420632,"statfs":{"total":95995035648,"available":95662727168,"internally_reserved":0,"allocated":7307264,"data_stored":6348223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":324984404},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:15:45 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":3.298,"15min":3.96},"min":{"1min":0.19800000000000001,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":5.4109999999999996,"5min":31.510999999999999,"15min":31.510999999999999},"last":0.307},{"interface":"front","average":{"1min":1.9770000000000001,"5min":3.7280000000000002,"15min":4.3819999999999997},"min":{"1min":0.254,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":7.5590000000000002,"5min":31.378,"15min":31.378},"last":0.28299999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":7.133,"15min":3.5539999999999998},"min":{"1min":0.26200000000000001,"5min":0.189,"15min":0.17000000000000001},"max":{"1min":13.593,"5min":103.81,"15min":122.20099999999999},"last":0.434},{"interface":"front","average":{"1min":3.4279999999999999,"5min":6.077,"15min":3.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.18099999999999999},"max":{"1min":9.7479999999999993,"5min":108.19499999999999,"15min":122.071},"last":0.23499999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:16:32 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":9.8650000000000002,"15min":3.9460000000000002},"min":{"1min":0.27200000000000002,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":146.315,"5min":146.315,"15min":146.315},"last":0.372},{"interface":"front","average":{"1min":17.911999999999999,"5min":10.164999999999999,"15min":4.3300000000000001},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":145.517,"5min":145.517,"15min":145.517},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.5230000000000001,"15min":1.649},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.9950000000000001,"5min":31.341999999999999,"15min":31.341999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.4710000000000001,"5min":3.282,"15min":1.72},"min":{"1min":0.29699999999999999,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":13.566000000000001,"5min":99.168999999999997,"15min":99.168999999999997},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.992,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29699999999999999},"max":{"1min":9.6020000000000003,"5min":109.369,"15min":109.369},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.2170000000000001,"15min":1.857},"min":{"1min":0.42999999999999999,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":4.4219999999999997,"5min":109.387,"15min":109.387},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.996,"15min":1.752},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":13.529999999999999,"5min":99.114999999999995,"15min":99.114999999999995},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.8650000000000002,"15min":1.677},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":9.6460000000000008,"5min":98.921999999999997,"15min":98.921999999999997},"last":0.499}]},{"osd":7,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":3.181,"15min":1.802},"min":{"1min":0.44600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.6690000000000005,"5min":99.549999999999997,"15min":99.549999999999997},"last":0.45300000000000001},{"interface":"front","average":{"1min":1.702,"5min":2.2770000000000001,"15min":1.5549999999999999},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9720000000000004,"5min":31.225999999999999,"15min":31.225999999999999},"last":0.61399999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566214,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":347520,"kb_used_data":2488,"kb_used_omap":11,"kb_used_meta":345012,"kb_avail":93397632,"statfs":{"total":95995035648,"available":95639175168,"internally_reserved":0,"allocated":2547712,"data_stored":1592167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":353293072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":4.9969999999999999,"15min":2.2229999999999999},"min":{"1min":0.27400000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.085000000000001,"5min":87.132999999999996,"15min":87.132999999999996},"last":0.29099999999999998},{"interface":"front","average":{"1min":3.931,"5min":3.8490000000000002,"15min":1.831},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.833,"5min":84.820999999999998,"15min":84.820999999999998},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":3.1309999999999998,"5min":5.492,"15min":5.1900000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":11.058999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.313},{"interface":"front","average":{"1min":2.0590000000000002,"5min":3.782,"15min":2.96},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":10.962999999999999,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:16:30 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":7.4589999999999996,"15min":3.177},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":29.297000000000001,"5min":87.108000000000004,"15min":87.108000000000004},"last":0.36399999999999999},{"interface":"front","average":{"1min":7.899,"5min":6.8819999999999997,"15min":3.0369999999999999},"min":{"1min":0.32200000000000001,"5min":0.23499999999999999,"15min":0.20300000000000001},"max":{"1min":31.440999999999999,"5min":86.158000000000001,"15min":86.158000000000001},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.129,"15min":1.831},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.824,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.8849999999999998,"5min":4.016,"15min":2.4020000000000001},"min":{"1min":0.42399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.988,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.51300000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":3.266,"15min":2.0579999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.193999999999999,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.8690000000000002,"15min":1.9179999999999999},"min":{"1min":0.441,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.865,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.1200000000000001,"15min":2.0369999999999999},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.891999999999999,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.48199999999999998},{"interface":"front","average":{"1min":2.9729999999999999,"5min":3.5750000000000002,"15min":2.0939999999999999},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.949999999999999,"5min":87.055000000000007,"15min":87.055000000000007},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":3.3519999999999999,"15min":1.915},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":16.239000000000001,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.966,"5min":3.0630000000000002,"15min":1.8049999999999999},"min":{"1min":0.48899999999999999,"5min":0.38600000000000001,"15min":0.34200000000000003},"max":{"1min":10.916,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.62}]}]},{"osd":3,"up_from":110,"seq":472446402687,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334020,"kb_used_data":6524,"kb_used_omap":3,"kb_used_meta":327484,"kb_avail":93411132,"statfs":{"total":95995035648,"available":95652999168,"internally_reserved":0,"allocated":6680576,"data_stored":5759364,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":335343785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":45,"apply_latency_ms":45,"commit_latency_ns":45000000,"apply_latency_ns":45000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":2.9510000000000001,"15min":2.2050000000000001},"min":{"1min":0.20100000000000001,"5min":0.19600000000000001,"15min":0.17899999999999999},"max":{"1min":18.864999999999998,"5min":25.111999999999998,"15min":79.611000000000004},"last":0.23599999999999999},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.222,"15min":2.3250000000000002},"min":{"1min":0.315,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":18.800000000000001,"5min":18.800000000000001,"15min":79.656000000000006},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 15:15:49 2023","interfaces":[{"interface":"back","average":{"1min":3.4729999999999999,"5min":4.702,"15min":4.6239999999999997},"min":{"1min":0.27100000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":12.566000000000001,"5min":28.315000000000001,"15min":28.315000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":3.5339999999999998,"5min":4.5609999999999999,"15min":4.7510000000000003},"min":{"1min":0.23799999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":21.539000000000001,"5min":21.539000000000001,"15min":21.539000000000001},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4320000000000004,"5min":4.3650000000000002,"15min":4.0119999999999996},"min":{"1min":0.33400000000000002,"5min":0.255,"15min":0.22},"max":{"1min":18.893999999999998,"5min":61.732999999999997,"15min":109.077},"last":0.33400000000000002},{"interface":"front","average":{"1min":4.8819999999999997,"5min":4.5339999999999998,"15min":4.2439999999999998},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.25700000000000001},"max":{"1min":21.661999999999999,"5min":60.305999999999997,"15min":119.72799999999999},"last":0.27800000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6560000000000001,"5min":2.9609999999999999,"15min":1.6359999999999999},"min":{"1min":0.40100000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":12.958,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":3.0539999999999998,"5min":2.8919999999999999,"15min":1.79},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.497999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.621,"15min":1.776},"min":{"1min":0.371,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.371},{"interface":"front","average":{"1min":3.5670000000000002,"5min":2.6139999999999999,"15min":1.99},"min":{"1min":0.40200000000000002,"5min":0.315,"15min":0.315},"max":{"1min":12.667999999999999,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":3.3319999999999999,"15min":1.911},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.029999999999999,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":3.3639999999999999,"5min":3.323,"15min":2.1299999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.32300000000000001},"max":{"1min":11.117000000000001,"5min":78.028000000000006,"15min":78.028000000000006},"last":0.46400000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5840000000000001,"15min":1.911},"min":{"1min":0.47599999999999998,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.502},{"interface":"front","average":{"1min":3.5630000000000002,"5min":2.593,"15min":1.5569999999999999},"min":{"1min":0.42199999999999999,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":18.696000000000002,"5min":18.696000000000002,"15min":18.696000000000002},"last":0.47499999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986453,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":208264,"kb_used_data":2240,"kb_used_omap":0,"kb_used_meta":206016,"kb_avail":93536888,"statfs":{"total":95995035648,"available":95781773312,"internally_reserved":0,"allocated":2293760,"data_stored":1555548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":210960384},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":2.5830000000000002,"15min":2.6200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.8250000000000002,"5min":65.988,"15min":122.69199999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":1.552,"5min":2.5219999999999998,"15min":2.698},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":5.7149999999999999,"5min":65.396000000000001,"15min":122.745},"last":0.375}]},{"osd":1,"last update":"Sat Sep 23 15:15:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7390000000000001,"5min":3.6920000000000002,"15min":4.2629999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.7539999999999996,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.8860000000000001,"5min":4.3369999999999997,"15min":4.8300000000000001},"min":{"1min":0.29899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":10.114000000000001,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.63100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":4.1260000000000003,"15min":3.1080000000000001},"min":{"1min":0.37,"5min":0.255,"15min":0.255},"max":{"1min":12.595000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.61199999999999999},{"interface":"front","average":{"1min":3.2869999999999999,"5min":4.383,"15min":3.3530000000000002},"min":{"1min":0.34300000000000003,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":7.173,"5min":147.88900000000001,"15min":147.88900000000001},"last":0.46899999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":4.319,"5min":9.6549999999999994,"15min":4.0369999999999999},"min":{"1min":0.46600000000000003,"5min":0.28399999999999997,"15min":0.252},"max":{"1min":23.715,"5min":163.97499999999999,"15min":163.97499999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":4.5860000000000003,"5min":9.7769999999999992,"15min":3.8399999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":22.876000000000001,"5min":164.447,"15min":164.447},"last":0.53900000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:15:43 2023","interfaces":[{"interface":"back","average":{"1min":5.0860000000000003,"5min":1.966,"15min":1.502},"min":{"1min":0.28100000000000003,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":87.912999999999997,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.64400000000000002},{"interface":"front","average":{"1min":5.4960000000000004,"5min":1.9450000000000001,"15min":1.395},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":88.602999999999994,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:16:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":3.6160000000000001,"15min":2.0510000000000002},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.3820000000000001,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.89600000000000002,"5min":3.7080000000000002,"15min":1.956},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.7749999999999999,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.67600000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.621,"15min":1.123},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.417,"5min":5.399,"15min":18.23},"last":0.65400000000000003},{"interface":"front","average":{"1min":1.038,"5min":0.86599999999999999,"15min":1.2030000000000001},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.314},"max":{"1min":3.851,"5min":6.1399999999999997,"15min":14.653},"last":0.73799999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724815,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":209640,"kb_used_data":2208,"kb_used_omap":0,"kb_used_meta":207424,"kb_avail":93535512,"statfs":{"total":95995035648,"available":95780364288,"internally_reserved":0,"allocated":2260992,"data_stored":1534945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":212402176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.9430000000000001,"15min":1.4079999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":7.8319999999999999,"5min":16.981999999999999,"15min":24.962},"last":0.32500000000000001},{"interface":"front","average":{"1min":3.972,"5min":2.1589999999999998,"15min":1.4550000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":12.531000000000001,"5min":18.814,"15min":25.317},"last":0.40699999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":2.347,"5min":2.3730000000000002,"15min":1.9059999999999999},"min":{"1min":0.252,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":8.6110000000000007,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.36499999999999999},{"interface":"front","average":{"1min":4.1109999999999998,"5min":2.7730000000000001,"15min":2.4369999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":11.423,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.266,"15min":2.4609999999999999},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3889999999999993,"5min":52.414000000000001,"15min":97.087999999999994},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.5070000000000001,"5min":3.3570000000000002,"15min":2.4209999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":52.375999999999998,"15min":97.305999999999997},"last":0.34399999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":10.236000000000001,"5min":6.5570000000000004,"15min":3.1299999999999999},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":118.208,"5min":118.208,"15min":118.208},"last":0.49099999999999999},{"interface":"front","average":{"1min":8.9410000000000007,"5min":6.2590000000000003,"15min":2.9009999999999998},"min":{"1min":0.33400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":118.42,"5min":118.42,"15min":118.42},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:15:48 2023","interfaces":[{"interface":"back","average":{"1min":8.7479999999999993,"5min":3.2589999999999999,"15min":1.718},"min":{"1min":0.33300000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":138.68600000000001,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":8.7729999999999997,"5min":3.415,"15min":1.788},"min":{"1min":0.34300000000000003,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":138.49600000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 15:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88900000000000001,"15min":1.1339999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.5550000000000002,"5min":5.8760000000000003,"15min":15.196},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.86499999999999999,"15min":1.0229999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9990000000000001,"5min":4.8819999999999997,"15min":14.965999999999999},"last":0.52300000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.86599999999999999,"15min":1.2829999999999999},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.0249999999999999,"5min":8.2200000000000006,"15min":15.301},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.79100000000000004,"15min":1.2589999999999999},"min":{"1min":0.38500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.5379999999999998,"5min":5.6559999999999997,"15min":15.762},"last":0.50700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:16:46.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:16:46.309 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:46.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:46.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:46.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[215646]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[215646]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[215646]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T15:16:46.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[215646]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[222878]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[222878]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[222878]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T15:16:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:46 smithi017 ceph-mon[222878]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.444 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:46.490 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:46.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:46 smithi027 ceph-mon[195233]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:46 smithi027 ceph-mon[195233]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:46 smithi027 ceph-mon[195233]: osdmap e225: 8 total, 8 up, 7 in 2023-09-23T15:16:46.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:46 smithi027 ceph-mon[195233]: failed to encode map e225 with expected crc 2023-09-23T15:16:46.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:46.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:46.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:46.623 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:46.667 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:46.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:46.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:46.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:46.802 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:47.253 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:47.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:47.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:47.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:47.440 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:47.442 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:47.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:47.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:47.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.519 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: 1.7 scrub starts 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: 1.7 scrub ok 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: pgmap v627: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 unknown, 1 active+remapped, 85 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: 1.c scrub starts 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[215646]: 1.c scrub ok 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: 1.7 scrub starts 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: 1.7 scrub ok 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: pgmap v627: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 unknown, 1 active+remapped, 85 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: 1.c scrub starts 2023-09-23T15:16:47.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:47 smithi017 ceph-mon[222878]: 1.c scrub ok 2023-09-23T15:16:47.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:47.580 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: osdmap e226: 8 total, 8 up, 7 in 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: failed to encode map e226 with expected crc 2023-09-23T15:16:47.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: 1.7 scrub starts 2023-09-23T15:16:47.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: 1.7 scrub ok 2023-09-23T15:16:47.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: pgmap v627: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 unknown, 1 active+remapped, 85 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2023-09-23T15:16:47.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: 1.c scrub starts 2023-09-23T15:16:47.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:47 smithi027 ceph-mon[195233]: 1.c scrub ok 2023-09-23T15:16:47.758 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:16:47.758 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:16:47.802 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:47.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:47.806 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:47.906 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:47.940 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:48.035 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:48] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.43.0" 2023-09-23T15:16:48.367 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:16:48.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:48.368 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:48.794 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:48.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:48.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:48.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:16:48.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:16:48.897 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:48.927 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:48.976 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:48.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:48.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:49.019 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:49.019 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:49.038 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:49.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:49.078 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:49.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:49.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:49.109 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:49.116 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:49.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.136 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.137 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.137 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:16:49.210 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:49.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:49.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:49.228 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:16:49.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:49.343 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: pgmap v629: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 active+remapped, 86 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: 1.3 scrub starts 2023-09-23T15:16:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: 1.3 scrub ok 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: 1.1 scrub ok 2023-09-23T15:16:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1215663310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: pgmap v629: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 active+remapped, 86 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: 1.3 scrub starts 2023-09-23T15:16:49.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: 1.3 scrub ok 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: 1.1 scrub ok 2023-09-23T15:16:49.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1215663310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3796693213' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: osdmap e227: 8 total, 8 up, 7 in 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: pgmap v629: 109 pgs: 13 active+undersized+remapped, 2 creating+activating, 3 activating+undersized+remapped, 4 creating+peering, 1 active+remapped, 86 active+clean; 4.5 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail; 124 KiB/s, 0 objects/s recovering 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: 1.3 scrub starts 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: 1.3 scrub ok 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: failed to encode map e227 with expected crc 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: 1.1 scrub ok 2023-09-23T15:16:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1215663310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:49.873 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:49.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:49.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:49.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:50.009 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:50.155 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:50.158 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:50.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:50.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:50.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:50.295 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:50.447 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:50.449 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:50 smithi027 ceph-mon[195233]: 1.e scrub starts 2023-09-23T15:16:50.449 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:50 smithi027 ceph-mon[195233]: 1.e scrub ok 2023-09-23T15:16:50.449 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:50 smithi027 ceph-mon[195233]: 1.f scrub starts 2023-09-23T15:16:50.449 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:50 smithi027 ceph-mon[195233]: 1.f scrub ok 2023-09-23T15:16:50.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:50.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:50.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:50.583 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[215646]: 1.e scrub starts 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[215646]: 1.e scrub ok 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[215646]: 1.f scrub starts 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[215646]: 1.f scrub ok 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[222878]: 1.e scrub starts 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[222878]: 1.e scrub ok 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[222878]: 1.f scrub starts 2023-09-23T15:16:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:50 smithi017 ceph-mon[222878]: 1.f scrub ok 2023-09-23T15:16:50.734 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:50.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:50.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:50.869 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:50.917 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:50.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:50.920 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:51.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:16:51.047 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:51.296 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:16:51.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:51.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:16:51.383 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T15:16:51.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:51.429 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:51.474 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:16:51.474 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:16:51.475 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:16:51.475 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:16:51.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: 1.6 scrub ok 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: pgmap v630: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: 1.b scrub starts 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: 1.b scrub ok 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1960105444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: 1.6 scrub ok 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: pgmap v630: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: 1.b scrub starts 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: 1.b scrub ok 2023-09-23T15:16:51.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1960105444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: 1.6 scrub ok 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: pgmap v630: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: 1.b scrub starts 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: 1.b scrub ok 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1960105444' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:16:51.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:16:52.113 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:16:52.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:16:52.185 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:52.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:52.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:52.209 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:52.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:52.318 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:52.371 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:52.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:52.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:52.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:52.478 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:16:52.482 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:16:52.482 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":820,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":228,"num_osds":8,"num_up_osds":8,"osd_up_since":1695481961,"num_in_osds":7,"osd_in_since":1695481893,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":109}],"num_pgs":109,"num_pools":5,"num_objects":28,"data_bytes":4969095,"bytes_used":1854648320,"bytes_avail":670110601216,"bytes_total":671965249536},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T15:16:30.439961+0000","services":{}},"progress_events":{"0334783f-841f-4596-84fe-9d12c2639bb3":{"message":"Global Recovery Event (7m)\n [=====================.......] (remaining: 2m)","progress":0.77981650829315186,"add_to_ceph_s":true},"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T15:16:52.509 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:52.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: 1.d scrub ok 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: failed to encode map e228 with expected crc 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: 1.13 scrub starts 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[222878]: 1.13 scrub ok 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: 1.d scrub ok 2023-09-23T15:16:52.604 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: failed to encode map e228 with expected crc 2023-09-23T15:16:52.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:16:52.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T15:16:52.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: 1.13 scrub starts 2023-09-23T15:16:52.605 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:52 smithi017 ceph-mon[215646]: 1.13 scrub ok 2023-09-23T15:16:52.672 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: 1.d scrub ok 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: failed to encode map e228 with expected crc 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: osdmap e228: 8 total, 8 up, 7 in 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: 1.13 scrub starts 2023-09-23T15:16:52.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:52 smithi027 ceph-mon[195233]: 1.13 scrub ok 2023-09-23T15:16:53.064 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T15:16:53.067 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:16:53.067 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:16:53.117 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:16:53.118 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:16:53.118 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_422824 2023-09-23T15:16:53.118 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:16:53.118 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2023-09-23T15:16:53.471 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:53.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:53.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:53.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:53.600 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T15:16:53.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:16:53.617 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:53.633 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:53.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:16:53.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[222878]: 1.5 scrub ok 2023-09-23T15:16:53.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[222878]: pgmap v632: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:53.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4071246230' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:53.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:16:53.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[215646]: 1.5 scrub ok 2023-09-23T15:16:53.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[215646]: pgmap v632: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:53.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4071246230' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:53.727 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:53 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:16:53.727 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:53 smithi027 ceph-mon[195233]: 1.5 scrub ok 2023-09-23T15:16:53.727 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:53 smithi027 ceph-mon[195233]: pgmap v632: 109 pgs: 109 active+clean; 4.7 MiB data, 1.7 GiB used, 624 GiB / 626 GiB avail 2023-09-23T15:16:53.727 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4071246230' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:16:53.749 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:53.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:53.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:53.862 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:53.955 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:53.978 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:16:53.979 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:16:53.979 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:16:53.979 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:16:53.979 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:16:54.120 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 2702 2686 167.517 167.875 0.00473496 0.00594301 2023-09-23T15:16:54.219 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:54.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:54.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:54.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:54.367 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:54.428 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:54.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:54.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:54.531 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:54.559 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:54.602 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:54.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:54.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:54.706 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:16:54.744 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:54.998 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:16:55.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:55.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:16:55.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:16:55.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:16:55.120 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 15 4823 4808 150.083 132.625 0.00763613 0.00664822 2023-09-23T15:16:55.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:55.224 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:16:55.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:55.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:16:55.330 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:55.375 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:55.456 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:55.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:55.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:55.534 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:16:55.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:16:55.589 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:16:55.630 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:55.811 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:55 smithi027 ceph-mon[195233]: pgmap v633: 109 pgs: 109 active+clean; 76 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 8.7 MiB/s wr, 128 op/s 2023-09-23T15:16:55.812 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1774665751' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:55.812 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[215646]: pgmap v633: 109 pgs: 109 active+clean; 76 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 8.7 MiB/s wr, 128 op/s 2023-09-23T15:16:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1774665751' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:55.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:55.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[222878]: pgmap v633: 109 pgs: 109 active+clean; 76 MiB data, 1.8 GiB used, 624 GiB / 626 GiB avail; 8.7 MiB/s wr, 128 op/s 2023-09-23T15:16:55.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1774665751' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:55.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:16:56.066 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:16:56.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:56.076 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:16:56.120 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 7515 7499 156.103 168.188 0.00467819 0.00639369 2023-09-23T15:16:56.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:56.244 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: 1.14 scrub starts 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: 1.14 scrub ok 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:56.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T15:16:56.508 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.508 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:56 smithi027 ceph-mon[195233]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.509 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: 1.14 scrub starts 2023-09-23T15:16:56.509 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: 1.14 scrub ok 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[215646]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: 1.14 scrub starts 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: 1.14 scrub ok 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.510 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:16:56.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: osdmap e229: 8 total, 8 up, 7 in 2023-09-23T15:16:56.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: failed to encode map e229 with expected crc 2023-09-23T15:16:56.511 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:56 smithi017 ceph-mon[222878]: failed to encode map e229 with expected crc 2023-09-23T15:16:57.121 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 9269 9253 144.46 109.625 0.00260461 0.00691264 2023-09-23T15:16:57.215 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:57.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:57.325 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:57.365 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:57.376 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:16:57.431 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:57.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:57.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:57.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:57.597 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:57.698 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106167 5.c 5:3b2646bb:::benchmark_data_smithi017_288048_object106166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 476.98514263800001, 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26369647800000001, 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.699 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106167 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710225+0000", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710224+0000", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710278+0000", 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4126000000000003e-05 2023-09-23T15:16:57.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710288+0000", 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4380000000000001e-06 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.710293+0000", 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9969999999999998e-06 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712898+0000", 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026053460000000001 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713002+0000", 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103749 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713127+0000", 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012499599999999999 2023-09-23T15:16:57.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719754+0000", 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066268890000000004 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973752+0000", 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25399865999999999 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973807+0000", 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4224999999999997e-05 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973921+0000", 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114744 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106172 5.c 5:3a30f215:::benchmark_data_smithi017_288048_object106171:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 476.98239776399998, 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26115257600000003, 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106172 2023-09-23T15:16:57.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712970+0000", 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.712969+0000", 2023-09-23T15:16:57.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713027+0000", 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7998999999999998e-05 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713038+0000", 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1790999999999999e-05 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.713046+0000", 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.323e-06 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716305+0000", 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032597160000000002 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716430+0000", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124528 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.716566+0000", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135735 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723883+0000", 2023-09-23T15:16:57.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073173500000000002 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.973975+0000", 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.250091695 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974026+0000", 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1768000000000003e-05 2023-09-23T15:16:57.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.974122+0000", 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5847999999999994e-05 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106171 5.a e134/114 5:591ee17c:::benchmark_data_smithi017_288048_object106170:head v 134'6617, mlcod=134'6615)", 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:16:57.713 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 476.97582293200003, 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.264097527, 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719545+0000", 2023-09-23T15:16:57.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719544+0000", 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719581+0000", 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6968999999999998e-05 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719591+0000", 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9680000000000001e-06 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719596+0000", 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4990000000000002e-06 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719639+0000", 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2762000000000002e-05 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.719657+0000", 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8224999999999999e-05 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983596+0000", 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26393868399999998 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983642+0000", 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6028e-05 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106177 5.a e134/114 5:561ec9dc:::benchmark_data_smithi017_288048_object106176:head v 134'6618, mlcod=134'6615)", 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 476.97145599499999, 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25975826499999999, 2023-09-23T15:16:57.718 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723911+0000", 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723910+0000", 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723946+0000", 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6072000000000003e-05 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723957+0000", 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0461e-05 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.723963+0000", 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9569999999999999e-06 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725450+0000", 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001486769 2023-09-23T15:16:57.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725471+0000", 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1826000000000001e-05 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983647+0000", 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25817548600000001 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983670+0000", 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2768e-05 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:106178 5.a e134/114 5:56b4f3e4:::benchmark_data_smithi017_288048_object106177:head v 134'6619, mlcod=134'6615)", 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 476.97012638500001, 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25845162399999999, 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.723 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725241+0000", 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725276+0000", 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5089000000000002e-05 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725287+0000", 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1552999999999999e-05 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.725293+0000", 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8449999999999999e-06 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977314+0000", 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.252021461 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.977334+0000", 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9160999999999998e-05 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983673+0000", 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006339028 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983693+0000", 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0047e-05 2023-09-23T15:16:57.727 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166941 5.1 5:8b30db7e:::benchmark_data_smithi017_288048_object55172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.60901528400001, 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40214831299999998, 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166941 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:16:57.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086351+0000", 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086356+0000", 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1039999999999998e-06 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086368+0000", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1618e-05 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086374+0000", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5490000000000003e-06 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086536+0000", 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161743 2023-09-23T15:16:57.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086652+0000", 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011594 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086770+0000", 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011798600000000001 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091315+0000", 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045446310000000004 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488382+0000", 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39706780000000003 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488406+0000", 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3218e-05 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488454+0000", 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8301999999999999e-05 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488501+0000", 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6573000000000003e-05 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166942 5.e 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.60898094999999, 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40238844899999998, 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166942 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086386+0000", 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086389+0000", 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5830000000000001e-06 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086395+0000", 2023-09-23T15:16:57.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086399+0000", 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2300000000000002e-06 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086603+0000", 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020372899999999999 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086701+0000", 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8332000000000007e-05 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086804+0000", 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103294 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093768+0000", 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006964034 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488259+0000", 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39449091200000003 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488702+0000", 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00044246200000000002 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488734+0000", 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2069000000000001e-05 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488775+0000", 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1278e-05 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166947 5.e 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.60851728799997, 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40205635099999998, 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166947 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086849+0000", 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086854+0000", 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6290000000000001e-06 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086866+0000", 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2478e-05 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086873+0000", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1299999999999998e-06 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090167+0000", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032944319999999999 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090246+0000", 2023-09-23T15:16:57.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9369999999999994e-05 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090328+0000", 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1706999999999995e-05 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488279+0000", 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397950631 2023-09-23T15:16:57.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488321+0000", 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2725999999999997e-05 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488813+0000", 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000491991 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488869+0000", 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5350999999999999e-05 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488907+0000", 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7762999999999997e-05 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166944 5.0 e147/132 5:0733dbd2:::benchmark_data_smithi017_288048_object55175:head v 147'10299, mlcod=147'10297)", 2023-09-23T15:16:57.748 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.60828688999999, 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40144166599999997, 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087080+0000", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087086+0000", 2023-09-23T15:16:57.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5890000000000002e-06 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087096+0000", 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0434999999999999e-05 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087101+0000", 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4890000000000003e-06 2023-09-23T15:16:57.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087141+0000", 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9147999999999999e-05 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087162+0000", 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0928e-05 2023-09-23T15:16:57.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099400+0000", 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012238123 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488522+0000", 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38912261599999998 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.754 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166953 5.c 5:3faa0b3c:::benchmark_data_smithi017_288048_object55184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.604240516, 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39750094200000002, 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166953 2023-09-23T15:16:57.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091126+0000", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091130+0000", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.014e-06 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091144+0000", 2023-09-23T15:16:57.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4717e-05 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091149+0000", 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9590000000000003e-06 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091258+0000", 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108806 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091386+0000", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012736 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091496+0000", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109987 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488306+0000", 2023-09-23T15:16:57.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39681003300000001 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488328+0000", 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2175999999999999e-05 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488380+0000", 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1913999999999998e-05 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488628+0000", 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024825900000000002 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166945 5.0 e147/132 5:0c8902eb:::benchmark_data_smithi017_288048_object55176:head v 147'10300, mlcod=147'10297)", 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.60318409299998, 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39637069699999999, 2023-09-23T15:16:57.761 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092188+0000", 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9760000000000003e-06 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092197+0000", 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2019999999999993e-06 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092203+0000", 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8019999999999997e-06 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092271+0000", 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8647999999999997e-05 2023-09-23T15:16:57.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092292+0000", 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0990999999999999e-05 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488527+0000", 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39623427999999999 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488554+0000", 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7458e-05 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166939 5.f e147/114 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head v 147'10605, mlcod=147'10602)", 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59901231700002, 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40201396099999998, 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.766 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096354+0000", 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096363+0000", 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2139999999999996e-06 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096374+0000", 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1022000000000001e-05 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096379+0000", 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488409+0000", 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39202973600000002 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488494+0000", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5271000000000004e-05 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498336+0000", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098418459999999996 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498369+0000", 2023-09-23T15:16:57.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2783000000000001e-05 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166950 5.f e147/114 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head v 147'10606, mlcod=147'10602)", 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59896452300001, 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40582205900000001, 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096406+0000", 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4649999999999999e-06 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096412+0000", 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7450000000000001e-06 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096415+0000", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5650000000000002e-06 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493677+0000", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397262008 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493697+0000", 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9083e-05 2023-09-23T15:16:57.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502182+0000", 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084849629999999999 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502225+0000", 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.35e-05 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166955 5.5 e147/132 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head v 147'10623, mlcod=147'10621)", 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59892730899998, 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.410760976, 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096443+0000", 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1700000000000001e-06 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096449+0000", 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096453+0000", 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4520000000000002e-06 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488252+0000", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39179915599999998 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488294+0000", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2344e-05 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507165+0000", 2023-09-23T15:16:57.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018870505999999999 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507201+0000", 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6368999999999997e-05 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166947 5.e e147/114 ondisk, result = 0)", 2023-09-23T15:16:57.780 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59890160600003, 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39185894700000001, 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096468+0000", 2023-09-23T15:16:57.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.357e-06 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096473+0000", 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2220000000000001e-06 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096477+0000", 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.399e-06 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488307+0000", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39183000600000001 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488315+0000", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488325+0000", 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0294e-05 2023-09-23T15:16:57.784 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166953 5.c e147/132 ondisk, result = 0)", 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59888354399999, 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39182831099999998, 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:16:57.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096485+0000", 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6139999999999999e-06 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096490+0000", 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8550000000000001e-06 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096493+0000", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9909999999999998e-06 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488288+0000", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.391795069 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488298+0000", 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4709999999999996e-06 2023-09-23T15:16:57.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488312+0000", 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4516e-05 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166941 5.1 e147/114 ondisk, result = 0)", 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:16:57.789 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.59499087199998, 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38801138200000002, 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:16:57.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100376+0000", 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100382+0000", 2023-09-23T15:16:57.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0780000000000001e-06 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488291+0000", 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38790906000000003 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488299+0000", 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8660000000000006e-06 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488367+0000", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8133000000000006e-05 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488376+0000", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.935e-06 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488388+0000", 2023-09-23T15:16:57.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2139e-05 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78602 6.e 6:7eeb81e7:::benchmark_data_smithi017_350392_object78601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.146767689, 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6491542219999999, 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.794 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78602 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548599+0000", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548629+0000", 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9839999999999999e-05 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548639+0000", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6940000000000007e-06 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548644+0000", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8779999999999997e-06 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548691+0000", 2023-09-23T15:16:57.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7481000000000002e-05 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548837+0000", 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014598700000000001 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548993+0000", 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015569999999999999 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565316+0000", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016322705 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565327+0000", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0994e-05 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847765+0000", 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28243886899999998 2023-09-23T15:16:57.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847776+0000", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0519999999999999e-05 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197689+0000", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3499132 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197754+0000", 2023-09-23T15:16:57.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78611 6.e 6:7d1942e2:::benchmark_data_smithi017_350392_object78610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 256.13780835599999, 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733330400000002, 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.801 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78611 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:16:57.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557557+0000", 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557602+0000", 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4189e-05 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557615+0000", 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3869999999999999e-05 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557627+0000", 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1229999999999999e-05 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565352+0000", 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077249190000000002 2023-09-23T15:16:57.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565359+0000", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0609999999999998e-06 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847801+0000", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28244210600000003 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847808+0000", 2023-09-23T15:16:57.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5240000000000003e-06 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201122+0000", 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.353314205 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201310+0000", 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018789999999999999 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201482+0000", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017193500000000001 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.204567+0000", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030849520000000002 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.210867+0000", 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062997979999999997 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228500+0000", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017632618999999999 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228573+0000", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3799000000000003e-05 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.230892+0000", 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023187149999999998 2023-09-23T15:16:57.808 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:147647 6.2 6:4814bf33:::benchmark_data_smithi017_350392_object47256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.75644704800001, 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25844827100000001, 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:16:57.809 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 147647 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938919+0000", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938928+0000", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938946+0000", 2023-09-23T15:16:57.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8111e-05 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938954+0000", 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1780000000000003e-06 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938996+0000", 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2521999999999997e-05 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939088+0000", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1113000000000007e-05 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939182+0000", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4487000000000005e-05 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.942153+0000", 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029708239999999999 2023-09-23T15:16:57.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.143408+0000", 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20125490800000001 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197272+0000", 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053864243999999999 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197325+0000", 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2802e-05 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197369+0000", 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3869000000000003e-05 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:16:57.815 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:16:57.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:57.817 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:16:57.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:16:57.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:16:57.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:16:57.868 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:16:57.909 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:16:57.917 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:16:57.946 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:16:57.957 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:57.984 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:16:58.019 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:16:58.021 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:57 smithi027 ceph-mon[195233]: pgmap v635: 109 pgs: 109 active+clean; 190 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 356 op/s 2023-09-23T15:16:58.021 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3219850881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:58.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:58.022 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:16:58.026 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:16:58.064 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:16:58.074 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.074 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.074 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.074 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.075 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:16:58.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:16:58.126 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 11463 11447 142.842 137.125 0.00349478 0.00699164 2023-09-23T15:16:58.176 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:16:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:16:58] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.43.0" 2023-09-23T15:16:58.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:57 smithi017 ceph-mon[215646]: pgmap v635: 109 pgs: 109 active+clean; 190 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 356 op/s 2023-09-23T15:16:58.176 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3219850881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:58.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:57 smithi017 ceph-mon[222878]: pgmap v635: 109 pgs: 109 active+clean; 190 MiB data, 2.1 GiB used, 624 GiB / 626 GiB avail; 23 MiB/s wr, 356 op/s 2023-09-23T15:16:58.177 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3219850881' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:16:58.207 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:58.576 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:16:58.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:58.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:16:58.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:16:58.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:16:58.722 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:58.801 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:16:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:16:58.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:16:58.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:16:58.965 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:58.965 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:58.970 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:16:59.127 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 13456 13440 139.785 124.562 0.00680096 0.00714651 2023-09-23T15:16:59.178 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:16:59.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:16:59.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:59.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:59.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:16:59.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:16:59.969 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:16:59 smithi027 ceph-mon[195233]: pgmap v636: 109 pgs: 109 active+clean; 223 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s 2023-09-23T15:17:00.127 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 14987 14971 133.492 95.6875 0.00341144 0.00748308 2023-09-23T15:17:00.340 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:00.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:00.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:00.349 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:16:59 smithi017 ceph-mon[215646]: pgmap v636: 109 pgs: 109 active+clean; 223 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s 2023-09-23T15:17:00.349 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:16:59 smithi017 ceph-mon[222878]: pgmap v636: 109 pgs: 109 active+clean; 223 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s 2023-09-23T15:17:00.450 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:00.492 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:00.564 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:00.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:00.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:00.730 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:00.748 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:00.872 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:00.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:00.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:00.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:01.014 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:01.014 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:01.017 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:01.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 16946 16930 132.103 122.438 0.00396636 0.00756126 2023-09-23T15:17:01.228 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:01.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:01.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:01.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:01.367 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:01.423 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:01.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:01.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:01.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:01.542 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:17:01.544 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:17:01.559 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:01.604 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:01.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:01.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:01.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:01.764 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:01.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:17:02.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 18511 18495 128.295 97.8125 0.00155055 0.00774049 2023-09-23T15:17:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:01 smithi027 ceph-mon[195233]: pgmap v637: 109 pgs: 109 active+clean; 851 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T15:17:02.414 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:02.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:01 smithi017 ceph-mon[222878]: pgmap v637: 109 pgs: 109 active+clean; 851 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T15:17:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:01 smithi017 ceph-mon[215646]: pgmap v637: 109 pgs: 109 active+clean; 851 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 93 MiB/s wr, 1.48k op/s 2023-09-23T15:17:02.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:17:03.036 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:03.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:03.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:03.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 20066 20050 125.185 97.1875 0.00242884 0.00796137 2023-09-23T15:17:03.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:03.170 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:03.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:02 smithi017 ceph-mon[215646]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:03.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:02 smithi017 ceph-mon[222878]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:03.215 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:03.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:03.319 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:03.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:02 smithi027 ceph-mon[195233]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:03.346 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:03.390 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:03.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:03.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:03.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:03.581 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":637,"stamp":"2023-09-23T15:17:00.448202+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":892213171,"num_objects":13569,"num_object_clones":0,"num_object_copies":40707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13569,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":14448,"num_write_kb":879349,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4724630,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":17954,"ondisk_log_size":17954,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4160532,"kb_used_data":2388508,"kb_used_omap":30,"kb_used_meta":1771873,"kb_avail":652055532,"statfs":{"total":671965249536,"available":667704864768,"internally_reserved":0,"allocated":2445832192,"data_stored":2439620673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1814398552},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":887244076,"num_objects":13541,"num_object_clones":0,"num_object_copies":40623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13541,"num_whiteouts":0,"num_read":-147,"num_read_kb":-277,"num_write":13468,"num_write_kb":865891,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-536699,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.071718"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":223,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381904+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:16:51.381904+0000","last_peered":"2023-09-23T15:16:51.381904+0000","last_clean":"2023-09-23T15:16:51.381904+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:16:51.381904+0000","last_undegraded":"2023-09-23T15:16:51.381904+0000","last_fullsized":"2023-09-23T15:16:51.381904+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":158,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465485+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:16:51.465485+0000","last_peered":"2023-09-23T15:16:51.465485+0000","last_clean":"2023-09-23T15:16:51.465485+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:16:51.465485+0000","last_undegraded":"2023-09-23T15:16:51.465485+0000","last_fullsized":"2023-09-23T15:16:51.465485+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":106,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.377994+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:16:51.377994+0000","last_peered":"2023-09-23T15:16:51.377994+0000","last_clean":"2023-09-23T15:16:51.377994+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:16:51.377994+0000","last_undegraded":"2023-09-23T15:16:51.377994+0000","last_fullsized":"2023-09-23T15:16:51.377994+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":195,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.606097+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:16:51.606097+0000","last_peered":"2023-09-23T15:16:51.606097+0000","last_clean":"2023-09-23T15:16:51.606097+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:16:51.606097+0000","last_undegraded":"2023-09-23T15:16:51.606097+0000","last_fullsized":"2023-09-23T15:16:51.606097+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":227,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464594+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:16:51.464594+0000","last_peered":"2023-09-23T15:16:51.464594+0000","last_clean":"2023-09-23T15:16:51.464594+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:16:51.464594+0000","last_undegraded":"2023-09-23T15:16:51.464594+0000","last_fullsized":"2023-09-23T15:16:51.464594+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.606031+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:16:51.606031+0000","last_peered":"2023-09-23T15:16:51.606031+0000","last_clean":"2023-09-23T15:16:51.606031+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:16:51.606031+0000","last_undegraded":"2023-09-23T15:16:51.606031+0000","last_fullsized":"2023-09-23T15:16:51.606031+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":174,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605993+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:16:51.605993+0000","last_peered":"2023-09-23T15:16:51.605993+0000","last_clean":"2023-09-23T15:16:51.605993+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:16:51.605993+0000","last_undegraded":"2023-09-23T15:16:51.605993+0000","last_fullsized":"2023-09-23T15:16:51.605993+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464692+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:16:51.464692+0000","last_peered":"2023-09-23T15:16:51.464692+0000","last_clean":"2023-09-23T15:16:51.464692+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:16:51.464692+0000","last_undegraded":"2023-09-23T15:16:51.464692+0000","last_fullsized":"2023-09-23T15:16:51.464692+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":527,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.370503+0000","last_change":"2023-09-23T15:16:55.370503+0000","last_active":"2023-09-23T15:16:55.370503+0000","last_peered":"2023-09-23T15:16:55.370503+0000","last_clean":"2023-09-23T15:16:55.370503+0000","last_became_active":"2023-09-23T15:16:46.275526+0000","last_became_peered":"2023-09-23T15:16:46.275526+0000","last_unstale":"2023-09-23T15:16:55.370503+0000","last_undegraded":"2023-09-23T15:16:55.370503+0000","last_fullsized":"2023-09-23T15:16:55.370503+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:54.310407+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":219,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605839+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:16:51.605839+0000","last_peered":"2023-09-23T15:16:51.605839+0000","last_clean":"2023-09-23T15:16:51.605839+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:16:51.605839+0000","last_undegraded":"2023-09-23T15:16:51.605839+0000","last_fullsized":"2023-09-23T15:16:51.605839+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464721+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:16:51.464721+0000","last_peered":"2023-09-23T15:16:51.464721+0000","last_clean":"2023-09-23T15:16:51.464721+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:16:51.464721+0000","last_undegraded":"2023-09-23T15:16:51.464721+0000","last_fullsized":"2023-09-23T15:16:51.464721+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":92,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465398+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:16:51.465398+0000","last_peered":"2023-09-23T15:16:51.465398+0000","last_clean":"2023-09-23T15:16:51.465398+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:16:51.465398+0000","last_undegraded":"2023-09-23T15:16:51.465398+0000","last_fullsized":"2023-09-23T15:16:51.465398+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378325+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:16:51.378325+0000","last_peered":"2023-09-23T15:16:51.378325+0000","last_clean":"2023-09-23T15:16:51.378325+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:16:51.378325+0000","last_undegraded":"2023-09-23T15:16:51.378325+0000","last_fullsized":"2023-09-23T15:16:51.378325+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":213,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382190+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:16:51.382190+0000","last_peered":"2023-09-23T15:16:51.382190+0000","last_clean":"2023-09-23T15:16:51.382190+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:16:51.382190+0000","last_undegraded":"2023-09-23T15:16:51.382190+0000","last_fullsized":"2023-09-23T15:16:51.382190+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":204,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382039+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:16:51.382039+0000","last_peered":"2023-09-23T15:16:51.382039+0000","last_clean":"2023-09-23T15:16:51.382039+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:16:51.382039+0000","last_undegraded":"2023-09-23T15:16:51.382039+0000","last_fullsized":"2023-09-23T15:16:51.382039+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":535,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605613+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:16:51.605613+0000","last_peered":"2023-09-23T15:16:51.605613+0000","last_clean":"2023-09-23T15:16:51.605613+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:16:51.605613+0000","last_undegraded":"2023-09-23T15:16:51.605613+0000","last_fullsized":"2023-09-23T15:16:51.605613+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":91,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382090+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:16:51.382090+0000","last_peered":"2023-09-23T15:16:51.382090+0000","last_clean":"2023-09-23T15:16:51.382090+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:16:51.382090+0000","last_undegraded":"2023-09-23T15:16:51.382090+0000","last_fullsized":"2023-09-23T15:16:51.382090+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":220,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.466320+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:16:51.466320+0000","last_peered":"2023-09-23T15:16:51.466320+0000","last_clean":"2023-09-23T15:16:51.466320+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:16:51.466320+0000","last_undegraded":"2023-09-23T15:16:51.466320+0000","last_fullsized":"2023-09-23T15:16:51.466320+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":528,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.604976+0000","last_change":"2023-09-23T15:16:46.276165+0000","last_active":"2023-09-23T15:16:51.604976+0000","last_peered":"2023-09-23T15:16:51.604976+0000","last_clean":"2023-09-23T15:16:51.604976+0000","last_became_active":"2023-09-23T15:16:46.275685+0000","last_became_peered":"2023-09-23T15:16:46.275685+0000","last_unstale":"2023-09-23T15:16:51.604976+0000","last_undegraded":"2023-09-23T15:16:51.604976+0000","last_fullsized":"2023-09-23T15:16:51.604976+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:36.377771+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":194,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605553+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:16:51.605553+0000","last_peered":"2023-09-23T15:16:51.605553+0000","last_clean":"2023-09-23T15:16:51.605553+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:16:51.605553+0000","last_undegraded":"2023-09-23T15:16:51.605553+0000","last_fullsized":"2023-09-23T15:16:51.605553+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":197,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605452+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:16:51.605452+0000","last_peered":"2023-09-23T15:16:51.605452+0000","last_clean":"2023-09-23T15:16:51.605452+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:16:51.605452+0000","last_undegraded":"2023-09-23T15:16:51.605452+0000","last_fullsized":"2023-09-23T15:16:51.605452+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":482,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.467653+0000","last_change":"2023-09-23T15:16:51.467653+0000","last_active":"2023-09-23T15:16:51.467653+0000","last_peered":"2023-09-23T15:16:51.467653+0000","last_clean":"2023-09-23T15:16:51.467653+0000","last_became_active":"2023-09-23T15:16:46.277151+0000","last_became_peered":"2023-09-23T15:16:46.277151+0000","last_unstale":"2023-09-23T15:16:51.467653+0000","last_undegraded":"2023-09-23T15:16:51.467653+0000","last_fullsized":"2023-09-23T15:16:51.467653+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:46.522265+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":75,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381972+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:16:51.381972+0000","last_peered":"2023-09-23T15:16:51.381972+0000","last_clean":"2023-09-23T15:16:51.381972+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:16:51.381972+0000","last_undegraded":"2023-09-23T15:16:51.381972+0000","last_fullsized":"2023-09-23T15:16:51.381972+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":183,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.606158+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:16:51.606158+0000","last_peered":"2023-09-23T15:16:51.606158+0000","last_clean":"2023-09-23T15:16:51.606158+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:16:51.606158+0000","last_undegraded":"2023-09-23T15:16:51.606158+0000","last_fullsized":"2023-09-23T15:16:51.606158+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":528,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605336+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:16:51.605336+0000","last_peered":"2023-09-23T15:16:51.605336+0000","last_clean":"2023-09-23T15:16:51.605336+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:16:51.605336+0000","last_undegraded":"2023-09-23T15:16:51.605336+0000","last_fullsized":"2023-09-23T15:16:51.605336+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":195,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.604903+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:16:51.604903+0000","last_peered":"2023-09-23T15:16:51.604903+0000","last_clean":"2023-09-23T15:16:51.604903+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:16:51.604903+0000","last_undegraded":"2023-09-23T15:16:51.604903+0000","last_fullsized":"2023-09-23T15:16:51.604903+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":220,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.466042+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:16:51.466042+0000","last_peered":"2023-09-23T15:16:51.466042+0000","last_clean":"2023-09-23T15:16:51.466042+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:16:51.466042+0000","last_undegraded":"2023-09-23T15:16:51.466042+0000","last_fullsized":"2023-09-23T15:16:51.466042+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"229'534","reported_seq":549,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:56.478285+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:16:56.478285+0000","last_peered":"2023-09-23T15:16:56.478285+0000","last_clean":"2023-09-23T15:16:56.478285+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:16:56.478285+0000","last_undegraded":"2023-09-23T15:16:56.478285+0000","last_fullsized":"2023-09-23T15:16:56.478285+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":534,"log_dups_size":0,"ondisk_log_size":534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34996224,"num_objects":534,"num_object_clones":0,"num_object_copies":1602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":534,"num_write_kb":34176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":526,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382160+0000","last_change":"2023-09-23T15:16:51.284220+0000","last_active":"2023-09-23T15:16:51.382160+0000","last_peered":"2023-09-23T15:16:51.382160+0000","last_clean":"2023-09-23T15:16:51.382160+0000","last_became_active":"2023-09-23T15:16:46.276402+0000","last_became_peered":"2023-09-23T15:16:46.276402+0000","last_unstale":"2023-09-23T15:16:51.382160+0000","last_undegraded":"2023-09-23T15:16:51.382160+0000","last_fullsized":"2023-09-23T15:16:51.382160+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.284115+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.284115+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:27.778741+0000","scrub_duration":0.001563902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":103,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382338+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:16:51.382338+0000","last_peered":"2023-09-23T15:16:51.382338+0000","last_clean":"2023-09-23T15:16:51.382338+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:16:51.382338+0000","last_undegraded":"2023-09-23T15:16:51.382338+0000","last_fullsized":"2023-09-23T15:16:51.382338+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":180,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605024+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:16:51.605024+0000","last_peered":"2023-09-23T15:16:51.605024+0000","last_clean":"2023-09-23T15:16:51.605024+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:16:51.605024+0000","last_undegraded":"2023-09-23T15:16:51.605024+0000","last_fullsized":"2023-09-23T15:16:51.605024+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"229'924","reported_seq":939,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.764106+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:16:59.764106+0000","last_peered":"2023-09-23T15:16:59.764106+0000","last_clean":"2023-09-23T15:16:59.764106+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:16:59.764106+0000","last_undegraded":"2023-09-23T15:16:59.764106+0000","last_fullsized":"2023-09-23T15:16:59.764106+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":924,"log_dups_size":0,"ondisk_log_size":924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60555264,"num_objects":924,"num_object_clones":0,"num_object_copies":2772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":924,"num_write_kb":59136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":531,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.466248+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:16:51.466248+0000","last_peered":"2023-09-23T15:16:51.466248+0000","last_clean":"2023-09-23T15:16:51.466248+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:16:51.466248+0000","last_undegraded":"2023-09-23T15:16:51.466248+0000","last_fullsized":"2023-09-23T15:16:51.466248+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":103,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382390+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:16:51.382390+0000","last_peered":"2023-09-23T15:16:51.382390+0000","last_clean":"2023-09-23T15:16:51.382390+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:16:51.382390+0000","last_undegraded":"2023-09-23T15:16:51.382390+0000","last_fullsized":"2023-09-23T15:16:51.382390+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":221,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381870+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:16:51.381870+0000","last_peered":"2023-09-23T15:16:51.381870+0000","last_clean":"2023-09-23T15:16:51.381870+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:16:51.381870+0000","last_undegraded":"2023-09-23T15:16:51.381870+0000","last_fullsized":"2023-09-23T15:16:51.381870+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"229'924","reported_seq":939,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.495655+0000","last_change":"2023-09-23T15:16:44.269684+0000","last_active":"2023-09-23T15:16:59.495655+0000","last_peered":"2023-09-23T15:16:59.495655+0000","last_clean":"2023-09-23T15:16:59.495655+0000","last_became_active":"2023-09-23T15:16:44.269411+0000","last_became_peered":"2023-09-23T15:16:44.269411+0000","last_unstale":"2023-09-23T15:16:59.495655+0000","last_undegraded":"2023-09-23T15:16:59.495655+0000","last_fullsized":"2023-09-23T15:16:59.495655+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":924,"log_dups_size":0,"ondisk_log_size":924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:33.481012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60555264,"num_objects":924,"num_object_clones":0,"num_object_copies":2772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":924,"num_write_kb":59136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":451,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605741+0000","last_change":"2023-09-23T15:16:49.361068+0000","last_active":"2023-09-23T15:16:51.605741+0000","last_peered":"2023-09-23T15:16:51.605741+0000","last_clean":"2023-09-23T15:16:51.605741+0000","last_became_active":"2023-09-23T15:16:46.276691+0000","last_became_peered":"2023-09-23T15:16:46.276691+0000","last_unstale":"2023-09-23T15:16:51.605741+0000","last_undegraded":"2023-09-23T15:16:51.605741+0000","last_fullsized":"2023-09-23T15:16:51.605741+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:16.914979+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":89,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605388+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:16:51.605388+0000","last_peered":"2023-09-23T15:16:51.605388+0000","last_clean":"2023-09-23T15:16:51.605388+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:16:51.605388+0000","last_undegraded":"2023-09-23T15:16:51.605388+0000","last_fullsized":"2023-09-23T15:16:51.605388+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":217,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605843+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:16:51.605843+0000","last_peered":"2023-09-23T15:16:51.605843+0000","last_clean":"2023-09-23T15:16:51.605843+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:16:51.605843+0000","last_undegraded":"2023-09-23T15:16:51.605843+0000","last_fullsized":"2023-09-23T15:16:51.605843+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"229'810","reported_seq":825,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:58.785699+0000","last_change":"2023-09-23T15:16:44.271841+0000","last_active":"2023-09-23T15:16:58.785699+0000","last_peered":"2023-09-23T15:16:58.785699+0000","last_clean":"2023-09-23T15:16:58.785699+0000","last_became_active":"2023-09-23T15:16:44.271547+0000","last_became_peered":"2023-09-23T15:16:44.271547+0000","last_unstale":"2023-09-23T15:16:58.785699+0000","last_undegraded":"2023-09-23T15:16:58.785699+0000","last_fullsized":"2023-09-23T15:16:58.785699+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":810,"log_dups_size":0,"ondisk_log_size":810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:03.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53084160,"num_objects":810,"num_object_clones":0,"num_object_copies":2430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":810,"num_write_kb":51840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":526,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382237+0000","last_change":"2023-09-23T15:16:49.264186+0000","last_active":"2023-09-23T15:16:51.382237+0000","last_peered":"2023-09-23T15:16:51.382237+0000","last_clean":"2023-09-23T15:16:51.382237+0000","last_became_active":"2023-09-23T15:16:46.275415+0000","last_became_peered":"2023-09-23T15:16:46.275415+0000","last_unstale":"2023-09-23T15:16:51.382237+0000","last_undegraded":"2023-09-23T15:16:51.382237+0000","last_fullsized":"2023-09-23T15:16:51.382237+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.264096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.264096+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:25.863800+0000","scrub_duration":0.0040937940000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":188,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605358+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:51.605358+0000","last_peered":"2023-09-23T15:16:51.605358+0000","last_clean":"2023-09-23T15:16:51.605358+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:16:51.605358+0000","last_undegraded":"2023-09-23T15:16:51.605358+0000","last_fullsized":"2023-09-23T15:16:51.605358+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":189,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605724+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:16:51.605724+0000","last_peered":"2023-09-23T15:16:51.605724+0000","last_clean":"2023-09-23T15:16:51.605724+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:16:51.605724+0000","last_undegraded":"2023-09-23T15:16:51.605724+0000","last_fullsized":"2023-09-23T15:16:51.605724+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"229'873","reported_seq":888,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.504123+0000","last_change":"2023-09-23T15:16:44.263685+0000","last_active":"2023-09-23T15:16:59.504123+0000","last_peered":"2023-09-23T15:16:59.504123+0000","last_clean":"2023-09-23T15:16:59.504123+0000","last_became_active":"2023-09-23T15:16:44.263391+0000","last_became_peered":"2023-09-23T15:16:44.263391+0000","last_unstale":"2023-09-23T15:16:59.504123+0000","last_undegraded":"2023-09-23T15:16:59.504123+0000","last_fullsized":"2023-09-23T15:16:59.504123+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":873,"log_dups_size":0,"ondisk_log_size":873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:33.410338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57212928,"num_objects":873,"num_object_clones":0,"num_object_copies":2619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":873,"num_write_kb":55872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":527,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464596+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:16:51.464596+0000","last_peered":"2023-09-23T15:16:51.464596+0000","last_clean":"2023-09-23T15:16:51.464596+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:16:51.464596+0000","last_undegraded":"2023-09-23T15:16:51.464596+0000","last_fullsized":"2023-09-23T15:16:51.464596+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":192,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465642+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:16:51.465642+0000","last_peered":"2023-09-23T15:16:51.465642+0000","last_clean":"2023-09-23T15:16:51.465642+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:16:51.465642+0000","last_undegraded":"2023-09-23T15:16:51.465642+0000","last_fullsized":"2023-09-23T15:16:51.465642+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382192+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:16:51.382192+0000","last_peered":"2023-09-23T15:16:51.382192+0000","last_clean":"2023-09-23T15:16:51.382192+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:16:51.382192+0000","last_undegraded":"2023-09-23T15:16:51.382192+0000","last_fullsized":"2023-09-23T15:16:51.382192+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"229'865","reported_seq":880,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:58.787450+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:16:58.787450+0000","last_peered":"2023-09-23T15:16:58.787450+0000","last_clean":"2023-09-23T15:16:58.787450+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:16:58.787450+0000","last_undegraded":"2023-09-23T15:16:58.787450+0000","last_fullsized":"2023-09-23T15:16:58.787450+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":865,"log_dups_size":0,"ondisk_log_size":865,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56688640,"num_objects":865,"num_object_clones":0,"num_object_copies":2595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":865,"num_write_kb":55360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":533,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.604859+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:16:51.604859+0000","last_peered":"2023-09-23T15:16:51.604859+0000","last_clean":"2023-09-23T15:16:51.604859+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:16:51.604859+0000","last_undegraded":"2023-09-23T15:16:51.604859+0000","last_fullsized":"2023-09-23T15:16:51.604859+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":204,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605656+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:16:51.605656+0000","last_peered":"2023-09-23T15:16:51.605656+0000","last_clean":"2023-09-23T15:16:51.605656+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:16:51.605656+0000","last_undegraded":"2023-09-23T15:16:51.605656+0000","last_fullsized":"2023-09-23T15:16:51.605656+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":221,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382318+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:16:51.382318+0000","last_peered":"2023-09-23T15:16:51.382318+0000","last_clean":"2023-09-23T15:16:51.382318+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:16:51.382318+0000","last_undegraded":"2023-09-23T15:16:51.382318+0000","last_fullsized":"2023-09-23T15:16:51.382318+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"229'851","reported_seq":865,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.163996+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:16:59.163996+0000","last_peered":"2023-09-23T15:16:59.163996+0000","last_clean":"2023-09-23T15:16:59.163996+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:16:59.163996+0000","last_undegraded":"2023-09-23T15:16:59.163996+0000","last_fullsized":"2023-09-23T15:16:59.163996+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":851,"log_dups_size":0,"ondisk_log_size":851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55771136,"num_objects":851,"num_object_clones":0,"num_object_copies":2553,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":851,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":851,"num_write_kb":54464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":482,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378613+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:16:51.378613+0000","last_peered":"2023-09-23T15:16:51.378613+0000","last_clean":"2023-09-23T15:16:51.378613+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:16:51.378613+0000","last_undegraded":"2023-09-23T15:16:51.378613+0000","last_fullsized":"2023-09-23T15:16:51.378613+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":212,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381754+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:16:51.381754+0000","last_peered":"2023-09-23T15:16:51.381754+0000","last_clean":"2023-09-23T15:16:51.381754+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:16:51.381754+0000","last_undegraded":"2023-09-23T15:16:51.381754+0000","last_fullsized":"2023-09-23T15:16:51.381754+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":204,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382455+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:16:51.382455+0000","last_peered":"2023-09-23T15:16:51.382455+0000","last_clean":"2023-09-23T15:16:51.382455+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:16:51.382455+0000","last_undegraded":"2023-09-23T15:16:51.382455+0000","last_fullsized":"2023-09-23T15:16:51.382455+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"229'837","reported_seq":851,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.187353+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:16:59.187353+0000","last_peered":"2023-09-23T15:16:59.187353+0000","last_clean":"2023-09-23T15:16:59.187353+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:16:59.187353+0000","last_undegraded":"2023-09-23T15:16:59.187353+0000","last_fullsized":"2023-09-23T15:16:59.187353+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":837,"log_dups_size":0,"ondisk_log_size":837,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54853632,"num_objects":837,"num_object_clones":0,"num_object_copies":2511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":837,"num_write_kb":53568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":528,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605239+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:16:51.605239+0000","last_peered":"2023-09-23T15:16:51.605239+0000","last_clean":"2023-09-23T15:16:51.605239+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:16:51.605239+0000","last_undegraded":"2023-09-23T15:16:51.605239+0000","last_fullsized":"2023-09-23T15:16:51.605239+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":221,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464594+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:16:51.464594+0000","last_peered":"2023-09-23T15:16:51.464594+0000","last_clean":"2023-09-23T15:16:51.464594+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:16:51.464594+0000","last_undegraded":"2023-09-23T15:16:51.464594+0000","last_fullsized":"2023-09-23T15:16:51.464594+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":106,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378805+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:16:51.378805+0000","last_peered":"2023-09-23T15:16:51.378805+0000","last_clean":"2023-09-23T15:16:51.378805+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:16:51.378805+0000","last_undegraded":"2023-09-23T15:16:51.378805+0000","last_fullsized":"2023-09-23T15:16:51.378805+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"229'868","reported_seq":882,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.352366+0000","last_change":"2023-09-23T15:16:44.271607+0000","last_active":"2023-09-23T15:16:59.352366+0000","last_peered":"2023-09-23T15:16:59.352366+0000","last_clean":"2023-09-23T15:16:59.352366+0000","last_became_active":"2023-09-23T15:16:44.271334+0000","last_became_peered":"2023-09-23T15:16:44.271334+0000","last_unstale":"2023-09-23T15:16:59.352366+0000","last_undegraded":"2023-09-23T15:16:59.352366+0000","last_fullsized":"2023-09-23T15:16:59.352366+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":868,"log_dups_size":0,"ondisk_log_size":868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:58.532783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56885248,"num_objects":868,"num_object_clones":0,"num_object_copies":2604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":868,"num_write_kb":55552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":451,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605962+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:16:51.605962+0000","last_peered":"2023-09-23T15:16:51.605962+0000","last_clean":"2023-09-23T15:16:51.605962+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:16:51.605962+0000","last_undegraded":"2023-09-23T15:16:51.605962+0000","last_fullsized":"2023-09-23T15:16:51.605962+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":406,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515026+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:16:55.515026+0000","last_peered":"2023-09-23T15:16:55.515026+0000","last_clean":"2023-09-23T15:16:55.515026+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:16:55.515026+0000","last_undegraded":"2023-09-23T15:16:55.515026+0000","last_fullsized":"2023-09-23T15:16:55.515026+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605349+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:51.605349+0000","last_peered":"2023-09-23T15:16:51.605349+0000","last_clean":"2023-09-23T15:16:51.605349+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:16:51.605349+0000","last_undegraded":"2023-09-23T15:16:51.605349+0000","last_fullsized":"2023-09-23T15:16:51.605349+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":151,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605180+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:16:51.605180+0000","last_peered":"2023-09-23T15:16:51.605180+0000","last_clean":"2023-09-23T15:16:51.605180+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:16:51.605180+0000","last_undegraded":"2023-09-23T15:16:51.605180+0000","last_fullsized":"2023-09-23T15:16:51.605180+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"229'831","reported_seq":845,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.162708+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:16:59.162708+0000","last_peered":"2023-09-23T15:16:59.162708+0000","last_clean":"2023-09-23T15:16:59.162708+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:16:59.162708+0000","last_undegraded":"2023-09-23T15:16:59.162708+0000","last_fullsized":"2023-09-23T15:16:59.162708+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":831,"log_dups_size":0,"ondisk_log_size":831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54460416,"num_objects":831,"num_object_clones":0,"num_object_copies":2493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":831,"num_write_kb":53184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":526,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605067+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:16:51.605067+0000","last_peered":"2023-09-23T15:16:51.605067+0000","last_clean":"2023-09-23T15:16:51.605067+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:16:51.605067+0000","last_undegraded":"2023-09-23T15:16:51.605067+0000","last_fullsized":"2023-09-23T15:16:51.605067+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":747,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605560+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:16:51.605560+0000","last_peered":"2023-09-23T15:16:51.605560+0000","last_clean":"2023-09-23T15:16:51.605560+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:16:51.605560+0000","last_undegraded":"2023-09-23T15:16:51.605560+0000","last_fullsized":"2023-09-23T15:16:51.605560+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":188,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382021+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:16:51.382021+0000","last_peered":"2023-09-23T15:16:51.382021+0000","last_clean":"2023-09-23T15:16:51.382021+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:16:51.382021+0000","last_undegraded":"2023-09-23T15:16:51.382021+0000","last_fullsized":"2023-09-23T15:16:51.382021+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":122,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514566+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:16:55.514566+0000","last_peered":"2023-09-23T15:16:55.514566+0000","last_clean":"2023-09-23T15:16:55.514566+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:16:55.514566+0000","last_undegraded":"2023-09-23T15:16:55.514566+0000","last_fullsized":"2023-09-23T15:16:55.514566+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"229'870","reported_seq":885,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.488575+0000","last_change":"2023-09-23T15:16:44.269240+0000","last_active":"2023-09-23T15:16:59.488575+0000","last_peered":"2023-09-23T15:16:59.488575+0000","last_clean":"2023-09-23T15:16:59.488575+0000","last_became_active":"2023-09-23T15:16:44.268966+0000","last_became_peered":"2023-09-23T15:16:44.268966+0000","last_unstale":"2023-09-23T15:16:59.488575+0000","last_undegraded":"2023-09-23T15:16:59.488575+0000","last_fullsized":"2023-09-23T15:16:59.488575+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":870,"log_dups_size":0,"ondisk_log_size":870,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:01.694002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57016320,"num_objects":870,"num_object_clones":0,"num_object_copies":2610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":870,"num_write_kb":55680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":535,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605653+0000","last_change":"2023-09-23T15:16:46.276901+0000","last_active":"2023-09-23T15:16:51.605653+0000","last_peered":"2023-09-23T15:16:51.605653+0000","last_clean":"2023-09-23T15:16:51.605653+0000","last_became_active":"2023-09-23T15:16:46.276619+0000","last_became_peered":"2023-09-23T15:16:46.276619+0000","last_unstale":"2023-09-23T15:16:51.605653+0000","last_undegraded":"2023-09-23T15:16:51.605653+0000","last_fullsized":"2023-09-23T15:16:51.605653+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:27.244809+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":842,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.604988+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:16:51.604988+0000","last_peered":"2023-09-23T15:16:51.604988+0000","last_clean":"2023-09-23T15:16:51.604988+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:16:51.604988+0000","last_undegraded":"2023-09-23T15:16:51.604988+0000","last_fullsized":"2023-09-23T15:16:51.604988+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378167+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:16:51.378167+0000","last_peered":"2023-09-23T15:16:51.378167+0000","last_clean":"2023-09-23T15:16:51.378167+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:16:51.378167+0000","last_undegraded":"2023-09-23T15:16:51.378167+0000","last_fullsized":"2023-09-23T15:16:51.378167+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":125,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378716+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:16:51.378716+0000","last_peered":"2023-09-23T15:16:51.378716+0000","last_clean":"2023-09-23T15:16:51.378716+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:16:51.378716+0000","last_undegraded":"2023-09-23T15:16:51.378716+0000","last_fullsized":"2023-09-23T15:16:51.378716+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"229'884","reported_seq":899,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.495358+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:16:59.495358+0000","last_peered":"2023-09-23T15:16:59.495358+0000","last_clean":"2023-09-23T15:16:59.495358+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:16:59.495358+0000","last_undegraded":"2023-09-23T15:16:59.495358+0000","last_fullsized":"2023-09-23T15:16:59.495358+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":884,"log_dups_size":0,"ondisk_log_size":884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57933824,"num_objects":884,"num_object_clones":0,"num_object_copies":2652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":884,"num_write_kb":56576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":516,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605264+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:16:51.605264+0000","last_peered":"2023-09-23T15:16:51.605264+0000","last_clean":"2023-09-23T15:16:51.605264+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:16:51.605264+0000","last_undegraded":"2023-09-23T15:16:51.605264+0000","last_fullsized":"2023-09-23T15:16:51.605264+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":518,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.604917+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:16:51.604917+0000","last_peered":"2023-09-23T15:16:51.604917+0000","last_clean":"2023-09-23T15:16:51.604917+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:16:51.604917+0000","last_undegraded":"2023-09-23T15:16:51.604917+0000","last_fullsized":"2023-09-23T15:16:51.604917+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":144,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465037+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:16:51.465037+0000","last_peered":"2023-09-23T15:16:51.465037+0000","last_clean":"2023-09-23T15:16:51.465037+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:16:51.465037+0000","last_undegraded":"2023-09-23T15:16:51.465037+0000","last_fullsized":"2023-09-23T15:16:51.465037+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":179,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605659+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:16:51.605659+0000","last_peered":"2023-09-23T15:16:51.605659+0000","last_clean":"2023-09-23T15:16:51.605659+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:16:51.605659+0000","last_undegraded":"2023-09-23T15:16:51.605659+0000","last_fullsized":"2023-09-23T15:16:51.605659+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"229'752","reported_seq":767,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:58.789354+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:16:58.789354+0000","last_peered":"2023-09-23T15:16:58.789354+0000","last_clean":"2023-09-23T15:16:58.789354+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:16:58.789354+0000","last_undegraded":"2023-09-23T15:16:58.789354+0000","last_fullsized":"2023-09-23T15:16:58.789354+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":752,"log_dups_size":0,"ondisk_log_size":752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49283072,"num_objects":752,"num_object_clones":0,"num_object_copies":2256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":752,"num_write_kb":48128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":482,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378546+0000","last_change":"2023-09-23T15:16:48.526093+0000","last_active":"2023-09-23T15:16:51.378546+0000","last_peered":"2023-09-23T15:16:51.378546+0000","last_clean":"2023-09-23T15:16:51.378546+0000","last_became_active":"2023-09-23T15:16:46.276681+0000","last_became_peered":"2023-09-23T15:16:46.276681+0000","last_unstale":"2023-09-23T15:16:51.378546+0000","last_undegraded":"2023-09-23T15:16:51.378546+0000","last_fullsized":"2023-09-23T15:16:51.378546+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:34.078052+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":313,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378804+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:16:51.378804+0000","last_peered":"2023-09-23T15:16:51.378804+0000","last_clean":"2023-09-23T15:16:51.378804+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:16:51.378804+0000","last_undegraded":"2023-09-23T15:16:51.378804+0000","last_fullsized":"2023-09-23T15:16:51.378804+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":174,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605457+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:16:51.605457+0000","last_peered":"2023-09-23T15:16:51.605457+0000","last_clean":"2023-09-23T15:16:51.605457+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:16:51.605457+0000","last_undegraded":"2023-09-23T15:16:51.605457+0000","last_fullsized":"2023-09-23T15:16:51.605457+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":204,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605887+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:16:51.605887+0000","last_peered":"2023-09-23T15:16:51.605887+0000","last_clean":"2023-09-23T15:16:51.605887+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:16:51.605887+0000","last_undegraded":"2023-09-23T15:16:51.605887+0000","last_fullsized":"2023-09-23T15:16:51.605887+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"229'950","reported_seq":964,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.717074+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:16:59.717074+0000","last_peered":"2023-09-23T15:16:59.717074+0000","last_clean":"2023-09-23T15:16:59.717074+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:16:59.717074+0000","last_undegraded":"2023-09-23T15:16:59.717074+0000","last_fullsized":"2023-09-23T15:16:59.717074+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":950,"log_dups_size":0,"ondisk_log_size":950,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62259200,"num_objects":950,"num_object_clones":0,"num_object_copies":2850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":950,"num_write_kb":60800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":451,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605548+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:16:51.605548+0000","last_peered":"2023-09-23T15:16:51.605548+0000","last_clean":"2023-09-23T15:16:51.605548+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:16:51.605548+0000","last_undegraded":"2023-09-23T15:16:51.605548+0000","last_fullsized":"2023-09-23T15:16:51.605548+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":850,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605924+0000","last_change":"2023-09-23T15:16:38.973875+0000","last_active":"2023-09-23T15:16:51.605924+0000","last_peered":"2023-09-23T15:16:51.605924+0000","last_clean":"2023-09-23T15:16:51.605924+0000","last_became_active":"2023-09-23T15:16:38.973269+0000","last_became_peered":"2023-09-23T15:16:38.973269+0000","last_unstale":"2023-09-23T15:16:51.605924+0000","last_undegraded":"2023-09-23T15:16:51.605924+0000","last_fullsized":"2023-09-23T15:16:51.605924+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:17:00.522554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":102,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514755+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:16:55.514755+0000","last_peered":"2023-09-23T15:16:55.514755+0000","last_clean":"2023-09-23T15:16:55.514755+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:16:55.514755+0000","last_undegraded":"2023-09-23T15:16:55.514755+0000","last_fullsized":"2023-09-23T15:16:55.514755+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":179,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605937+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:16:51.605937+0000","last_peered":"2023-09-23T15:16:51.605937+0000","last_clean":"2023-09-23T15:16:51.605937+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:16:51.605937+0000","last_undegraded":"2023-09-23T15:16:51.605937+0000","last_fullsized":"2023-09-23T15:16:51.605937+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"229'891","reported_seq":906,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.760983+0000","last_change":"2023-09-23T15:16:44.260057+0000","last_active":"2023-09-23T15:16:59.760983+0000","last_peered":"2023-09-23T15:16:59.760983+0000","last_clean":"2023-09-23T15:16:59.760983+0000","last_became_active":"2023-09-23T15:16:44.259815+0000","last_became_peered":"2023-09-23T15:16:44.259815+0000","last_unstale":"2023-09-23T15:16:59.760983+0000","last_undegraded":"2023-09-23T15:16:59.760983+0000","last_fullsized":"2023-09-23T15:16:59.760983+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":891,"log_dups_size":0,"ondisk_log_size":891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:50.147826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58392576,"num_objects":891,"num_object_clones":0,"num_object_copies":2673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":891,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":891,"num_write_kb":57024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":526,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:52.317618+0000","last_change":"2023-09-23T15:16:52.317545+0000","last_active":"2023-09-23T15:16:52.317618+0000","last_peered":"2023-09-23T15:16:52.317618+0000","last_clean":"2023-09-23T15:16:52.317618+0000","last_became_active":"2023-09-23T15:16:46.275975+0000","last_became_peered":"2023-09-23T15:16:46.275975+0000","last_unstale":"2023-09-23T15:16:52.317618+0000","last_undegraded":"2023-09-23T15:16:52.317618+0000","last_fullsized":"2023-09-23T15:16:52.317618+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:52.317424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:52.317424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:29.771978+0000","scrub_duration":0.001669299,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":205,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465958+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:16:51.465958+0000","last_peered":"2023-09-23T15:16:51.465958+0000","last_clean":"2023-09-23T15:16:51.465958+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:16:51.465958+0000","last_undegraded":"2023-09-23T15:16:51.465958+0000","last_fullsized":"2023-09-23T15:16:51.465958+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":689,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.378349+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:16:51.378349+0000","last_peered":"2023-09-23T15:16:51.378349+0000","last_clean":"2023-09-23T15:16:51.378349+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:16:51.378349+0000","last_undegraded":"2023-09-23T15:16:51.378349+0000","last_fullsized":"2023-09-23T15:16:51.378349+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":210,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464725+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:16:51.464725+0000","last_peered":"2023-09-23T15:16:51.464725+0000","last_clean":"2023-09-23T15:16:51.464725+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:16:51.464725+0000","last_undegraded":"2023-09-23T15:16:51.464725+0000","last_fullsized":"2023-09-23T15:16:51.464725+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"229'877","reported_seq":892,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:59.755584+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:16:59.755584+0000","last_peered":"2023-09-23T15:16:59.755584+0000","last_clean":"2023-09-23T15:16:59.755584+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:16:59.755584+0000","last_undegraded":"2023-09-23T15:16:59.755584+0000","last_fullsized":"2023-09-23T15:16:59.755584+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":877,"log_dups_size":0,"ondisk_log_size":877,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57475072,"num_objects":877,"num_object_clones":0,"num_object_copies":2631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":877,"num_write_kb":56128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":527,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.466163+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:16:51.466163+0000","last_peered":"2023-09-23T15:16:51.466163+0000","last_clean":"2023-09-23T15:16:51.466163+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:16:51.466163+0000","last_undegraded":"2023-09-23T15:16:51.466163+0000","last_fullsized":"2023-09-23T15:16:51.466163+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":705,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605876+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:16:51.605876+0000","last_peered":"2023-09-23T15:16:51.605876+0000","last_clean":"2023-09-23T15:16:51.605876+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:16:51.605876+0000","last_undegraded":"2023-09-23T15:16:51.605876+0000","last_fullsized":"2023-09-23T15:16:51.605876+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":192,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605974+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:16:51.605974+0000","last_peered":"2023-09-23T15:16:51.605974+0000","last_clean":"2023-09-23T15:16:51.605974+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:16:51.605974+0000","last_undegraded":"2023-09-23T15:16:51.605974+0000","last_fullsized":"2023-09-23T15:16:51.605974+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381864+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:16:51.381864+0000","last_peered":"2023-09-23T15:16:51.381864+0000","last_clean":"2023-09-23T15:16:51.381864+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:16:51.381864+0000","last_undegraded":"2023-09-23T15:16:51.381864+0000","last_fullsized":"2023-09-23T15:16:51.381864+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":181,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605844+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:16:51.605844+0000","last_peered":"2023-09-23T15:16:51.605844+0000","last_clean":"2023-09-23T15:16:51.605844+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:16:51.605844+0000","last_undegraded":"2023-09-23T15:16:51.605844+0000","last_fullsized":"2023-09-23T15:16:51.605844+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":229,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605590+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:16:51.605590+0000","last_peered":"2023-09-23T15:16:51.605590+0000","last_clean":"2023-09-23T15:16:51.605590+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:16:51.605590+0000","last_undegraded":"2023-09-23T15:16:51.605590+0000","last_fullsized":"2023-09-23T15:16:51.605590+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":207,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605450+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:16:51.605450+0000","last_peered":"2023-09-23T15:16:51.605450+0000","last_clean":"2023-09-23T15:16:51.605450+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:16:51.605450+0000","last_undegraded":"2023-09-23T15:16:51.605450+0000","last_fullsized":"2023-09-23T15:16:51.605450+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":221,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.382123+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:16:51.382123+0000","last_peered":"2023-09-23T15:16:51.382123+0000","last_clean":"2023-09-23T15:16:51.382123+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:16:51.382123+0000","last_undegraded":"2023-09-23T15:16:51.382123+0000","last_fullsized":"2023-09-23T15:16:51.382123+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":173,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.605280+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:16:51.605280+0000","last_peered":"2023-09-23T15:16:51.605280+0000","last_clean":"2023-09-23T15:16:51.605280+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:16:51.605280+0000","last_undegraded":"2023-09-23T15:16:51.605280+0000","last_fullsized":"2023-09-23T15:16:51.605280+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":204,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381866+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:16:51.381866+0000","last_peered":"2023-09-23T15:16:51.381866+0000","last_clean":"2023-09-23T15:16:51.381866+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:16:51.381866+0000","last_undegraded":"2023-09-23T15:16:51.381866+0000","last_fullsized":"2023-09-23T15:16:51.381866+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":201,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.465977+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:16:51.465977+0000","last_peered":"2023-09-23T15:16:51.465977+0000","last_clean":"2023-09-23T15:16:51.465977+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:16:51.465977+0000","last_undegraded":"2023-09-23T15:16:51.465977+0000","last_fullsized":"2023-09-23T15:16:51.465977+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":215,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.464703+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:16:51.464703+0000","last_peered":"2023-09-23T15:16:51.464703+0000","last_clean":"2023-09-23T15:16:51.464703+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:16:51.464703+0000","last_undegraded":"2023-09-23T15:16:51.464703+0000","last_fullsized":"2023-09-23T15:16:51.464703+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":177,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.606039+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:16:51.606039+0000","last_peered":"2023-09-23T15:16:51.606039+0000","last_clean":"2023-09-23T15:16:51.606039+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:16:51.606039+0000","last_undegraded":"2023-09-23T15:16:51.606039+0000","last_fullsized":"2023-09-23T15:16:51.606039+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":197,"reported_epoch":228,"state":"active+clean","last_fresh":"2023-09-23T15:16:51.381690+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:16:51.381690+0000","last_peered":"2023-09-23T15:16:51.381690+0000","last_clean":"2023-09-23T15:16:51.381690+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:16:51.381690+0000","last_undegraded":"2023-09-23T15:16:51.381690+0000","last_fullsized":"2023-09-23T15:16:51.381690+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":887422976,"num_objects":13541,"num_object_clones":0,"num_object_copies":40623,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":13541,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13541,"num_write_kb":866624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2596732928,"data_stored":2596732928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13541,"ondisk_log_size":13541,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201547,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463183,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":412796,"kb_used_data":240884,"kb_used_omap":0,"kb_used_meta":171904,"kb_avail":93332356,"statfs":{"total":95995035648,"available":95572332544,"internally_reserved":0,"allocated":246665216,"data_stored":245922685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":176029696},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":2.1469999999999998,"15min":2.7120000000000002},"min":{"1min":0.23899999999999999,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":5.2519999999999998,"5min":41.491,"15min":108.39400000000001},"last":1.5700000000000001},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.4649999999999999,"15min":2.9289999999999998},"min":{"1min":0.27200000000000002,"5min":0.223,"15min":0.223},"max":{"1min":8.1029999999999998,"5min":43.151000000000003,"15min":108.452},"last":1.0580000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":2.9489999999999998,"15min":3.23},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.5470000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.73999999999999999},{"interface":"front","average":{"1min":1.1830000000000001,"5min":3.7890000000000001,"15min":3.6669999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.1299999999999999,"5min":42.704000000000001,"15min":42.704000000000001},"last":1.276}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5129999999999999,"5min":5.7489999999999997,"15min":6.0949999999999998},"min":{"1min":0.38900000000000001,"5min":0.255,"15min":0.255},"max":{"1min":68.828000000000003,"5min":112.352,"15min":180.374},"last":1.1659999999999999},{"interface":"front","average":{"1min":5.681,"5min":5.8140000000000001,"15min":6.0709999999999997},"min":{"1min":0.29599999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":68.730999999999995,"5min":117.667,"15min":181.685},"last":0.629}]},{"osd":3,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":7.0979999999999999,"15min":3.1539999999999999},"min":{"1min":0.42699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.1440000000000001,"5min":102.254,"15min":102.254},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.8109999999999999,"5min":7.234,"15min":3.1640000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7030000000000003,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.42399999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0349999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.7150000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.26400000000000001},{"interface":"front","average":{"1min":0.624,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0019999999999998,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.28100000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.0099999999999998,"15min":1.423},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":3.1789999999999998,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.312},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.0249999999999999,"15min":1.4590000000000001},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0510000000000002,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.33100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.82499999999999996,"15min":1.0289999999999999},"min":{"1min":0.36599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":1.8700000000000001,"5min":5.125,"15min":17},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.77000000000000002,"15min":0.98999999999999999},"min":{"1min":0.438,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":1.8180000000000001,"5min":3.754,"15min":7.7809999999999997},"last":0.503}]}]},{"osd":1,"up_from":183,"seq":785979015220,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":438848,"kb_used_data":227128,"kb_used_omap":6,"kb_used_meta":211705,"kb_avail":93306304,"statfs":{"total":95995035648,"available":95545655296,"internally_reserved":0,"allocated":232579072,"data_stored":231489530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":216785995},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":3.1760000000000002,"15min":3.5939999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":14.227,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.53100000000000003},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.4459999999999997,"15min":4.5549999999999997},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":19.044,"5min":19.044,"15min":19.044},"last":5.2830000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":4.3099999999999996,"15min":4.1879999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":33.880000000000003,"5min":33.880000000000003,"15min":33.880000000000003},"last":1.369},{"interface":"front","average":{"1min":2.5299999999999998,"5min":3.7320000000000002,"15min":4.0590000000000002},"min":{"1min":0.30199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":10.843999999999999,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":15.66,"5min":9.3330000000000002,"15min":10.308},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":211.405,"5min":211.405,"15min":211.405},"last":0.40699999999999997},{"interface":"front","average":{"1min":15.292999999999999,"5min":9.8729999999999993,"15min":11.667999999999999},"min":{"1min":0.34799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":210.988,"5min":210.988,"15min":210.988},"last":1.4350000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.952,"5min":2.7469999999999999,"15min":2.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.430999999999999,"5min":11.430999999999999,"15min":11.430999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.887,"15min":3.3340000000000001},"min":{"1min":0.39300000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.750999999999999,"5min":16.779,"15min":16.779},"last":0.45000000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.1099999999999999,"15min":3.556},"min":{"1min":0.39100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.84,"5min":16.166,"15min":16.166},"last":1.1839999999999999},{"interface":"front","average":{"1min":2.956,"5min":3.0569999999999999,"15min":2.9500000000000002},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.487,"5min":11.487,"15min":11.487},"last":1.2090000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.0920000000000001,"15min":3.5699999999999998},"min":{"1min":0.442,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":11.797000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":1.1459999999999999},{"interface":"front","average":{"1min":3.0129999999999999,"5min":3.0529999999999999,"15min":2.8500000000000001},"min":{"1min":0.46500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":11.531000000000001,"5min":12.448,"15min":12.448},"last":0.96499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":3.1099999999999999,"15min":3.141},"min":{"1min":0.495,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":11.561,"5min":11.561,"15min":11.561},"last":1.323},{"interface":"front","average":{"1min":1.4370000000000001,"5min":2.9399999999999999,"15min":3.2160000000000002},"min":{"1min":0.46899999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.801,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.498}]}]},{"osd":0,"up_from":95,"seq":408021893270,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":723400,"kb_used_data":405952,"kb_used_omap":8,"kb_used_meta":317367,"kb_avail":93021752,"statfs":{"total":95995035648,"available":95254274048,"internally_reserved":0,"allocated":415694848,"data_stored":414713520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":324984404},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:16:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":3.048,"15min":3.8769999999999998},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":15.462,"5min":31.510999999999999,"15min":31.510999999999999},"last":3.5779999999999998},{"interface":"front","average":{"1min":2.048,"5min":3.1949999999999998,"15min":4.2050000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.099,"5min":31.378,"15min":31.378},"last":2.589}]},{"osd":2,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":7.133,"15min":3.5539999999999998},"min":{"1min":0.26200000000000001,"5min":0.189,"15min":0.17000000000000001},"max":{"1min":13.593,"5min":103.81,"15min":122.20099999999999},"last":3.8889999999999998},{"interface":"front","average":{"1min":3.4279999999999999,"5min":6.077,"15min":3.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.18099999999999999},"max":{"1min":9.7479999999999993,"5min":108.19499999999999,"15min":122.071},"last":2.2010000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:16:32 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":9.8650000000000002,"15min":3.9460000000000002},"min":{"1min":0.27200000000000002,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":146.315,"5min":146.315,"15min":146.315},"last":4.1989999999999998},{"interface":"front","average":{"1min":17.911999999999999,"5min":10.164999999999999,"15min":4.3300000000000001},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":145.517,"5min":145.517,"15min":145.517},"last":4.2610000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.5230000000000001,"15min":1.649},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.9950000000000001,"5min":31.341999999999999,"15min":31.341999999999999},"last":2.6269999999999998},{"interface":"front","average":{"1min":2.4710000000000001,"5min":3.282,"15min":1.72},"min":{"1min":0.29699999999999999,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":13.566000000000001,"5min":99.168999999999997,"15min":99.168999999999997},"last":2.5619999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.992,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29699999999999999},"max":{"1min":9.6020000000000003,"5min":109.369,"15min":109.369},"last":2.1219999999999999},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.2170000000000001,"15min":1.857},"min":{"1min":0.42999999999999999,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":4.4219999999999997,"5min":109.387,"15min":109.387},"last":2.6960000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.996,"15min":1.752},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":13.529999999999999,"5min":99.114999999999995,"15min":99.114999999999995},"last":2.6099999999999999},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.8650000000000002,"15min":1.677},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":9.6460000000000008,"5min":98.921999999999997,"15min":98.921999999999997},"last":2.1680000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":3.181,"15min":1.802},"min":{"1min":0.44600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.6690000000000005,"5min":99.549999999999997,"15min":99.549999999999997},"last":2.2349999999999999},{"interface":"front","average":{"1min":1.702,"5min":2.2770000000000001,"15min":1.5549999999999999},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9720000000000004,"5min":31.225999999999999,"15min":31.225999999999999},"last":2.6619999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566217,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":671360,"kb_used_data":330424,"kb_used_omap":11,"kb_used_meta":340916,"kb_avail":93073792,"statfs":{"total":95995035648,"available":95307563008,"internally_reserved":0,"allocated":338354176,"data_stored":337383365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":349098768},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":4.9969999999999999,"15min":2.2229999999999999},"min":{"1min":0.27400000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.085000000000001,"5min":87.132999999999996,"15min":87.132999999999996},"last":3.3719999999999999},{"interface":"front","average":{"1min":3.931,"5min":3.8490000000000002,"15min":1.831},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.833,"5min":84.820999999999998,"15min":84.820999999999998},"last":2.8780000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":5.5570000000000004,"15min":5.2119999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":31.513999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":1.859},{"interface":"front","average":{"1min":3.7690000000000001,"5min":4.0259999999999998,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":18.821000000000002,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.89800000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:16:30 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":7.4589999999999996,"15min":3.177},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":29.297000000000001,"5min":87.108000000000004,"15min":87.108000000000004},"last":1.149},{"interface":"front","average":{"1min":7.899,"5min":6.8819999999999997,"15min":3.0369999999999999},"min":{"1min":0.32200000000000001,"5min":0.23499999999999999,"15min":0.20300000000000001},"max":{"1min":31.440999999999999,"5min":86.158000000000001,"15min":86.158000000000001},"last":1.891}]},{"osd":4,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.129,"15min":1.831},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.824,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.79100000000000004},{"interface":"front","average":{"1min":2.8849999999999998,"5min":4.016,"15min":2.4020000000000001},"min":{"1min":0.42399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.988,"5min":87.019999999999996,"15min":87.019999999999996},"last":1.0089999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":3.266,"15min":2.0579999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.193999999999999,"5min":84.498000000000005,"15min":84.498000000000005},"last":1.782},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.8690000000000002,"15min":1.9179999999999999},"min":{"1min":0.441,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.865,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.83299999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.1200000000000001,"15min":2.0369999999999999},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.891999999999999,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.93100000000000005},{"interface":"front","average":{"1min":2.9729999999999999,"5min":3.5750000000000002,"15min":2.0939999999999999},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.949999999999999,"5min":87.055000000000007,"15min":87.055000000000007},"last":1.0509999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":3.3519999999999999,"15min":1.915},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":16.239000000000001,"5min":84.596000000000004,"15min":84.596000000000004},"last":1.819},{"interface":"front","average":{"1min":1.966,"5min":3.0630000000000002,"15min":1.8049999999999999},"min":{"1min":0.48899999999999999,"5min":0.38600000000000001,"15min":0.34200000000000003},"max":{"1min":10.916,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.86299999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402691,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908256,"kb_used_data":588952,"kb_used_omap":3,"kb_used_meta":319292,"kb_avail":92836896,"statfs":{"total":95995035648,"available":95064981504,"internally_reserved":0,"allocated":603086848,"data_stored":602140058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":326955177},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":2.9510000000000001,"15min":2.2050000000000001},"min":{"1min":0.20100000000000001,"5min":0.19600000000000001,"15min":0.17899999999999999},"max":{"1min":18.864999999999998,"5min":25.111999999999998,"15min":79.611000000000004},"last":1.5600000000000001},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.222,"15min":2.3250000000000002},"min":{"1min":0.315,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":18.800000000000001,"5min":18.800000000000001,"15min":79.656000000000006},"last":0.629}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7490000000000001,"5min":4.5339999999999998,"15min":4.5679999999999996},"min":{"1min":0.25600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.939,"5min":28.315000000000001,"15min":28.315000000000001},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.673,"5min":4.3259999999999996,"15min":4.6719999999999997},"min":{"1min":0.29499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.832999999999998,"5min":21.539000000000001,"15min":21.539000000000001},"last":1.6459999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4320000000000004,"5min":4.3650000000000002,"15min":4.0119999999999996},"min":{"1min":0.33400000000000002,"5min":0.255,"15min":0.22},"max":{"1min":18.893999999999998,"5min":61.732999999999997,"15min":109.077},"last":1.677},{"interface":"front","average":{"1min":4.8819999999999997,"5min":4.5339999999999998,"15min":4.2439999999999998},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.25700000000000001},"max":{"1min":21.661999999999999,"5min":60.305999999999997,"15min":119.72799999999999},"last":0.40400000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6560000000000001,"5min":2.9609999999999999,"15min":1.6359999999999999},"min":{"1min":0.40100000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":12.958,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.52000000000000002},{"interface":"front","average":{"1min":3.0539999999999998,"5min":2.8919999999999999,"15min":1.79},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.497999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.38400000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.621,"15min":1.776},"min":{"1min":0.371,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.443},{"interface":"front","average":{"1min":3.5670000000000002,"5min":2.6139999999999999,"15min":1.99},"min":{"1min":0.40200000000000002,"5min":0.315,"15min":0.315},"max":{"1min":12.667999999999999,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.42099999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":3.3319999999999999,"15min":1.911},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.029999999999999,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.59599999999999997},{"interface":"front","average":{"1min":3.3639999999999999,"5min":3.323,"15min":2.1299999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.32300000000000001},"max":{"1min":11.117000000000001,"5min":78.028000000000006,"15min":78.028000000000006},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5840000000000001,"15min":1.911},"min":{"1min":0.47599999999999998,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.5630000000000002,"5min":2.593,"15min":1.5569999999999999},"min":{"1min":0.42199999999999999,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":18.696000000000002,"5min":18.696000000000002,"15min":18.696000000000002},"last":1.599}]}]},{"osd":4,"up_from":31,"seq":133143986456,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":529404,"kb_used_data":330420,"kb_used_omap":0,"kb_used_meta":198976,"kb_avail":93215748,"statfs":{"total":95995035648,"available":95452925952,"internally_reserved":0,"allocated":338350080,"data_stored":337607572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":203751424},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":2.5830000000000002,"15min":2.6200000000000001},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.8250000000000002,"5min":65.988,"15min":122.69199999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.552,"5min":2.5219999999999998,"15min":2.698},"min":{"1min":0.27700000000000002,"5min":0.221,"15min":0.221},"max":{"1min":5.7149999999999999,"5min":65.396000000000001,"15min":122.745},"last":2.8759999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":3.0219999999999998,"15min":4.04},"min":{"1min":0.33200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.6589999999999998,"5min":53.832000000000001,"15min":53.832000000000001},"last":2.762},{"interface":"front","average":{"1min":2.2989999999999999,"5min":3.7810000000000001,"15min":4.6449999999999996},"min":{"1min":0.28899999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.443,"5min":53.078000000000003,"15min":53.078000000000003},"last":1.996}]},{"osd":2,"last update":"Sat Sep 23 15:15:59 2023","interfaces":[{"interface":"back","average":{"1min":3.968,"5min":4.1260000000000003,"15min":3.1080000000000001},"min":{"1min":0.37,"5min":0.255,"15min":0.255},"max":{"1min":12.595000000000001,"5min":148.08000000000001,"15min":148.08000000000001},"last":2.8039999999999998},{"interface":"front","average":{"1min":3.2869999999999999,"5min":4.383,"15min":3.3530000000000002},"min":{"1min":0.34300000000000003,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":7.173,"5min":147.88900000000001,"15min":147.88900000000001},"last":2.843}]},{"osd":3,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":4.319,"5min":9.6549999999999994,"15min":4.0369999999999999},"min":{"1min":0.46600000000000003,"5min":0.28399999999999997,"15min":0.252},"max":{"1min":23.715,"5min":163.97499999999999,"15min":163.97499999999999},"last":0.55600000000000005},{"interface":"front","average":{"1min":4.5860000000000003,"5min":9.7769999999999992,"15min":3.8399999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":22.876000000000001,"5min":164.447,"15min":164.447},"last":1.3979999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.4730000000000001,"15min":1.5189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.8100000000000001,"5min":87.912999999999997,"15min":87.912999999999997},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.628,"5min":1.623,"15min":1.3999999999999999},"min":{"1min":0.309,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":2.4220000000000002,"5min":88.602999999999994,"15min":88.602999999999994},"last":1.292}]},{"osd":6,"last update":"Sat Sep 23 15:16:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":3.6160000000000001,"15min":2.0510000000000002},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.3820000000000001,"5min":210.44300000000001,"15min":210.44300000000001},"last":2.1549999999999998},{"interface":"front","average":{"1min":0.89600000000000002,"5min":3.7080000000000002,"15min":1.956},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.7749999999999999,"5min":210.78899999999999,"15min":210.78899999999999},"last":2.1070000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.621,"15min":1.123},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.417,"5min":5.399,"15min":18.23},"last":3.0979999999999999},{"interface":"front","average":{"1min":1.038,"5min":0.86599999999999999,"15min":1.2030000000000001},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.314},"max":{"1min":3.851,"5min":6.1399999999999997,"15min":14.653},"last":1.1200000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724819,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476468,"kb_used_data":264748,"kb_used_omap":0,"kb_used_meta":211712,"kb_avail":93268684,"statfs":{"total":95995035648,"available":95507132416,"internally_reserved":0,"allocated":271101952,"data_stored":270363943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":216793088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.282,"5min":1.9430000000000001,"15min":1.4079999999999999},"min":{"1min":0.246,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":7.8319999999999999,"5min":16.981999999999999,"15min":24.962},"last":0.372},{"interface":"front","average":{"1min":3.972,"5min":2.1589999999999998,"15min":1.4550000000000001},"min":{"1min":0.248,"5min":0.248,"15min":0.222},"max":{"1min":12.531000000000001,"5min":18.814,"15min":25.317},"last":0.626}]},{"osd":1,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.327,"15min":1.891},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9329999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.46,"5min":2.6120000000000001,"15min":2.383},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5199999999999996,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.503}]},{"osd":2,"last update":"Sat Sep 23 15:16:00 2023","interfaces":[{"interface":"back","average":{"1min":2.9969999999999999,"5min":3.266,"15min":2.4609999999999999},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":8.3889999999999993,"5min":52.414000000000001,"15min":97.087999999999994},"last":0.64300000000000002},{"interface":"front","average":{"1min":2.5070000000000001,"5min":3.3570000000000002,"15min":2.4209999999999998},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.254},"max":{"1min":6.2359999999999998,"5min":52.375999999999998,"15min":97.305999999999997},"last":0.73099999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":10.236000000000001,"5min":6.5570000000000004,"15min":3.1299999999999999},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":118.208,"5min":118.208,"15min":118.208},"last":0.47799999999999998},{"interface":"front","average":{"1min":8.9410000000000007,"5min":6.2590000000000003,"15min":2.9009999999999998},"min":{"1min":0.33400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":118.42,"5min":118.42,"15min":118.42},"last":1.0940000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.956,"15min":1.7250000000000001},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.1920000000000002,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.56899999999999995,"5min":2.8180000000000001,"15min":1.792},"min":{"1min":0.29199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8410000000000002,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88900000000000001,"15min":1.1339999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.5550000000000002,"5min":5.8760000000000003,"15min":15.196},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.86499999999999999,"15min":1.0229999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9990000000000001,"5min":4.8819999999999997,"15min":14.965999999999999},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.86599999999999999,"15min":1.2829999999999999},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.0249999999999999,"5min":8.2200000000000006,"15min":15.301},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.79100000000000004,"15min":1.2589999999999999},"min":{"1min":0.38500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.5379999999999998,"5min":5.6559999999999997,"15min":15.762},"last":0.438}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":463994880,"data_stored":463994880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":237174784,"data_stored":237174784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":366018560,"data_stored":366018560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":598802432,"data_stored":598802432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":340787200,"data_stored":340787200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":309788672,"data_stored":309788672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":280166400,"data_stored":280166400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:17:03.589 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:03.714 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:03.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:03.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:03.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:03.886 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:03.958 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:03.981 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:03.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:03.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:04.050 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:04 smithi027 ceph-mon[195233]: pgmap v638: 109 pgs: 109 active+clean; 891 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 89 MiB/s wr, 1.41k op/s 2023-09-23T15:17:04.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:04.121 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:04.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 21298 21282 120.808 77 0.00804558 0.00827083 2023-09-23T15:17:04.231 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:04.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:04.266 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:04 smithi017 ceph-mon[215646]: pgmap v638: 109 pgs: 109 active+clean; 891 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 89 MiB/s wr, 1.41k op/s 2023-09-23T15:17:04.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:04 smithi017 ceph-mon[222878]: pgmap v638: 109 pgs: 109 active+clean; 891 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 89 MiB/s wr, 1.41k op/s 2023-09-23T15:17:04.307 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:04.307 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:04.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:04.372 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:04.540 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:04.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:04.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:04.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:04.685 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:04.729 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:04.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:04.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:04.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:04.874 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:04.978 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:04.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:04.987 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:05.011 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:17:05.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:05.115 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:05.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 22543 22527 117.227 77.8125 0.00185048 0.0084289 2023-09-23T15:17:05.165 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:05.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:05.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:05.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/215874609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/215874609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:17:05.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:05.295 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:05.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/215874609' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:05.352 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:05.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:05.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:05.458 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:05.524 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:05.642 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:06.050 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:06.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:06.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:06.128 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 23922 23906 114.841 86.1875 0.0087805 0.00864418 2023-09-23T15:17:06.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:06.184 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:06.228 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: pgmap v639: 109 pgs: 109 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T15:17:06.229 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:06.229 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.229 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.229 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:06.229 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:06 smithi027 ceph-mon[195233]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T15:17:06.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: pgmap v639: 109 pgs: 109 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[222878]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: pgmap v639: 109 pgs: 109 active+clean; 1.1 GiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 103 MiB/s wr, 1.65k op/s 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:06.367 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: failed to encode map e230 with expected crc 2023-09-23T15:17:06.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:06.368 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:06 smithi017 ceph-mon[215646]: osdmap e230: 8 total, 8 up, 7 in 2023-09-23T15:17:06.472 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:06.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:06.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:06.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:17:06.612 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:06.907 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:06.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:06.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:07.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:07.049 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:07.080 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:07.121 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:07.130 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 25208 25192 112.368 80.375 0.0122593 0.008893 2023-09-23T15:17:07.168 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:07.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:07.210 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:07.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:07.247 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:07.285 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:07.314 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:07.324 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:07.328 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:07.354 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:07.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:17:07.424 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:07.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:07.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166939 5.f 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:17:07.447 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.36455218100002, 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.42086912300000001, 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166939 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079753+0000", 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079752+0000", 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079756+0000", 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7440000000000001e-06 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079765+0000", 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9819999999999997e-06 2023-09-23T15:17:07.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.079770+0000", 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6439999999999999e-06 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086956+0000", 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071860470000000001 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087060+0000", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104073 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087174+0000", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114411 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371249+0000", 2023-09-23T15:17:07.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28407477599999997 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371500+0000", 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00025060700000000002 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500542+0000", 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.129042504 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500587+0000", 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4487000000000003e-05 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.500622+0000", 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5672e-05 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166948 5.b e147/114 5:dfd8a27b:::benchmark_data_smithi017_288048_object55179:head v 147'10455, mlcod=147'10453)", 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.35364052300002, 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28076174100000001, 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090665+0000", 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090664+0000", 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090672+0000", 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3639999999999999e-06 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090684+0000", 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1531e-05 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090690+0000", 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.229e-06 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090801+0000", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011069900000000001 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090821+0000", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0027e-05 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371402+0000", 2023-09-23T15:17:07.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28058097599999998 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371427+0000", 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4839999999999999e-05 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166950 5.f 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:17:07.459 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.35359179099999, 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.41173430799999999, 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166950 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090714+0000", 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090712+0000", 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090716+0000", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.296e-06 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090725+0000", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3890000000000005e-06 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090730+0000", 2023-09-23T15:17:07.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.792e-06 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091549+0000", 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00081936800000000001 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091663+0000", 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011382700000000001 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091780+0000", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117632 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371458+0000", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27967720699999998 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371548+0000", 2023-09-23T15:17:07.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0597e-05 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502366+0000", 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13081746899999999 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502413+0000", 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6909000000000001e-05 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502448+0000", 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5142000000000003e-05 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166942 5.e e147/114 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head v 147'10334, mlcod=147'10332)", 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.35211168799998, 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27917522900000002, 2023-09-23T15:17:07.466 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092194+0000", 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092193+0000", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092201+0000", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8779999999999998e-06 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092211+0000", 2023-09-23T15:17:07.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0341e-05 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092216+0000", 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3469999999999998e-06 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092399+0000", 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018310200000000001 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092418+0000", 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8604e-05 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371292+0000", 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27887434100000003 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371369+0000", 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6413000000000002e-05 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166947 5.e e147/114 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head v 147'10335, mlcod=147'10332)", 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.35205550299997, 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.279469315, 2023-09-23T15:17:07.471 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092250+0000", 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092253+0000", 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2320000000000001e-06 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092258+0000", 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9749999999999998e-06 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092261+0000", 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1719999999999999e-06 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096022+0000", 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037614689999999999 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096042+0000", 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9712999999999999e-05 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371373+0000", 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27533052099999999 2023-09-23T15:17:07.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371719+0000", 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000346409 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166949 5.2 e147/114 5:4bbd8647:::benchmark_data_smithi017_288048_object55180:head v 147'10607, mlcod=147'10605)", 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:17:07.476 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.351987479, 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27920739500000002, 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092318+0000", 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092317+0000", 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092323+0000", 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1909999999999999e-06 2023-09-23T15:17:07.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092331+0000", 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0409999999999998e-06 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092335+0000", 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2050000000000004e-06 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096032+0000", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036968489999999999 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096049+0000", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7271000000000001e-05 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371369+0000", 2023-09-23T15:17:07.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27532035900000001 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371525+0000", 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015582500000000001 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166954 5.3 e147/114 5:c22244f8:::benchmark_data_smithi017_288048_object55185:head v 147'10165, mlcod=147'10163)", 2023-09-23T15:17:07.481 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.350431372, 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27758365299999999, 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093874+0000", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093873+0000", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093881+0000", 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3590000000000008e-06 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093894+0000", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2184000000000001e-05 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093912+0000", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8244999999999999e-05 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.094053+0000", 2023-09-23T15:17:07.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141366 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.094074+0000", 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0511999999999999e-05 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371431+0000", 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27735731899999999 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371458+0000", 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6616000000000001e-05 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166946 5.4 e147/114 5:2ea27988:::benchmark_data_smithi017_288048_object55177:head v 147'10343, mlcod=147'10340)", 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.34913200699998, 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27613599900000002, 2023-09-23T15:17:07.486 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095173+0000", 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095172+0000", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095181+0000", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3909999999999998e-06 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095192+0000", 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1131999999999999e-05 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095198+0000", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0769999999999996e-06 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095238+0000", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0147e-05 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095257+0000", 2023-09-23T15:17:07.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8853000000000001e-05 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371269+0000", 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27601161800000001 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371309+0000", 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0626e-05 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.490 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166951 5.4 e147/114 5:254d5c06:::benchmark_data_smithi017_288048_object55182:head v 147'10344, mlcod=147'10341)", 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.34909098899999, 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27622131900000002, 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:17:07.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095214+0000", 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095218+0000", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8010000000000001e-06 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095223+0000", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5629999999999999e-06 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095227+0000", 2023-09-23T15:17:07.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5149999999999998e-06 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098400+0000", 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031727639999999998 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098417+0000", 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7609999999999999e-05 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371408+0000", 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27299110700000001 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371436+0000", 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7199000000000002e-05 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166952 5.b e147/114 5:da70994f:::benchmark_data_smithi017_288048_object55183:head v 147'10456, mlcod=147'10453)", 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.34906408900002, 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.276238488, 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:17:07.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095241+0000", 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095245+0000", 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7129999999999999e-06 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095250+0000", 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8470000000000001e-06 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095254+0000", 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3139999999999999e-06 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095675+0000", 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00042140999999999999 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095691+0000", 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5928000000000001e-05 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371460+0000", 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27576851099999999 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371480+0000", 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0111000000000001e-05 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200838 5.b e156/114 5:d7ba4b77:::benchmark_data_smithi017_288048_object89069:head v 156'12612, mlcod=156'12606)", 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.31039472100002, 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31912336400000002, 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133911+0000", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133910+0000", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133917+0000", 2023-09-23T15:17:07.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7519999999999999e-06 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133928+0000", 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.151e-05 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133934+0000", 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6740000000000002e-06 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137529+0000", 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035948030000000001 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137544+0000", 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5594999999999999e-05 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.452976+0000", 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31543138300000001 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453034+0000", 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8356000000000002e-05 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200843 5.3 e156/114 5:c8957fbc:::benchmark_data_smithi017_288048_object89074:head v 156'12232, mlcod=156'12230)", 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.310308604, 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31897129600000002, 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:17:07.505 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133997+0000", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.133996+0000", 2023-09-23T15:17:07.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134004+0000", 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5989999999999996e-06 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134015+0000", 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1121000000000001e-05 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.134020+0000", 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4670000000000003e-06 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137524+0000", 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035037229999999998 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.137543+0000", 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8983000000000001e-05 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.452968+0000", 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31542512 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200840 5.b e156/114 5:d0cff80c:::benchmark_data_smithi017_288048_object89071:head v 156'12613, mlcod=156'12606)", 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.30523417500001, 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31398826699999999, 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.509 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139071+0000", 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139078+0000", 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3420000000000002e-06 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139086+0000", 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3799999999999994e-06 2023-09-23T15:17:07.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.139090+0000", 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1230000000000002e-06 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296790+0000", 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15769914900000001 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.298031+0000", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001241942 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453041+0000", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15500998099999999 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.453059+0000", 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7944999999999998e-05 2023-09-23T15:17:07.513 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200853 5.7 5:ef3ce6d5:::benchmark_data_smithi017_288048_object89084:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.14776148700003, 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.36030263699999998, 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:07.514 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200853 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296544+0000", 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296542+0000", 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296548+0000", 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4489999999999998e-06 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296560+0000", 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2655e-05 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296567+0000", 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1650000000000003e-06 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.297205+0000", 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063859400000000003 2023-09-23T15:17:07.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.297306+0000", 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100739 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.298062+0000", 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00075588599999999999 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632346+0000", 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.33428428199999999 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.656846+0000", 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024500299 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.519 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200858 5.7 5:e5e3a1c9:::benchmark_data_smithi017_288048_object89089:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.147729895, 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.36027551899999999, 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200858 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296575+0000", 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296578+0000", 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.965e-06 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296584+0000", 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2720000000000003e-06 2023-09-23T15:17:07.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.296589+0000", 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3800000000000004e-06 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304686+0000", 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0080975849999999992 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304839+0000", 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015270600000000001 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.304987+0000", 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014832699999999999 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632380+0000", 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32739249399999998 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.656851+0000", 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.024471211999999999 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:200853 5.7 e156/132)", 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.13570757999997, 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32381470600000001, 2023-09-23T15:17:07.525 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308598+0000", 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308597+0000", 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308604+0000", 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1910000000000003e-06 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308617+0000", 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3037000000000001e-05 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.308624+0000", 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9380000000000003e-06 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632400+0000", 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32377659600000003 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.632412+0000", 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1933000000000001e-05 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:200861 5.e e157/114 5:7029cdc5:::benchmark_data_smithi017_288048_object89092:head v 157'12376, mlcod=156'12374)", 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 363.11544145300002, 2023-09-23T15:17:07.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28701705900000002, 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328864+0000", 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328863+0000", 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328871+0000", 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1450000000000002e-06 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328883+0000", 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2225000000000001e-05 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.328890+0000", 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.934e-06 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.565469+0000", 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236579123 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.565513+0000", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3636e-05 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.615832+0000", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.050319323999999999 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.615881+0000", 2023-09-23T15:17:07.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9051999999999997e-05 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78612 6.d 6:bb45bf81:::benchmark_data_smithi017_350392_object78611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 265.87652880399997, 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6694258959999999, 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.534 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78612 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567775+0000", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567840+0000", 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4871999999999998e-05 2023-09-23T15:17:07.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567851+0000", 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0589e-05 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567857+0000", 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3589999999999996e-06 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568037+0000", 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017947699999999999 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568207+0000", 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000170431 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568367+0000", 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015976799999999999 2023-09-23T15:17:07.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574569+0000", 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006201771 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574582+0000", 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3179999999999999e-05 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.844996+0000", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27041409399999999 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845010+0000", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4292e-05 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201824+0000", 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.356813662 2023-09-23T15:17:07.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203608+0000", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017844499999999999 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237060+0000", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.033452123 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237202+0000", 2023-09-23T15:17:07.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014193699999999999 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78627 6.d 6:b5bf0d91:::benchmark_data_smithi017_350392_object78626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 265.84144962699997, 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.637516355, 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.542 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78627 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602854+0000", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602946+0000", 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1372999999999994e-05 2023-09-23T15:17:07.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602960+0000", 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4171999999999999e-05 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602965+0000", 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3709999999999999e-06 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603026+0000", 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1115e-05 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603042+0000", 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5851000000000001e-05 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845043+0000", 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24200096700000001 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845051+0000", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9529999999999999e-06 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203676+0000", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.358624947 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203805+0000", 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128534 2023-09-23T15:17:07.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.204009+0000", 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020406299999999999 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.213055+0000", 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090466460000000002 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.218171+0000", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051154820000000002 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237245+0000", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019074157000000001 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237307+0000", 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1816000000000006e-05 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240372+0000", 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030652040000000002 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78647 6.d 6:b57ccd88:::benchmark_data_smithi017_350392_object78646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 265.81394211100002, 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.61263826, 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:07.550 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78647 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630362+0000", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630419+0000", 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.698e-05 2023-09-23T15:17:07.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630433+0000", 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3936e-05 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630440+0000", 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5939999999999999e-06 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630499+0000", 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9676000000000003e-05 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630508+0000", 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7760000000000003e-06 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845613+0000", 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21510510299999999 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845622+0000", 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8729999999999996e-06 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206337+0000", 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3607147209999999 2023-09-23T15:17:07.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206504+0000", 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016709 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206715+0000", 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021077199999999999 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.217101+0000", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010385926 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.218207+0000", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011067220000000001 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240451+0000", 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022243489000000002 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240524+0000", 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3198e-05 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.243001+0000", 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024774049999999998 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:07.559 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:07.559 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:17:07.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:07.629 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:08.130 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 26567 26551 110.54 84.9375 0.00922793 0.00904106 2023-09-23T15:17:08.347 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:08 smithi027 ceph-mon[195233]: pgmap v641: 109 pgs: 109 active+clean; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:08.347 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3925148256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:08.367 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:17:08.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:08 smithi017 ceph-mon[215646]: pgmap v641: 109 pgs: 109 active+clean; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:08.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3925148256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:08.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:08] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-09-23T15:17:08.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:08 smithi017 ceph-mon[222878]: pgmap v641: 109 pgs: 109 active+clean; 1.3 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:08.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3925148256' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:08.585 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:17:08.676 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:08.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:08.681 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:08.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:08.820 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:08.874 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:08.884 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:08.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:08.991 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:09.023 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:09.130 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 27794 27778 108.425 76.6875 0.00881408 0.00921568 2023-09-23T15:17:09.138 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:09.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:09.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:09.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:09.285 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:09.411 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:09.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:09.519 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:09.559 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:09.614 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:09.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:09.616 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:09.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:09.743 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:09.787 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:09.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:09.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:09.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:09.923 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:09.925 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:10.030 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:10 smithi017 ceph-mon[222878]: pgmap v642: 109 pgs: 109 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:10.130 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 29142 29126 107.003 84.25 0.00741586 0.00933935 2023-09-23T15:17:10.167 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:10.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:10.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:10.273 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:10.305 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:10 smithi027 ceph-mon[195233]: pgmap v642: 109 pgs: 109 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:10.305 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:10 smithi017 ceph-mon[215646]: pgmap v642: 109 pgs: 109 active+clean; 1.3 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 115 MiB/s wr, 1.84k op/s 2023-09-23T15:17:10.308 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:10.473 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:10.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:10.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:10.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:10.671 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:10.696 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:29953 7.c 7:379c618d:::benchmark_data_smithi017_422824_object29952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:17:10.693902+0000", 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0028596580000000002, 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028845899999999998, 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:10.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 29953 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693902+0000", 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693902+0000", 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693902+0000", 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:10.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693950+0000", 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8727999999999998e-05 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693960+0000", 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2939999999999994e-06 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693964+0000", 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0799999999999999e-06 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694014+0000", 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0519999999999997e-05 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694175+0000", 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016114100000000001 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694275+0000", 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9475000000000006e-05 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.695561+0000", 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012861789999999999 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:29955 7.7 7:e7cd7e25:::benchmark_data_smithi017_422824_object29954:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:10.703 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:17:10.693972+0000", 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.002789616, 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028692100000000001, 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 29955 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:10.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693972+0000", 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693972+0000", 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.693972+0000", 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694020+0000", 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8136000000000001e-05 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694028+0000", 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9000000000000006e-06 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694033+0000", 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0730000000000004e-06 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694070+0000", 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6961000000000001e-05 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694177+0000", 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106593 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694315+0000", 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138391 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.695632+0000", 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001316596 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:29957 7.d 7:bb781e7e:::benchmark_data_smithi017_422824_object29956:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:17:10.694046+0000", 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0027156670000000002, 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028427980000000001, 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:10.710 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 29957 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694046+0000", 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694046+0000", 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694046+0000", 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:10.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694116+0000", 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0022000000000001e-05 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694125+0000", 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8710000000000003e-06 2023-09-23T15:17:10.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694131+0000", 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8760000000000002e-06 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694227+0000", 2023-09-23T15:17:10.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6875999999999999e-05 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694332+0000", 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104798 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.694475+0000", 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000143008 2023-09-23T15:17:10.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:17:10.695760+0000", 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001284758 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 3 2023-09-23T15:17:10.716 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:17:10.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:17:10.789 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:10.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:10.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:10.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:10.932 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:17:10.947 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:17:11.048 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:11.131 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 30546 30530 105.934 87.75 0.0127021 0.00943309 2023-09-23T15:17:11.159 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:11.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:11.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:11.264 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:11.291 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:11.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:17:11.777 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:11.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:11.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:11.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:11.922 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:12.041 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:12.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:12.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:12.056 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:12.131 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 31809 31793 104.509 78.9375 0.00416575 0.00956308 2023-09-23T15:17:12.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:12.177 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:12 smithi027 ceph-mon[195233]: pgmap v643: 109 pgs: 109 active+clean; 1.7 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:17:12.177 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:12 smithi027 ceph-mon[195233]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:12.182 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:12.302 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:12.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:12.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:12.408 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:12.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:12 smithi017 ceph-mon[222878]: pgmap v643: 109 pgs: 109 active+clean; 1.7 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:17:12.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:12 smithi017 ceph-mon[222878]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:12.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:12 smithi017 ceph-mon[215646]: pgmap v643: 109 pgs: 109 active+clean; 1.7 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:17:12.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:12 smithi017 ceph-mon[215646]: from='client.35589 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:17:12.442 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:12.484 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":643,"stamp":"2023-09-23T15:17:10.452846+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1835538355,"num_objects":27963,"num_object_clones":0,"num_object_copies":83889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27963,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":28842,"num_write_kb":1800565,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4724630,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":32348,"ondisk_log_size":32348,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7155788,"kb_used_data":5296724,"kb_used_omap":30,"kb_used_meta":1858977,"kb_avail":649060276,"statfs":{"total":671965249536,"available":664637722624,"internally_reserved":0,"allocated":5423845376,"data_stored":5417633003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":1903593048},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":943325184,"num_objects":14394,"num_object_clones":0,"num_object_copies":43182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":14394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14394,"num_write_kb":921216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004643"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":224,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514685+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:16:55.514685+0000","last_peered":"2023-09-23T15:16:55.514685+0000","last_clean":"2023-09-23T15:16:55.514685+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:16:55.514685+0000","last_undegraded":"2023-09-23T15:16:55.514685+0000","last_fullsized":"2023-09-23T15:16:55.514685+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":159,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518715+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:16:55.518715+0000","last_peered":"2023-09-23T15:16:55.518715+0000","last_clean":"2023-09-23T15:16:55.518715+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:16:55.518715+0000","last_undegraded":"2023-09-23T15:16:55.518715+0000","last_fullsized":"2023-09-23T15:16:55.518715+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":107,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514428+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:16:55.514428+0000","last_peered":"2023-09-23T15:16:55.514428+0000","last_clean":"2023-09-23T15:16:55.514428+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:16:55.514428+0000","last_undegraded":"2023-09-23T15:16:55.514428+0000","last_fullsized":"2023-09-23T15:16:55.514428+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":196,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522586+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:16:55.522586+0000","last_peered":"2023-09-23T15:16:55.522586+0000","last_clean":"2023-09-23T15:16:55.522586+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:16:55.522586+0000","last_undegraded":"2023-09-23T15:16:55.522586+0000","last_fullsized":"2023-09-23T15:16:55.522586+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":228,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518048+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:16:55.518048+0000","last_peered":"2023-09-23T15:16:55.518048+0000","last_clean":"2023-09-23T15:16:55.518048+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:16:55.518048+0000","last_undegraded":"2023-09-23T15:16:55.518048+0000","last_fullsized":"2023-09-23T15:16:55.518048+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522007+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:16:55.522007+0000","last_peered":"2023-09-23T15:16:55.522007+0000","last_clean":"2023-09-23T15:16:55.522007+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:16:55.522007+0000","last_undegraded":"2023-09-23T15:16:55.522007+0000","last_fullsized":"2023-09-23T15:16:55.522007+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":175,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522906+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:16:55.522906+0000","last_peered":"2023-09-23T15:16:55.522906+0000","last_clean":"2023-09-23T15:16:55.522906+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:16:55.522906+0000","last_undegraded":"2023-09-23T15:16:55.522906+0000","last_fullsized":"2023-09-23T15:16:55.522906+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":210,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518127+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:16:55.518127+0000","last_peered":"2023-09-23T15:16:55.518127+0000","last_clean":"2023-09-23T15:16:55.518127+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:16:55.518127+0000","last_undegraded":"2023-09-23T15:16:55.518127+0000","last_fullsized":"2023-09-23T15:16:55.518127+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":528,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521585+0000","last_change":"2023-09-23T15:16:55.370503+0000","last_active":"2023-09-23T15:16:55.521585+0000","last_peered":"2023-09-23T15:16:55.521585+0000","last_clean":"2023-09-23T15:16:55.521585+0000","last_became_active":"2023-09-23T15:16:46.275526+0000","last_became_peered":"2023-09-23T15:16:46.275526+0000","last_unstale":"2023-09-23T15:16:55.521585+0000","last_undegraded":"2023-09-23T15:16:55.521585+0000","last_fullsized":"2023-09-23T15:16:55.521585+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:54.310407+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":220,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520628+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:16:55.520628+0000","last_peered":"2023-09-23T15:16:55.520628+0000","last_clean":"2023-09-23T15:16:55.520628+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:16:55.520628+0000","last_undegraded":"2023-09-23T15:16:55.520628+0000","last_fullsized":"2023-09-23T15:16:55.520628+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":153,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519216+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:16:55.519216+0000","last_peered":"2023-09-23T15:16:55.519216+0000","last_clean":"2023-09-23T15:16:55.519216+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:16:55.519216+0000","last_undegraded":"2023-09-23T15:16:55.519216+0000","last_fullsized":"2023-09-23T15:16:55.519216+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":93,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518632+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:16:55.518632+0000","last_peered":"2023-09-23T15:16:55.518632+0000","last_clean":"2023-09-23T15:16:55.518632+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:16:55.518632+0000","last_undegraded":"2023-09-23T15:16:55.518632+0000","last_fullsized":"2023-09-23T15:16:55.518632+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":102,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514831+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:16:55.514831+0000","last_peered":"2023-09-23T15:16:55.514831+0000","last_clean":"2023-09-23T15:16:55.514831+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:16:55.514831+0000","last_undegraded":"2023-09-23T15:16:55.514831+0000","last_fullsized":"2023-09-23T15:16:55.514831+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":214,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514872+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:16:55.514872+0000","last_peered":"2023-09-23T15:16:55.514872+0000","last_clean":"2023-09-23T15:16:55.514872+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:16:55.514872+0000","last_undegraded":"2023-09-23T15:16:55.514872+0000","last_fullsized":"2023-09-23T15:16:55.514872+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":205,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515194+0000","last_change":"2023-09-23T15:12:41.871388+0000","last_active":"2023-09-23T15:16:55.515194+0000","last_peered":"2023-09-23T15:16:55.515194+0000","last_clean":"2023-09-23T15:16:55.515194+0000","last_became_active":"2023-09-23T15:12:41.871022+0000","last_became_peered":"2023-09-23T15:12:41.871022+0000","last_unstale":"2023-09-23T15:16:55.515194+0000","last_undegraded":"2023-09-23T15:16:55.515194+0000","last_fullsized":"2023-09-23T15:16:55.515194+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:56.205783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":536,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522676+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:16:55.522676+0000","last_peered":"2023-09-23T15:16:55.522676+0000","last_clean":"2023-09-23T15:16:55.522676+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:16:55.522676+0000","last_undegraded":"2023-09-23T15:16:55.522676+0000","last_fullsized":"2023-09-23T15:16:55.522676+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":92,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515242+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:16:55.515242+0000","last_peered":"2023-09-23T15:16:55.515242+0000","last_clean":"2023-09-23T15:16:55.515242+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:16:55.515242+0000","last_undegraded":"2023-09-23T15:16:55.515242+0000","last_fullsized":"2023-09-23T15:16:55.515242+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":221,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520477+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:16:55.520477+0000","last_peered":"2023-09-23T15:16:55.520477+0000","last_clean":"2023-09-23T15:16:55.520477+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:16:55.520477+0000","last_undegraded":"2023-09-23T15:16:55.520477+0000","last_fullsized":"2023-09-23T15:16:55.520477+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":529,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521118+0000","last_change":"2023-09-23T15:16:46.276165+0000","last_active":"2023-09-23T15:16:55.521118+0000","last_peered":"2023-09-23T15:16:55.521118+0000","last_clean":"2023-09-23T15:16:55.521118+0000","last_became_active":"2023-09-23T15:16:46.275685+0000","last_became_peered":"2023-09-23T15:16:46.275685+0000","last_unstale":"2023-09-23T15:16:55.521118+0000","last_undegraded":"2023-09-23T15:16:55.521118+0000","last_fullsized":"2023-09-23T15:16:55.521118+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:36.377771+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":195,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522324+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:16:55.522324+0000","last_peered":"2023-09-23T15:16:55.522324+0000","last_clean":"2023-09-23T15:16:55.522324+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:16:55.522324+0000","last_undegraded":"2023-09-23T15:16:55.522324+0000","last_fullsized":"2023-09-23T15:16:55.522324+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":198,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520218+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:16:55.520218+0000","last_peered":"2023-09-23T15:16:55.520218+0000","last_clean":"2023-09-23T15:16:55.520218+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:16:55.520218+0000","last_undegraded":"2023-09-23T15:16:55.520218+0000","last_fullsized":"2023-09-23T15:16:55.520218+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":483,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515906+0000","last_change":"2023-09-23T15:16:51.467653+0000","last_active":"2023-09-23T15:16:55.515906+0000","last_peered":"2023-09-23T15:16:55.515906+0000","last_clean":"2023-09-23T15:16:55.515906+0000","last_became_active":"2023-09-23T15:16:46.277151+0000","last_became_peered":"2023-09-23T15:16:46.277151+0000","last_unstale":"2023-09-23T15:16:55.515906+0000","last_undegraded":"2023-09-23T15:16:55.515906+0000","last_fullsized":"2023-09-23T15:16:55.515906+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:46.522265+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":76,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514724+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:16:55.514724+0000","last_peered":"2023-09-23T15:16:55.514724+0000","last_clean":"2023-09-23T15:16:55.514724+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:16:55.514724+0000","last_undegraded":"2023-09-23T15:16:55.514724+0000","last_fullsized":"2023-09-23T15:16:55.514724+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":184,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522773+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:16:55.522773+0000","last_peered":"2023-09-23T15:16:55.522773+0000","last_clean":"2023-09-23T15:16:55.522773+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:16:55.522773+0000","last_undegraded":"2023-09-23T15:16:55.522773+0000","last_fullsized":"2023-09-23T15:16:55.522773+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":529,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519833+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:16:55.519833+0000","last_peered":"2023-09-23T15:16:55.519833+0000","last_clean":"2023-09-23T15:16:55.519833+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:16:55.519833+0000","last_undegraded":"2023-09-23T15:16:55.519833+0000","last_fullsized":"2023-09-23T15:16:55.519833+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":196,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520851+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:16:55.520851+0000","last_peered":"2023-09-23T15:16:55.520851+0000","last_clean":"2023-09-23T15:16:55.520851+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:16:55.520851+0000","last_undegraded":"2023-09-23T15:16:55.520851+0000","last_fullsized":"2023-09-23T15:16:55.520851+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":221,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520439+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:16:55.520439+0000","last_peered":"2023-09-23T15:16:55.520439+0000","last_clean":"2023-09-23T15:16:55.520439+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:16:55.520439+0000","last_undegraded":"2023-09-23T15:16:55.520439+0000","last_fullsized":"2023-09-23T15:16:55.520439+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"230'1632","reported_seq":1648,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:06.408017+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:17:06.408017+0000","last_peered":"2023-09-23T15:17:06.408017+0000","last_clean":"2023-09-23T15:17:06.408017+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:17:06.408017+0000","last_undegraded":"2023-09-23T15:17:06.408017+0000","last_fullsized":"2023-09-23T15:17:06.408017+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1632,"log_dups_size":0,"ondisk_log_size":1632,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106954752,"num_objects":1632,"num_object_clones":0,"num_object_copies":4896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1632,"num_write_kb":104448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":527,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514583+0000","last_change":"2023-09-23T15:16:51.284220+0000","last_active":"2023-09-23T15:16:55.514583+0000","last_peered":"2023-09-23T15:16:55.514583+0000","last_clean":"2023-09-23T15:16:55.514583+0000","last_became_active":"2023-09-23T15:16:46.276402+0000","last_became_peered":"2023-09-23T15:16:46.276402+0000","last_unstale":"2023-09-23T15:16:55.514583+0000","last_undegraded":"2023-09-23T15:16:55.514583+0000","last_fullsized":"2023-09-23T15:16:55.514583+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.284115+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.284115+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:27.778741+0000","scrub_duration":0.001563902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":104,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515268+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:16:55.515268+0000","last_peered":"2023-09-23T15:16:55.515268+0000","last_clean":"2023-09-23T15:16:55.515268+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:16:55.515268+0000","last_undegraded":"2023-09-23T15:16:55.515268+0000","last_fullsized":"2023-09-23T15:16:55.515268+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":181,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521295+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:16:55.521295+0000","last_peered":"2023-09-23T15:16:55.521295+0000","last_clean":"2023-09-23T15:16:55.521295+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:16:55.521295+0000","last_undegraded":"2023-09-23T15:16:55.521295+0000","last_fullsized":"2023-09-23T15:16:55.521295+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"230'1848","reported_seq":1864,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.647219+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:17:09.647219+0000","last_peered":"2023-09-23T15:17:09.647219+0000","last_clean":"2023-09-23T15:17:09.647219+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:17:09.647219+0000","last_undegraded":"2023-09-23T15:17:09.647219+0000","last_fullsized":"2023-09-23T15:17:09.647219+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1848,"log_dups_size":0,"ondisk_log_size":1848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121110528,"num_objects":1848,"num_object_clones":0,"num_object_copies":5544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1848,"num_write_kb":118272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":532,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520377+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:16:55.520377+0000","last_peered":"2023-09-23T15:16:55.520377+0000","last_clean":"2023-09-23T15:16:55.520377+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:16:55.520377+0000","last_undegraded":"2023-09-23T15:16:55.520377+0000","last_fullsized":"2023-09-23T15:16:55.520377+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":104,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514825+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:16:55.514825+0000","last_peered":"2023-09-23T15:16:55.514825+0000","last_clean":"2023-09-23T15:16:55.514825+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:16:55.514825+0000","last_undegraded":"2023-09-23T15:16:55.514825+0000","last_fullsized":"2023-09-23T15:16:55.514825+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":222,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514599+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:16:55.514599+0000","last_peered":"2023-09-23T15:16:55.514599+0000","last_clean":"2023-09-23T15:16:55.514599+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:16:55.514599+0000","last_undegraded":"2023-09-23T15:16:55.514599+0000","last_fullsized":"2023-09-23T15:16:55.514599+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"230'1771","reported_seq":1787,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.488210+0000","last_change":"2023-09-23T15:16:44.269684+0000","last_active":"2023-09-23T15:17:09.488210+0000","last_peered":"2023-09-23T15:17:09.488210+0000","last_clean":"2023-09-23T15:17:09.488210+0000","last_became_active":"2023-09-23T15:16:44.269411+0000","last_became_peered":"2023-09-23T15:16:44.269411+0000","last_unstale":"2023-09-23T15:17:09.488210+0000","last_undegraded":"2023-09-23T15:17:09.488210+0000","last_fullsized":"2023-09-23T15:17:09.488210+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1771,"log_dups_size":0,"ondisk_log_size":1771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:33.481012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116064256,"num_objects":1771,"num_object_clones":0,"num_object_copies":5313,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1771,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1771,"num_write_kb":113344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":452,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521892+0000","last_change":"2023-09-23T15:16:49.361068+0000","last_active":"2023-09-23T15:16:55.521892+0000","last_peered":"2023-09-23T15:16:55.521892+0000","last_clean":"2023-09-23T15:16:55.521892+0000","last_became_active":"2023-09-23T15:16:46.276691+0000","last_became_peered":"2023-09-23T15:16:46.276691+0000","last_unstale":"2023-09-23T15:16:55.521892+0000","last_undegraded":"2023-09-23T15:16:55.521892+0000","last_fullsized":"2023-09-23T15:16:55.521892+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:16.914979+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":90,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520145+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:16:55.520145+0000","last_peered":"2023-09-23T15:16:55.520145+0000","last_clean":"2023-09-23T15:16:55.520145+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:16:55.520145+0000","last_undegraded":"2023-09-23T15:16:55.520145+0000","last_fullsized":"2023-09-23T15:16:55.520145+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":218,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521629+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:16:55.521629+0000","last_peered":"2023-09-23T15:16:55.521629+0000","last_clean":"2023-09-23T15:16:55.521629+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:16:55.521629+0000","last_undegraded":"2023-09-23T15:16:55.521629+0000","last_fullsized":"2023-09-23T15:16:55.521629+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"230'1694","reported_seq":1710,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:08.739217+0000","last_change":"2023-09-23T15:16:44.271841+0000","last_active":"2023-09-23T15:17:08.739217+0000","last_peered":"2023-09-23T15:17:08.739217+0000","last_clean":"2023-09-23T15:17:08.739217+0000","last_became_active":"2023-09-23T15:16:44.271547+0000","last_became_peered":"2023-09-23T15:16:44.271547+0000","last_unstale":"2023-09-23T15:17:08.739217+0000","last_undegraded":"2023-09-23T15:17:08.739217+0000","last_fullsized":"2023-09-23T15:17:08.739217+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1694,"log_dups_size":0,"ondisk_log_size":1694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:03.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111017984,"num_objects":1694,"num_object_clones":0,"num_object_copies":5082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1694,"num_write_kb":108416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":527,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514694+0000","last_change":"2023-09-23T15:16:49.264186+0000","last_active":"2023-09-23T15:16:55.514694+0000","last_peered":"2023-09-23T15:16:55.514694+0000","last_clean":"2023-09-23T15:16:55.514694+0000","last_became_active":"2023-09-23T15:16:46.275415+0000","last_became_peered":"2023-09-23T15:16:46.275415+0000","last_unstale":"2023-09-23T15:16:55.514694+0000","last_undegraded":"2023-09-23T15:16:55.514694+0000","last_fullsized":"2023-09-23T15:16:55.514694+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.264096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.264096+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:25.863800+0000","scrub_duration":0.0040937940000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":189,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520410+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:55.520410+0000","last_peered":"2023-09-23T15:16:55.520410+0000","last_clean":"2023-09-23T15:16:55.520410+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:16:55.520410+0000","last_undegraded":"2023-09-23T15:16:55.520410+0000","last_fullsized":"2023-09-23T15:16:55.520410+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":190,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522322+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:16:55.522322+0000","last_peered":"2023-09-23T15:16:55.522322+0000","last_clean":"2023-09-23T15:16:55.522322+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:16:55.522322+0000","last_undegraded":"2023-09-23T15:16:55.522322+0000","last_fullsized":"2023-09-23T15:16:55.522322+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"230'1740","reported_seq":1755,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.509684+0000","last_change":"2023-09-23T15:16:44.263685+0000","last_active":"2023-09-23T15:17:09.509684+0000","last_peered":"2023-09-23T15:17:09.509684+0000","last_clean":"2023-09-23T15:17:09.509684+0000","last_became_active":"2023-09-23T15:16:44.263391+0000","last_became_peered":"2023-09-23T15:16:44.263391+0000","last_unstale":"2023-09-23T15:17:09.509684+0000","last_undegraded":"2023-09-23T15:17:09.509684+0000","last_fullsized":"2023-09-23T15:17:09.509684+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1740,"log_dups_size":0,"ondisk_log_size":1740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:33.410338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114032640,"num_objects":1740,"num_object_clones":0,"num_object_copies":5220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1740,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1740,"num_write_kb":111360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":528,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518995+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:16:55.518995+0000","last_peered":"2023-09-23T15:16:55.518995+0000","last_clean":"2023-09-23T15:16:55.518995+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:16:55.518995+0000","last_undegraded":"2023-09-23T15:16:55.518995+0000","last_fullsized":"2023-09-23T15:16:55.518995+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":193,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518116+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:16:55.518116+0000","last_peered":"2023-09-23T15:16:55.518116+0000","last_clean":"2023-09-23T15:16:55.518116+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:16:55.518116+0000","last_undegraded":"2023-09-23T15:16:55.518116+0000","last_fullsized":"2023-09-23T15:16:55.518116+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":188,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514914+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:16:55.514914+0000","last_peered":"2023-09-23T15:16:55.514914+0000","last_clean":"2023-09-23T15:16:55.514914+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:16:55.514914+0000","last_undegraded":"2023-09-23T15:16:55.514914+0000","last_fullsized":"2023-09-23T15:16:55.514914+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"230'1727","reported_seq":1743,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:08.784036+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:17:08.784036+0000","last_peered":"2023-09-23T15:17:08.784036+0000","last_clean":"2023-09-23T15:17:08.784036+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:17:08.784036+0000","last_undegraded":"2023-09-23T15:17:08.784036+0000","last_fullsized":"2023-09-23T15:17:08.784036+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1727,"log_dups_size":0,"ondisk_log_size":1727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113180672,"num_objects":1727,"num_object_clones":0,"num_object_copies":5181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1727,"num_write_kb":110528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":534,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521027+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:16:55.521027+0000","last_peered":"2023-09-23T15:16:55.521027+0000","last_clean":"2023-09-23T15:16:55.521027+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:16:55.521027+0000","last_undegraded":"2023-09-23T15:16:55.521027+0000","last_fullsized":"2023-09-23T15:16:55.521027+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":205,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521523+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:16:55.521523+0000","last_peered":"2023-09-23T15:16:55.521523+0000","last_clean":"2023-09-23T15:16:55.521523+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:16:55.521523+0000","last_undegraded":"2023-09-23T15:16:55.521523+0000","last_fullsized":"2023-09-23T15:16:55.521523+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":222,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515015+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:16:55.515015+0000","last_peered":"2023-09-23T15:16:55.515015+0000","last_clean":"2023-09-23T15:16:55.515015+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:16:55.515015+0000","last_undegraded":"2023-09-23T15:16:55.515015+0000","last_fullsized":"2023-09-23T15:16:55.515015+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"230'1765","reported_seq":1780,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.221620+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:17:09.221620+0000","last_peered":"2023-09-23T15:17:09.221620+0000","last_clean":"2023-09-23T15:17:09.221620+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:17:09.221620+0000","last_undegraded":"2023-09-23T15:17:09.221620+0000","last_fullsized":"2023-09-23T15:17:09.221620+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1765,"log_dups_size":0,"ondisk_log_size":1765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115671040,"num_objects":1765,"num_object_clones":0,"num_object_copies":5295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1765,"num_write_kb":112960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":483,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515823+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:16:55.515823+0000","last_peered":"2023-09-23T15:16:55.515823+0000","last_clean":"2023-09-23T15:16:55.515823+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:16:55.515823+0000","last_undegraded":"2023-09-23T15:16:55.515823+0000","last_fullsized":"2023-09-23T15:16:55.515823+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":213,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514373+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:16:55.514373+0000","last_peered":"2023-09-23T15:16:55.514373+0000","last_clean":"2023-09-23T15:16:55.514373+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:16:55.514373+0000","last_undegraded":"2023-09-23T15:16:55.514373+0000","last_fullsized":"2023-09-23T15:16:55.514373+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":205,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514875+0000","last_change":"2023-09-23T15:12:41.864145+0000","last_active":"2023-09-23T15:16:55.514875+0000","last_peered":"2023-09-23T15:16:55.514875+0000","last_clean":"2023-09-23T15:16:55.514875+0000","last_became_active":"2023-09-23T15:12:41.863809+0000","last_became_peered":"2023-09-23T15:12:41.863809+0000","last_unstale":"2023-09-23T15:16:55.514875+0000","last_undegraded":"2023-09-23T15:16:55.514875+0000","last_fullsized":"2023-09-23T15:16:55.514875+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:47:02.725141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"230'1755","reported_seq":1770,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.229668+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:17:09.229668+0000","last_peered":"2023-09-23T15:17:09.229668+0000","last_clean":"2023-09-23T15:17:09.229668+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:17:09.229668+0000","last_undegraded":"2023-09-23T15:17:09.229668+0000","last_fullsized":"2023-09-23T15:17:09.229668+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1755,"log_dups_size":0,"ondisk_log_size":1755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115015680,"num_objects":1755,"num_object_clones":0,"num_object_copies":5265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1755,"num_write_kb":112320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":529,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519743+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:16:55.519743+0000","last_peered":"2023-09-23T15:16:55.519743+0000","last_clean":"2023-09-23T15:16:55.519743+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:16:55.519743+0000","last_undegraded":"2023-09-23T15:16:55.519743+0000","last_fullsized":"2023-09-23T15:16:55.519743+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":223,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:17:04.820272+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:17:04.820272+0000","last_peered":"2023-09-23T15:17:04.820272+0000","last_clean":"2023-09-23T15:17:04.820272+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:17:04.820272+0000","last_undegraded":"2023-09-23T15:17:04.820272+0000","last_fullsized":"2023-09-23T15:17:04.820272+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":107,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515693+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:16:55.515693+0000","last_peered":"2023-09-23T15:16:55.515693+0000","last_clean":"2023-09-23T15:16:55.515693+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:16:55.515693+0000","last_undegraded":"2023-09-23T15:16:55.515693+0000","last_fullsized":"2023-09-23T15:16:55.515693+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"230'1757","reported_seq":1772,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.344677+0000","last_change":"2023-09-23T15:16:44.271607+0000","last_active":"2023-09-23T15:17:09.344677+0000","last_peered":"2023-09-23T15:17:09.344677+0000","last_clean":"2023-09-23T15:17:09.344677+0000","last_became_active":"2023-09-23T15:16:44.271334+0000","last_became_peered":"2023-09-23T15:16:44.271334+0000","last_unstale":"2023-09-23T15:17:09.344677+0000","last_undegraded":"2023-09-23T15:17:09.344677+0000","last_fullsized":"2023-09-23T15:17:09.344677+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1757,"log_dups_size":0,"ondisk_log_size":1757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:58.532783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115146752,"num_objects":1757,"num_object_clones":0,"num_object_copies":5271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1757,"num_write_kb":112448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":452,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522002+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:16:55.522002+0000","last_peered":"2023-09-23T15:16:55.522002+0000","last_clean":"2023-09-23T15:16:55.522002+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:16:55.522002+0000","last_undegraded":"2023-09-23T15:16:55.522002+0000","last_fullsized":"2023-09-23T15:16:55.522002+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":407,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:56.483954+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:16:56.483954+0000","last_peered":"2023-09-23T15:16:56.483954+0000","last_clean":"2023-09-23T15:16:56.483954+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:16:56.483954+0000","last_undegraded":"2023-09-23T15:16:56.483954+0000","last_fullsized":"2023-09-23T15:16:56.483954+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520043+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:16:55.520043+0000","last_peered":"2023-09-23T15:16:55.520043+0000","last_clean":"2023-09-23T15:16:55.520043+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:16:55.520043+0000","last_undegraded":"2023-09-23T15:16:55.520043+0000","last_fullsized":"2023-09-23T15:16:55.520043+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":152,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521470+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:16:55.521470+0000","last_peered":"2023-09-23T15:16:55.521470+0000","last_clean":"2023-09-23T15:16:55.521470+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:16:55.521470+0000","last_undegraded":"2023-09-23T15:16:55.521470+0000","last_fullsized":"2023-09-23T15:16:55.521470+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"230'1707","reported_seq":1721,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.240140+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:17:09.240140+0000","last_peered":"2023-09-23T15:17:09.240140+0000","last_clean":"2023-09-23T15:17:09.240140+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:17:09.240140+0000","last_undegraded":"2023-09-23T15:17:09.240140+0000","last_fullsized":"2023-09-23T15:17:09.240140+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1707,"log_dups_size":0,"ondisk_log_size":1707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1707,"num_write_kb":109248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":527,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519737+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:16:55.519737+0000","last_peered":"2023-09-23T15:16:55.519737+0000","last_clean":"2023-09-23T15:16:55.519737+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:16:55.519737+0000","last_undegraded":"2023-09-23T15:16:55.519737+0000","last_fullsized":"2023-09-23T15:16:55.519737+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":748,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520042+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:16:55.520042+0000","last_peered":"2023-09-23T15:16:55.520042+0000","last_clean":"2023-09-23T15:16:55.520042+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:16:55.520042+0000","last_undegraded":"2023-09-23T15:16:55.520042+0000","last_fullsized":"2023-09-23T15:16:55.520042+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515130+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:16:55.515130+0000","last_peered":"2023-09-23T15:16:55.515130+0000","last_clean":"2023-09-23T15:16:55.515130+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:16:55.515130+0000","last_undegraded":"2023-09-23T15:16:55.515130+0000","last_fullsized":"2023-09-23T15:16:55.515130+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":123,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:56.483914+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:16:56.483914+0000","last_peered":"2023-09-23T15:16:56.483914+0000","last_clean":"2023-09-23T15:16:56.483914+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:16:56.483914+0000","last_undegraded":"2023-09-23T15:16:56.483914+0000","last_fullsized":"2023-09-23T15:16:56.483914+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"230'1718","reported_seq":1734,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.505423+0000","last_change":"2023-09-23T15:16:44.269240+0000","last_active":"2023-09-23T15:17:09.505423+0000","last_peered":"2023-09-23T15:17:09.505423+0000","last_clean":"2023-09-23T15:17:09.505423+0000","last_became_active":"2023-09-23T15:16:44.268966+0000","last_became_peered":"2023-09-23T15:16:44.268966+0000","last_unstale":"2023-09-23T15:17:09.505423+0000","last_undegraded":"2023-09-23T15:17:09.505423+0000","last_fullsized":"2023-09-23T15:17:09.505423+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1718,"log_dups_size":0,"ondisk_log_size":1718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:01.694002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112590848,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1718,"num_write_kb":109952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":536,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522435+0000","last_change":"2023-09-23T15:16:46.276901+0000","last_active":"2023-09-23T15:16:55.522435+0000","last_peered":"2023-09-23T15:16:55.522435+0000","last_clean":"2023-09-23T15:16:55.522435+0000","last_became_active":"2023-09-23T15:16:46.276619+0000","last_became_peered":"2023-09-23T15:16:46.276619+0000","last_unstale":"2023-09-23T15:16:55.522435+0000","last_undegraded":"2023-09-23T15:16:55.522435+0000","last_fullsized":"2023-09-23T15:16:55.522435+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:27.244809+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":843,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520717+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:16:55.520717+0000","last_peered":"2023-09-23T15:16:55.520717+0000","last_clean":"2023-09-23T15:16:55.520717+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:16:55.520717+0000","last_undegraded":"2023-09-23T15:16:55.520717+0000","last_fullsized":"2023-09-23T15:16:55.520717+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":103,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514593+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:16:55.514593+0000","last_peered":"2023-09-23T15:16:55.514593+0000","last_clean":"2023-09-23T15:16:55.514593+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:16:55.514593+0000","last_undegraded":"2023-09-23T15:16:55.514593+0000","last_fullsized":"2023-09-23T15:16:55.514593+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":127,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.262009+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:17:05.262009+0000","last_peered":"2023-09-23T15:17:05.262009+0000","last_clean":"2023-09-23T15:17:05.262009+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:17:05.262009+0000","last_undegraded":"2023-09-23T15:17:05.262009+0000","last_fullsized":"2023-09-23T15:17:05.262009+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"230'1766","reported_seq":1782,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.482410+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:17:09.482410+0000","last_peered":"2023-09-23T15:17:09.482410+0000","last_clean":"2023-09-23T15:17:09.482410+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:17:09.482410+0000","last_undegraded":"2023-09-23T15:17:09.482410+0000","last_fullsized":"2023-09-23T15:17:09.482410+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1766,"log_dups_size":0,"ondisk_log_size":1766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115736576,"num_objects":1766,"num_object_clones":0,"num_object_copies":5298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1766,"num_write_kb":113024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":517,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520083+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:16:55.520083+0000","last_peered":"2023-09-23T15:16:55.520083+0000","last_clean":"2023-09-23T15:16:55.520083+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:16:55.520083+0000","last_undegraded":"2023-09-23T15:16:55.520083+0000","last_fullsized":"2023-09-23T15:16:55.520083+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":519,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522178+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:16:55.522178+0000","last_peered":"2023-09-23T15:16:55.522178+0000","last_clean":"2023-09-23T15:16:55.522178+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:16:55.522178+0000","last_undegraded":"2023-09-23T15:16:55.522178+0000","last_fullsized":"2023-09-23T15:16:55.522178+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":145,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.517951+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:16:55.517951+0000","last_peered":"2023-09-23T15:16:55.517951+0000","last_clean":"2023-09-23T15:16:55.517951+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:16:55.517951+0000","last_undegraded":"2023-09-23T15:16:55.517951+0000","last_fullsized":"2023-09-23T15:16:55.517951+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":180,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521781+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:16:55.521781+0000","last_peered":"2023-09-23T15:16:55.521781+0000","last_clean":"2023-09-23T15:16:55.521781+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:16:55.521781+0000","last_undegraded":"2023-09-23T15:16:55.521781+0000","last_fullsized":"2023-09-23T15:16:55.521781+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"230'1706","reported_seq":1722,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:08.782270+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:17:08.782270+0000","last_peered":"2023-09-23T15:17:08.782270+0000","last_clean":"2023-09-23T15:17:08.782270+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:17:08.782270+0000","last_undegraded":"2023-09-23T15:17:08.782270+0000","last_fullsized":"2023-09-23T15:17:08.782270+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1706,"log_dups_size":0,"ondisk_log_size":1706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111804416,"num_objects":1706,"num_object_clones":0,"num_object_copies":5118,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1706,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1706,"num_write_kb":109184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":483,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515739+0000","last_change":"2023-09-23T15:16:48.526093+0000","last_active":"2023-09-23T15:16:55.515739+0000","last_peered":"2023-09-23T15:16:55.515739+0000","last_clean":"2023-09-23T15:16:55.515739+0000","last_became_active":"2023-09-23T15:16:46.276681+0000","last_became_peered":"2023-09-23T15:16:46.276681+0000","last_unstale":"2023-09-23T15:16:55.515739+0000","last_undegraded":"2023-09-23T15:16:55.515739+0000","last_fullsized":"2023-09-23T15:16:55.515739+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:34.078052+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":314,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514503+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:16:55.514503+0000","last_peered":"2023-09-23T15:16:55.514503+0000","last_clean":"2023-09-23T15:16:55.514503+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:16:55.514503+0000","last_undegraded":"2023-09-23T15:16:55.514503+0000","last_fullsized":"2023-09-23T15:16:55.514503+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":176,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.337923+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:17:05.337923+0000","last_peered":"2023-09-23T15:17:05.337923+0000","last_clean":"2023-09-23T15:17:05.337923+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:17:05.337923+0000","last_undegraded":"2023-09-23T15:17:05.337923+0000","last_fullsized":"2023-09-23T15:17:05.337923+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":205,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522313+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:16:55.522313+0000","last_peered":"2023-09-23T15:16:55.522313+0000","last_clean":"2023-09-23T15:16:55.522313+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:16:55.522313+0000","last_undegraded":"2023-09-23T15:16:55.522313+0000","last_fullsized":"2023-09-23T15:16:55.522313+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"230'1825","reported_seq":1840,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.643707+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:17:09.643707+0000","last_peered":"2023-09-23T15:17:09.643707+0000","last_clean":"2023-09-23T15:17:09.643707+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:17:09.643707+0000","last_undegraded":"2023-09-23T15:17:09.643707+0000","last_fullsized":"2023-09-23T15:17:09.643707+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1825,"log_dups_size":0,"ondisk_log_size":1825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119603200,"num_objects":1825,"num_object_clones":0,"num_object_copies":5475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1825,"num_write_kb":116800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":452,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522018+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:16:55.522018+0000","last_peered":"2023-09-23T15:16:55.522018+0000","last_clean":"2023-09-23T15:16:55.522018+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:16:55.522018+0000","last_undegraded":"2023-09-23T15:16:55.522018+0000","last_fullsized":"2023-09-23T15:16:55.522018+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":851,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522471+0000","last_change":"2023-09-23T15:16:38.973875+0000","last_active":"2023-09-23T15:16:55.522471+0000","last_peered":"2023-09-23T15:16:55.522471+0000","last_clean":"2023-09-23T15:16:55.522471+0000","last_became_active":"2023-09-23T15:16:38.973269+0000","last_became_peered":"2023-09-23T15:16:38.973269+0000","last_unstale":"2023-09-23T15:16:55.522471+0000","last_undegraded":"2023-09-23T15:16:55.522471+0000","last_fullsized":"2023-09-23T15:16:55.522471+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:17:00.522554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":103,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:56.483936+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:16:56.483936+0000","last_peered":"2023-09-23T15:16:56.483936+0000","last_clean":"2023-09-23T15:16:56.483936+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:16:56.483936+0000","last_undegraded":"2023-09-23T15:16:56.483936+0000","last_fullsized":"2023-09-23T15:16:56.483936+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":181,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.337807+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:17:05.337807+0000","last_peered":"2023-09-23T15:17:05.337807+0000","last_clean":"2023-09-23T15:17:05.337807+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:17:05.337807+0000","last_undegraded":"2023-09-23T15:17:05.337807+0000","last_fullsized":"2023-09-23T15:17:05.337807+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"230'1772","reported_seq":1788,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.638298+0000","last_change":"2023-09-23T15:16:44.260057+0000","last_active":"2023-09-23T15:17:09.638298+0000","last_peered":"2023-09-23T15:17:09.638298+0000","last_clean":"2023-09-23T15:17:09.638298+0000","last_became_active":"2023-09-23T15:16:44.259815+0000","last_became_peered":"2023-09-23T15:16:44.259815+0000","last_unstale":"2023-09-23T15:17:09.638298+0000","last_undegraded":"2023-09-23T15:17:09.638298+0000","last_fullsized":"2023-09-23T15:17:09.638298+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1772,"log_dups_size":0,"ondisk_log_size":1772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:50.147826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116129792,"num_objects":1772,"num_object_clones":0,"num_object_copies":5316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1772,"num_write_kb":113408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":527,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514983+0000","last_change":"2023-09-23T15:16:52.317545+0000","last_active":"2023-09-23T15:16:55.514983+0000","last_peered":"2023-09-23T15:16:55.514983+0000","last_clean":"2023-09-23T15:16:55.514983+0000","last_became_active":"2023-09-23T15:16:46.275975+0000","last_became_peered":"2023-09-23T15:16:46.275975+0000","last_unstale":"2023-09-23T15:16:55.514983+0000","last_undegraded":"2023-09-23T15:16:55.514983+0000","last_fullsized":"2023-09-23T15:16:55.514983+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:52.317424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:52.317424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:29.771978+0000","scrub_duration":0.001669299,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":207,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:17:04.820473+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:17:04.820473+0000","last_peered":"2023-09-23T15:17:04.820473+0000","last_clean":"2023-09-23T15:17:04.820473+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:17:04.820473+0000","last_undegraded":"2023-09-23T15:17:04.820473+0000","last_fullsized":"2023-09-23T15:17:04.820473+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":691,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.261919+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:17:05.261919+0000","last_peered":"2023-09-23T15:17:05.261919+0000","last_clean":"2023-09-23T15:17:05.261919+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:17:05.261919+0000","last_undegraded":"2023-09-23T15:17:05.261919+0000","last_fullsized":"2023-09-23T15:17:05.261919+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":211,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.518209+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:16:55.518209+0000","last_peered":"2023-09-23T15:16:55.518209+0000","last_clean":"2023-09-23T15:16:55.518209+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:16:55.518209+0000","last_undegraded":"2023-09-23T15:16:55.518209+0000","last_fullsized":"2023-09-23T15:16:55.518209+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"230'1752","reported_seq":1768,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:09.635999+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:17:09.635999+0000","last_peered":"2023-09-23T15:17:09.635999+0000","last_clean":"2023-09-23T15:17:09.635999+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:17:09.635999+0000","last_undegraded":"2023-09-23T15:17:09.635999+0000","last_fullsized":"2023-09-23T15:17:09.635999+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":1752,"log_dups_size":0,"ondisk_log_size":1752,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1752,"num_write_kb":112128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":528,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519379+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:16:55.519379+0000","last_peered":"2023-09-23T15:16:55.519379+0000","last_clean":"2023-09-23T15:16:55.519379+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:16:55.519379+0000","last_undegraded":"2023-09-23T15:16:55.519379+0000","last_fullsized":"2023-09-23T15:16:55.519379+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":706,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521894+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:16:55.521894+0000","last_peered":"2023-09-23T15:16:55.521894+0000","last_clean":"2023-09-23T15:16:55.521894+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:16:55.521894+0000","last_undegraded":"2023-09-23T15:16:55.521894+0000","last_fullsized":"2023-09-23T15:16:55.521894+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":193,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.522407+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:16:55.522407+0000","last_peered":"2023-09-23T15:16:55.522407+0000","last_clean":"2023-09-23T15:16:55.522407+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:16:55.522407+0000","last_undegraded":"2023-09-23T15:16:55.522407+0000","last_fullsized":"2023-09-23T15:16:55.522407+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514520+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:16:55.514520+0000","last_peered":"2023-09-23T15:16:55.514520+0000","last_clean":"2023-09-23T15:16:55.514520+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:16:55.514520+0000","last_undegraded":"2023-09-23T15:16:55.514520+0000","last_fullsized":"2023-09-23T15:16:55.514520+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":183,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.337738+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:17:05.337738+0000","last_peered":"2023-09-23T15:17:05.337738+0000","last_clean":"2023-09-23T15:17:05.337738+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:17:05.337738+0000","last_undegraded":"2023-09-23T15:17:05.337738+0000","last_fullsized":"2023-09-23T15:17:05.337738+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":230,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.521894+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:16:55.521894+0000","last_peered":"2023-09-23T15:16:55.521894+0000","last_clean":"2023-09-23T15:16:55.521894+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:16:55.521894+0000","last_undegraded":"2023-09-23T15:16:55.521894+0000","last_fullsized":"2023-09-23T15:16:55.521894+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":208,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520478+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:16:55.520478+0000","last_peered":"2023-09-23T15:16:55.520478+0000","last_clean":"2023-09-23T15:16:55.520478+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:16:55.520478+0000","last_undegraded":"2023-09-23T15:16:55.520478+0000","last_fullsized":"2023-09-23T15:16:55.520478+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":222,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.515281+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:16:55.515281+0000","last_peered":"2023-09-23T15:16:55.515281+0000","last_clean":"2023-09-23T15:16:55.515281+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:16:55.515281+0000","last_undegraded":"2023-09-23T15:16:55.515281+0000","last_fullsized":"2023-09-23T15:16:55.515281+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":174,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519959+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:16:55.519959+0000","last_peered":"2023-09-23T15:16:55.519959+0000","last_clean":"2023-09-23T15:16:55.519959+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:16:55.519959+0000","last_undegraded":"2023-09-23T15:16:55.519959+0000","last_fullsized":"2023-09-23T15:16:55.519959+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":205,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514597+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:16:55.514597+0000","last_peered":"2023-09-23T15:16:55.514597+0000","last_clean":"2023-09-23T15:16:55.514597+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:16:55.514597+0000","last_undegraded":"2023-09-23T15:16:55.514597+0000","last_fullsized":"2023-09-23T15:16:55.514597+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":202,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.520195+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:16:55.520195+0000","last_peered":"2023-09-23T15:16:55.520195+0000","last_clean":"2023-09-23T15:16:55.520195+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:16:55.520195+0000","last_undegraded":"2023-09-23T15:16:55.520195+0000","last_fullsized":"2023-09-23T15:16:55.520195+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.519139+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:16:55.519139+0000","last_peered":"2023-09-23T15:16:55.519139+0000","last_clean":"2023-09-23T15:16:55.519139+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:16:55.519139+0000","last_undegraded":"2023-09-23T15:16:55.519139+0000","last_fullsized":"2023-09-23T15:16:55.519139+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":179,"reported_epoch":230,"state":"active+clean","last_fresh":"2023-09-23T15:17:05.339998+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:17:05.339998+0000","last_peered":"2023-09-23T15:17:05.339998+0000","last_clean":"2023-09-23T15:17:05.339998+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:17:05.339998+0000","last_undegraded":"2023-09-23T15:17:05.339998+0000","last_fullsized":"2023-09-23T15:17:05.339998+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":198,"reported_epoch":229,"state":"active+clean","last_fresh":"2023-09-23T15:16:55.514375+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:16:55.514375+0000","last_peered":"2023-09-23T15:16:55.514375+0000","last_clean":"2023-09-23T15:16:55.514375+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:16:55.514375+0000","last_undegraded":"2023-09-23T15:16:55.514375+0000","last_fullsized":"2023-09-23T15:16:55.514375+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":1830748160,"num_objects":27935,"num_object_clones":0,"num_object_copies":83805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":27935,"num_write_kb":1787840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5438767104,"data_stored":5438767104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":27935,"ondisk_log_size":27935,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201549,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463185,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":727748,"kb_used_data":547004,"kb_used_omap":0,"kb_used_meta":180736,"kb_avail":93017404,"statfs":{"total":95995035648,"available":95249821696,"internally_reserved":0,"allocated":560132096,"data_stored":559389443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":185073664},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:01 2023","interfaces":[{"interface":"back","average":{"1min":0.73899999999999999,"5min":1.851,"15min":2.7250000000000001},"min":{"1min":0.245,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":1.8149999999999999,"5min":41.491,"15min":108.39400000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.6379999999999999,"5min":2.4359999999999999,"15min":3},"min":{"1min":0.35399999999999998,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":8.2949999999999999,"5min":43.151000000000003,"15min":108.452},"last":2.278}]},{"osd":1,"last update":"Sat Sep 23 15:16:46 2023","interfaces":[{"interface":"back","average":{"1min":1.139,"5min":2.9489999999999998,"15min":3.23},"min":{"1min":0.29099999999999998,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":3.5470000000000002,"5min":43.377000000000002,"15min":43.377000000000002},"last":0.441},{"interface":"front","average":{"1min":1.1830000000000001,"5min":3.7890000000000001,"15min":3.6669999999999998},"min":{"1min":0.30599999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.1299999999999999,"5min":42.704000000000001,"15min":42.704000000000001},"last":2.3610000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:17:01 2023","interfaces":[{"interface":"back","average":{"1min":1.6240000000000001,"5min":5.2939999999999996,"15min":6.1619999999999999},"min":{"1min":0.32000000000000001,"5min":0.26200000000000001,"15min":0.255},"max":{"1min":7.694,"5min":112.352,"15min":180.374},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.1699999999999999,"5min":5.1799999999999997,"15min":6.1070000000000002},"min":{"1min":0.29899999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.4710000000000001,"5min":117.667,"15min":181.685},"last":2.448}]},{"osd":3,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":7.0979999999999999,"15min":3.1539999999999999},"min":{"1min":0.42699999999999999,"5min":0.249,"15min":0.249},"max":{"1min":8.1440000000000001,"5min":102.254,"15min":102.254},"last":3.5499999999999998},{"interface":"front","average":{"1min":1.8109999999999999,"5min":7.234,"15min":3.1640000000000001},"min":{"1min":0.40200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.7030000000000003,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.52700000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":1.0349999999999999,"15min":1.04},"min":{"1min":0.28799999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":1.7150000000000001,"5min":36.290999999999997,"15min":36.290999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.624,"5min":1.1279999999999999,"15min":1.1279999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.0019999999999998,"5min":36.758000000000003,"15min":36.758000000000003},"last":0.50800000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":2.0099999999999998,"15min":1.423},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":3.1789999999999998,"5min":131.90600000000001,"15min":131.90600000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.65400000000000003,"5min":2.0249999999999999,"15min":1.4590000000000001},"min":{"1min":0.317,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.0510000000000002,"5min":131.61000000000001,"15min":131.61000000000001},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:16:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.82499999999999996,"15min":1.0289999999999999},"min":{"1min":0.36599999999999999,"5min":0.29599999999999999,"15min":0.28999999999999998},"max":{"1min":1.8700000000000001,"5min":5.125,"15min":17},"last":2.306},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.77000000000000002,"15min":0.98999999999999999},"min":{"1min":0.438,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":1.8180000000000001,"5min":3.754,"15min":7.7809999999999997},"last":0.79000000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015222,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":888264,"kb_used_data":688832,"kb_used_omap":6,"kb_used_meta":199417,"kb_avail":92856888,"statfs":{"total":95995035648,"available":95085453312,"internally_reserved":0,"allocated":705363968,"data_stored":704274304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":204203083},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":3.1760000000000002,"15min":3.5939999999999999},"min":{"1min":0.19900000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":14.227,"5min":16.626000000000001,"15min":16.626000000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":3.7639999999999998,"5min":4.4459999999999997,"15min":4.5549999999999997},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":19.044,"5min":19.044,"15min":19.044},"last":0.28199999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":4.9660000000000002,"5min":4.3099999999999996,"15min":4.1879999999999997},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":33.880000000000003,"5min":33.880000000000003,"15min":33.880000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.5299999999999998,"5min":3.7320000000000002,"15min":4.0590000000000002},"min":{"1min":0.30199999999999999,"5min":0.253,"15min":0.253},"max":{"1min":10.843999999999999,"5min":16.882000000000001,"15min":16.882000000000001},"last":0.32000000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":15.66,"5min":9.3330000000000002,"15min":10.308},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":211.405,"5min":211.405,"15min":211.405},"last":0.39500000000000002},{"interface":"front","average":{"1min":15.292999999999999,"5min":9.8729999999999993,"15min":11.667999999999999},"min":{"1min":0.34799999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":210.988,"5min":210.988,"15min":210.988},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.952,"5min":2.7469999999999999,"15min":2.7130000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.430999999999999,"5min":11.430999999999999,"15min":11.430999999999999},"last":68.805999999999997},{"interface":"front","average":{"1min":1.3700000000000001,"5min":2.887,"15min":3.3340000000000001},"min":{"1min":0.39300000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":10.750999999999999,"5min":16.779,"15min":16.779},"last":68.861000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1619999999999999,"5min":3.1099999999999999,"15min":3.556},"min":{"1min":0.39100000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11.84,"5min":16.166,"15min":16.166},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.956,"5min":3.0569999999999999,"15min":2.9500000000000002},"min":{"1min":0.35799999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":11.487,"5min":11.487,"15min":11.487},"last":0.46500000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1659999999999999,"5min":3.0920000000000001,"15min":3.5699999999999998},"min":{"1min":0.442,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":11.797000000000001,"5min":16.074000000000002,"15min":16.074000000000002},"last":70.602000000000004},{"interface":"front","average":{"1min":3.0129999999999999,"5min":3.0529999999999999,"15min":2.8500000000000001},"min":{"1min":0.46500000000000002,"5min":0.377,"15min":0.377},"max":{"1min":11.531000000000001,"5min":12.448,"15min":12.448},"last":72.239999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0190000000000001,"5min":3.1099999999999999,"15min":3.141},"min":{"1min":0.495,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":11.561,"5min":11.561,"15min":11.561},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.4370000000000001,"5min":2.9399999999999999,"15min":3.2160000000000002},"min":{"1min":0.46899999999999997,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":10.801,"5min":16.841000000000001,"15min":16.841000000000001},"last":0.47999999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893272,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251216,"kb_used_data":907592,"kb_used_omap":8,"kb_used_meta":343607,"kb_avail":92493936,"statfs":{"total":95995035648,"available":94713790464,"internally_reserved":0,"allocated":929374208,"data_stored":928392758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":351854164},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:16:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0409999999999999,"5min":3.048,"15min":3.8769999999999998},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":15.462,"5min":31.510999999999999,"15min":31.510999999999999},"last":1.2829999999999999},{"interface":"front","average":{"1min":2.048,"5min":3.1949999999999998,"15min":4.2050000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":12.099,"5min":31.378,"15min":31.378},"last":2.9020000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":3.4289999999999998,"5min":7.133,"15min":3.5539999999999998},"min":{"1min":0.26200000000000001,"5min":0.189,"15min":0.17000000000000001},"max":{"1min":13.593,"5min":103.81,"15min":122.20099999999999},"last":4.0679999999999996},{"interface":"front","average":{"1min":3.4279999999999999,"5min":6.077,"15min":3.0920000000000001},"min":{"1min":0.26900000000000002,"5min":0.22500000000000001,"15min":0.18099999999999999},"max":{"1min":9.7479999999999993,"5min":108.19499999999999,"15min":122.071},"last":3.7549999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:16:32 2023","interfaces":[{"interface":"back","average":{"1min":16.754999999999999,"5min":9.8650000000000002,"15min":3.9460000000000002},"min":{"1min":0.27200000000000002,"5min":0.20599999999999999,"15min":0.20000000000000001},"max":{"1min":146.315,"5min":146.315,"15min":146.315},"last":1.649},{"interface":"front","average":{"1min":17.911999999999999,"5min":10.164999999999999,"15min":4.3300000000000001},"min":{"1min":0.307,"5min":0.27700000000000002,"15min":0.252},"max":{"1min":145.517,"5min":145.517,"15min":145.517},"last":1.798}]},{"osd":4,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":1.4510000000000001,"5min":2.5230000000000001,"15min":1.649},"min":{"1min":0.32400000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":3.9950000000000001,"5min":31.341999999999999,"15min":31.341999999999999},"last":2.867},{"interface":"front","average":{"1min":2.4710000000000001,"5min":3.282,"15min":1.72},"min":{"1min":0.29699999999999999,"5min":0.27800000000000002,"15min":0.27000000000000002},"max":{"1min":13.566000000000001,"5min":99.168999999999997,"15min":99.168999999999997},"last":1.605}]},{"osd":5,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5030000000000001,"5min":2.992,"15min":1.768},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.29699999999999999},"max":{"1min":9.6020000000000003,"5min":109.369,"15min":109.369},"last":1.702},{"interface":"front","average":{"1min":1.5029999999999999,"5min":3.2170000000000001,"15min":1.857},"min":{"1min":0.42999999999999999,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":4.4219999999999997,"5min":109.387,"15min":109.387},"last":2.8290000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.996,"15min":1.752},"min":{"1min":0.38200000000000001,"5min":0.309,"15min":0.309},"max":{"1min":13.529999999999999,"5min":99.114999999999995,"15min":99.114999999999995},"last":1.5589999999999999},{"interface":"front","average":{"1min":2.4609999999999999,"5min":2.8650000000000002,"15min":1.677},"min":{"1min":0.378,"5min":0.313,"15min":0.313},"max":{"1min":9.6460000000000008,"5min":98.921999999999997,"15min":98.921999999999997},"last":1.734}]},{"osd":7,"last update":"Sat Sep 23 15:16:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5670000000000002,"5min":3.181,"15min":1.802},"min":{"1min":0.44600000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.6690000000000005,"5min":99.549999999999997,"15min":99.549999999999997},"last":1.77},{"interface":"front","average":{"1min":1.702,"5min":2.2770000000000001,"15min":1.5549999999999999},"min":{"1min":0.46700000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.9720000000000004,"5min":31.225999999999999,"15min":31.225999999999999},"last":2.9620000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566219,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1112200,"kb_used_data":747456,"kb_used_omap":11,"kb_used_meta":364724,"kb_avail":92632952,"statfs":{"total":95995035648,"available":94856142848,"internally_reserved":0,"allocated":765394944,"data_stored":764424011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":373478160},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":4.79,"5min":4.9969999999999999,"15min":2.2229999999999999},"min":{"1min":0.27400000000000002,"5min":0.184,"15min":0.184},"max":{"1min":16.085000000000001,"5min":87.132999999999996,"15min":87.132999999999996},"last":0.69799999999999995},{"interface":"front","average":{"1min":3.931,"5min":3.8490000000000002,"15min":1.831},"min":{"1min":0.27500000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.833,"5min":84.820999999999998,"15min":84.820999999999998},"last":0.71399999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":5.3650000000000002,"5min":5.5570000000000004,"15min":5.2119999999999997},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":31.513999999999999,"5min":84.641000000000005,"15min":84.641000000000005},"last":1.248},{"interface":"front","average":{"1min":3.7690000000000001,"5min":4.0259999999999998,"15min":3.0409999999999999},"min":{"1min":0.28499999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":18.821000000000002,"5min":84.751999999999995,"15min":84.751999999999995},"last":3.2919999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:16:30 2023","interfaces":[{"interface":"back","average":{"1min":8.1159999999999997,"5min":7.4589999999999996,"15min":3.177},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.219},"max":{"1min":29.297000000000001,"5min":87.108000000000004,"15min":87.108000000000004},"last":0.42599999999999999},{"interface":"front","average":{"1min":7.899,"5min":6.8819999999999997,"15min":3.0369999999999999},"min":{"1min":0.32200000000000001,"5min":0.23499999999999999,"15min":0.20300000000000001},"max":{"1min":31.440999999999999,"5min":86.158000000000001,"15min":86.158000000000001},"last":1.28}]},{"osd":4,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":3.129,"15min":1.831},"min":{"1min":0.33600000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":10.824,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.67800000000000005},{"interface":"front","average":{"1min":2.8849999999999998,"5min":4.016,"15min":2.4020000000000001},"min":{"1min":0.42399999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":12.988,"5min":87.019999999999996,"15min":87.019999999999996},"last":3.3500000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.3340000000000001,"5min":3.266,"15min":2.0579999999999998},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":16.193999999999999,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.0870000000000002,"5min":2.8690000000000002,"15min":1.9179999999999999},"min":{"1min":0.441,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":10.865,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.60699999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":2.0649999999999999,"5min":3.1200000000000001,"15min":2.0369999999999999},"min":{"1min":0.42899999999999999,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":10.891999999999999,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.64900000000000002},{"interface":"front","average":{"1min":2.9729999999999999,"5min":3.5750000000000002,"15min":2.0939999999999999},"min":{"1min":0.39100000000000001,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":12.949999999999999,"5min":87.055000000000007,"15min":87.055000000000007},"last":0.73799999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:16:05 2023","interfaces":[{"interface":"back","average":{"1min":3.4060000000000001,"5min":3.3519999999999999,"15min":1.915},"min":{"1min":0.42799999999999999,"5min":0.36099999999999999,"15min":0.31900000000000001},"max":{"1min":16.239000000000001,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.504},{"interface":"front","average":{"1min":1.966,"5min":3.0630000000000002,"15min":1.8049999999999999},"min":{"1min":0.48899999999999999,"5min":0.38600000000000001,"15min":0.34200000000000003},"max":{"1min":10.916,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.76700000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402693,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1494120,"kb_used_data":1148576,"kb_used_omap":3,"kb_used_meta":345532,"kb_avail":92251032,"statfs":{"total":95995035648,"available":94465056768,"internally_reserved":0,"allocated":1176141824,"data_stored":1175194912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":353824937},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2890000000000001,"5min":2.9510000000000001,"15min":2.2050000000000001},"min":{"1min":0.20100000000000001,"5min":0.19600000000000001,"15min":0.17899999999999999},"max":{"1min":18.864999999999998,"5min":25.111999999999998,"15min":79.611000000000004},"last":2.5950000000000002},{"interface":"front","average":{"1min":4.8929999999999998,"5min":3.222,"15min":2.3250000000000002},"min":{"1min":0.315,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":18.800000000000001,"5min":18.800000000000001,"15min":79.656000000000006},"last":0.59799999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":3.7490000000000001,"5min":4.5339999999999998,"15min":4.5679999999999996},"min":{"1min":0.25600000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":17.939,"5min":28.315000000000001,"15min":28.315000000000001},"last":1.603},{"interface":"front","average":{"1min":3.673,"5min":4.3259999999999996,"15min":4.6719999999999997},"min":{"1min":0.29499999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":18.832999999999998,"5min":21.539000000000001,"15min":21.539000000000001},"last":2.7130000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":5.4320000000000004,"5min":4.3650000000000002,"15min":4.0119999999999996},"min":{"1min":0.33400000000000002,"5min":0.255,"15min":0.22},"max":{"1min":18.893999999999998,"5min":61.732999999999997,"15min":109.077},"last":2.8730000000000002},{"interface":"front","average":{"1min":4.8819999999999997,"5min":4.5339999999999998,"15min":4.2439999999999998},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.25700000000000001},"max":{"1min":21.661999999999999,"5min":60.305999999999997,"15min":119.72799999999999},"last":1.3089999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6560000000000001,"5min":2.9609999999999999,"15min":1.6359999999999999},"min":{"1min":0.40100000000000002,"5min":0.29499999999999998,"15min":0.27600000000000002},"max":{"1min":12.958,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":3.0539999999999998,"5min":2.8919999999999999,"15min":1.79},"min":{"1min":0.34999999999999998,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":10.497999999999999,"5min":52.228999999999999,"15min":52.228999999999999},"last":1.4059999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.6600000000000001,"5min":2.621,"15min":1.776},"min":{"1min":0.371,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":12.999000000000001,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.65400000000000003},{"interface":"front","average":{"1min":3.5670000000000002,"5min":2.6139999999999999,"15min":1.99},"min":{"1min":0.40200000000000002,"5min":0.315,"15min":0.315},"max":{"1min":12.667999999999999,"5min":12.667999999999999,"15min":56.981999999999999},"last":1.3799999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.8570000000000002,"5min":3.3319999999999999,"15min":1.911},"min":{"1min":0.432,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.029999999999999,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.69899999999999995},{"interface":"front","average":{"1min":3.3639999999999999,"5min":3.323,"15min":2.1299999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.32300000000000001},"max":{"1min":11.117000000000001,"5min":78.028000000000006,"15min":78.028000000000006},"last":1.339}]},{"osd":7,"last update":"Sat Sep 23 15:16:35 2023","interfaces":[{"interface":"back","average":{"1min":3.798,"5min":2.5840000000000001,"15min":1.911},"min":{"1min":0.47599999999999998,"5min":0.34899999999999998,"15min":0.28299999999999997},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":14.885999999999999},"last":1.427},{"interface":"front","average":{"1min":3.5630000000000002,"5min":2.593,"15min":1.5569999999999999},"min":{"1min":0.42199999999999999,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":18.696000000000002,"5min":18.696000000000002,"15min":18.696000000000002},"last":2.46}]}]},{"osd":4,"up_from":31,"seq":133143986458,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":876164,"kb_used_data":667900,"kb_used_omap":0,"kb_used_meta":208256,"kb_avail":92868988,"statfs":{"total":95995035648,"available":95097843712,"internally_reserved":0,"allocated":683929600,"data_stored":683186970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":213254144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.1970000000000001,"5min":2.4729999999999999,"15min":2.6640000000000001},"min":{"1min":0.28299999999999997,"5min":0.219,"15min":0.219},"max":{"1min":9.4260000000000002,"5min":65.988,"15min":122.69199999999999},"last":0.91500000000000004},{"interface":"front","average":{"1min":2.0609999999999999,"5min":2.7770000000000001,"15min":2.7999999999999998},"min":{"1min":0.30399999999999999,"5min":0.221,"15min":0.221},"max":{"1min":8.4540000000000006,"5min":65.396000000000001,"15min":122.745},"last":3.012}]},{"osd":1,"last update":"Sat Sep 23 15:16:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":3.0219999999999998,"15min":4.04},"min":{"1min":0.33200000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.6589999999999998,"5min":53.832000000000001,"15min":53.832000000000001},"last":1.177},{"interface":"front","average":{"1min":2.2989999999999999,"5min":3.7810000000000001,"15min":4.6449999999999996},"min":{"1min":0.28899999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":11.443,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.472,"5min":4.2510000000000003,"15min":3.1669999999999998},"min":{"1min":0.33000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":6.6920000000000002,"5min":148.08000000000001,"15min":148.08000000000001},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.9299999999999999,"5min":4.585,"15min":3.4329999999999998},"min":{"1min":0.37,"5min":0.32600000000000001,"15min":0.25700000000000001},"max":{"1min":9.6760000000000002,"5min":147.88900000000001,"15min":147.88900000000001},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":4.319,"5min":9.6549999999999994,"15min":4.0369999999999999},"min":{"1min":0.46600000000000003,"5min":0.28399999999999997,"15min":0.252},"max":{"1min":23.715,"5min":163.97499999999999,"15min":163.97499999999999},"last":0.875},{"interface":"front","average":{"1min":4.5860000000000003,"5min":9.7769999999999992,"15min":3.8399999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":22.876000000000001,"5min":164.447,"15min":164.447},"last":0.65900000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:16:44 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.4730000000000001,"15min":1.5189999999999999},"min":{"1min":0.27000000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":3.8100000000000001,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.628,"5min":1.623,"15min":1.3999999999999999},"min":{"1min":0.309,"5min":0.27100000000000002,"15min":0.252},"max":{"1min":2.4220000000000002,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:16:25 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":3.6160000000000001,"15min":2.0510000000000002},"min":{"1min":0.29999999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.3820000000000001,"5min":210.44300000000001,"15min":210.44300000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.89600000000000002,"5min":3.7080000000000002,"15min":1.956},"min":{"1min":0.33100000000000002,"5min":0.316,"15min":0.316},"max":{"1min":3.7749999999999999,"5min":210.78899999999999,"15min":210.78899999999999},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:16:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.621,"15min":1.123},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.417,"5min":5.399,"15min":18.23},"last":0.94999999999999996},{"interface":"front","average":{"1min":1.038,"5min":0.86599999999999999,"15min":1.2030000000000001},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.314},"max":{"1min":3.851,"5min":6.1399999999999997,"15min":14.653},"last":0.56799999999999995}]}]},{"osd":5,"up_from":39,"seq":167503724821,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":806076,"kb_used_data":589364,"kb_used_omap":0,"kb_used_meta":216704,"kb_avail":92939076,"statfs":{"total":95995035648,"available":95169613824,"internally_reserved":0,"allocated":603508736,"data_stored":602770605,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":221904896},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.23,"5min":1.893,"15min":1.454},"min":{"1min":0.248,"5min":0.222,"15min":0.20599999999999999},"max":{"1min":6.2859999999999996,"5min":16.981999999999999,"15min":24.962},"last":1.554},{"interface":"front","average":{"1min":0.98999999999999999,"5min":2.1200000000000001,"15min":1.486},"min":{"1min":0.247,"5min":0.247,"15min":0.222},"max":{"1min":5.1950000000000003,"5min":18.814,"15min":25.317},"last":0.504}]},{"osd":1,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":2.327,"15min":1.891},"min":{"1min":0.248,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9329999999999998,"5min":18.082999999999998,"15min":18.082999999999998},"last":1.464},{"interface":"front","average":{"1min":1.46,"5min":2.6120000000000001,"15min":2.383},"min":{"1min":0.25900000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.5199999999999996,"5min":17.963999999999999,"15min":17.963999999999999},"last":0.435}]},{"osd":2,"last update":"Sat Sep 23 15:17:04 2023","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":2.6269999999999998,"15min":2.4969999999999999},"min":{"1min":0.29299999999999998,"5min":0.27600000000000002,"15min":0.23599999999999999},"max":{"1min":4.6429999999999998,"5min":33.68,"15min":97.087999999999994},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.202,"5min":2.927,"15min":2.528},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":13.766999999999999,"5min":33.707999999999998,"15min":97.305999999999997},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":10.236000000000001,"5min":6.5570000000000004,"15min":3.1299999999999999},"min":{"1min":0.36399999999999999,"5min":0.31900000000000001,"15min":0.27500000000000002},"max":{"1min":118.208,"5min":118.208,"15min":118.208},"last":1.234},{"interface":"front","average":{"1min":8.9410000000000007,"5min":6.2590000000000003,"15min":2.9009999999999998},"min":{"1min":0.33400000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":118.42,"5min":118.42,"15min":118.42},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:16:50 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":2.956,"15min":1.7250000000000001},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":3.1920000000000002,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.31},{"interface":"front","average":{"1min":0.56899999999999995,"5min":2.8180000000000001,"15min":1.792},"min":{"1min":0.29199999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8410000000000002,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.45800000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:16:06 2023","interfaces":[{"interface":"back","average":{"1min":0.73599999999999999,"5min":0.88900000000000001,"15min":1.1339999999999999},"min":{"1min":0.317,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.5550000000000002,"5min":5.8760000000000003,"15min":15.196},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.72499999999999998,"5min":0.86499999999999999,"15min":1.0229999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.9990000000000001,"5min":4.8819999999999997,"15min":14.965999999999999},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:16:36 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.86599999999999999,"15min":1.2829999999999999},"min":{"1min":0.34200000000000003,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":2.0249999999999999,"5min":8.2200000000000006,"15min":15.301},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.68300000000000005,"5min":0.79100000000000004,"15min":1.2589999999999999},"min":{"1min":0.38500000000000001,"5min":0.311,"15min":0.311},"max":{"1min":3.5379999999999998,"5min":5.6559999999999997,"15min":15.762},"last":0.41299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":922812416,"data_stored":922812416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":700579840,"data_stored":700579840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":783220736,"data_stored":783220736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1168769024,"data_stored":1168769024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":683278336,"data_stored":683278336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":600899584,"data_stored":600899584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":579207168,"data_stored":579207168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:17:12.491 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:17:12.491 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:17:12.491 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T15:17:12.492 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:17:12.492 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:17:12.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T15:17:12.599 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:12.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:12.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:12.628 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:12.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:12.737 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:12.796 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:12.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:12.800 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:12.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:12.931 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:13.054 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:13.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:13.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:13.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:17:13.132606+0000 min lat: 0.00111397 max lat: 0.361358 avg lat: 0.00969621 2023-09-23T15:17:13.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:17:13.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 33019 33003 103.063 75.625 0.0132243 0.00969621 2023-09-23T15:17:13.168 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:13.199 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:13.312 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:13.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:13.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:13.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:13.457 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:13.619 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:13.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:13.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:13.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:13.771 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:13.817 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:13.819 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:13.827 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:17:13.828 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:17:13.828 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:17:13.828 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:17:13.828 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:17:13.920 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:13.954 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:14.069 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[222878]: pgmap v644: 109 pgs: 109 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T15:17:14.069 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:14.069 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2547403169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.070 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.070 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:14.073 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:17:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:14.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:14 smithi027 ceph-mon[195233]: pgmap v644: 109 pgs: 109 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T15:17:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2547403169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 34212 34196 101.707 74.5625 0.00483354 0.00982626 2023-09-23T15:17:14.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:14.238 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:14.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[215646]: pgmap v644: 109 pgs: 109 active+clean; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 89 MiB/s wr, 1.42k op/s 2023-09-23T15:17:14.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:14.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2547403169' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:14.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:17:14.882 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:15.085 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:15.101 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:15.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:15.110 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:15.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 35404 35388 100.47 74.5 0.00650868 0.00994656 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[222878]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:15 smithi017 ceph-mon[215646]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:15.281 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:15.342 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:15.342 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.342 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:15.342 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: osdmap e231: 8 total, 8 up, 7 in 2023-09-23T15:17:15.342 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:15 smithi027 ceph-mon[195233]: failed to encode map e231 with expected crc 2023-09-23T15:17:15.607 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:15.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:15.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:15.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:15.771 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:15.799 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:15.812 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:15.850 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:15.889 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:15.890 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:15.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:15.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:16.031 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:16.033 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:16.079 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:16.122 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:16.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 36572 36556 99.2763 73 0.00628065 0.010066 2023-09-23T15:17:16.156 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:16.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.167 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:17:16.190 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:16 smithi017 ceph-mon[215646]: pgmap v646: 109 pgs: 109 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T15:17:16.191 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/977704599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:16.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:16 smithi017 ceph-mon[222878]: pgmap v646: 109 pgs: 109 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T15:17:16.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/977704599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:16.297 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:16.304 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:16 smithi027 ceph-mon[195233]: pgmap v646: 109 pgs: 109 active+clean; 1.9 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 65 MiB/s wr, 1.03k op/s 2023-09-23T15:17:16.304 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/977704599' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:16.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:16.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:16.476 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:16.711 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:16.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:16.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:16.861 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:16.861 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:16.882 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:17.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:17:17.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 37908 37892 98.619 83.5 0.00594089 0.0101345 2023-09-23T15:17:17.401 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:17.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:17.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:17.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:17.538 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:17.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:17:17.741 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:17.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:17.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:17.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:17.881 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:17.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:17.941 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:17.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:17.967 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:17:17.968 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:17:17.968 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:17:18.046 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:18.086 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:18.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 38971 38955 97.332 66.4375 0.01023 0.010269 2023-09-23T15:17:18.154 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:18.166 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:18 smithi027 ceph-mon[195233]: pgmap v647: 109 pgs: 109 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T15:17:18.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:18.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:18.222 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:18.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:18.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:18 smithi017 ceph-mon[215646]: pgmap v647: 109 pgs: 109 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T15:17:18.337 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:18] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-09-23T15:17:18.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:18 smithi017 ceph-mon[222878]: pgmap v647: 109 pgs: 109 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.29k op/s 2023-09-23T15:17:18.342 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:18.414 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:18.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:18.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:18.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:18.553 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:18.661 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:18.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:18.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:18.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:18.832 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:19.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 40163 40147 96.454 74.5 0.132549 0.010358 2023-09-23T15:17:19.193 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:19.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:19.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:19.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:19.332 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:19.470 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:19.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:19.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:19.576 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:19.584 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:17:19.618 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:19.675 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:19.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:19.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:19.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:19.857 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:20.022 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:20.022 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:20.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:20 smithi017 ceph-mon[222878]: pgmap v648: 109 pgs: 109 active+clean; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T15:17:20.097 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:20 smithi017 ceph-mon[215646]: pgmap v648: 109 pgs: 109 active+clean; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T15:17:20.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 41552 41536 96.0968 86.8125 0.0109438 0.0104013 2023-09-23T15:17:20.171 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:20.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:20.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:20.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:20 smithi027 ceph-mon[195233]: pgmap v648: 109 pgs: 109 active+clean; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.28k op/s 2023-09-23T15:17:20.390 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:20.508 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:20.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:20.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:20.513 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:20.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:20.710 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:20.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:17:20.913 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:20.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:20.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:21.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:21.061 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:21.124 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:21.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:21.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:21.133 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 42707 42691 95.243 72.1875 0.0144935 0.0104935 2023-09-23T15:17:21.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:21.305 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:21.363 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:21.403 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:21.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:21.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:21.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:21.577 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:21.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:21.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:21.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:21.813 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:21.858 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:21.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:21.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:22.000 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:22.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:22.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:22.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:22.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 43709 43693 94.1185 62.625 0.0103338 0.0106176 2023-09-23T15:17:22.138 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:17:22.171 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:22.197 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:22.240 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:22.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:22 smithi027 ceph-mon[195233]: pgmap v649: 109 pgs: 109 active+clean; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:17:22.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3917406290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.242 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:22.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:22.343 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:22.374 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[215646]: pgmap v649: 109 pgs: 109 active+clean; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:17:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3917406290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[222878]: pgmap v649: 109 pgs: 109 active+clean; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 78 MiB/s wr, 1.24k op/s 2023-09-23T15:17:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3917406290' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:22.596 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:22.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:22.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:22.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:22.764 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:23.130 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:23.130 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:23.132 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 44847 44831 93.3521 71.125 0.0125087 0.0107058 2023-09-23T15:17:23.232 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:23.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:23.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:23.338 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[222878]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.402 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:23 smithi017 ceph-mon[215646]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.411 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:23.512 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:23.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:23.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:23.581 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: osdmap e232: 8 total, 8 up, 7 in 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:23 smithi027 ceph-mon[195233]: failed to encode map e232 with expected crc 2023-09-23T15:17:23.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:23.621 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:23.653 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:23.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:23.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:23.714 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:23.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:23.730 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:23.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:23.805 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:23.818 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:23.846 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:23.850 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.860 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:17:23.861 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:17:24.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:17:24.023 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:24.029 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:24.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:24.134 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 46068 46052 92.8024 76.3125 0.016189 0.0107705 2023-09-23T15:17:24.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:24 smithi017 ceph-mon[215646]: pgmap v651: 109 pgs: 109 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T15:17:24.146 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3408482227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:24.152 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T15:17:24.152 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:17:24.152 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T15:17:24.152 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 1.000000 2023-09-23T15:17:24.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T15:17:24.209 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:24 smithi027 ceph-mon[195233]: pgmap v651: 109 pgs: 109 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T15:17:24.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3408482227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:24.400 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:24.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:24.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:24 smithi017 ceph-mon[222878]: pgmap v651: 109 pgs: 109 active+clean; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 84 MiB/s wr, 1.34k op/s 2023-09-23T15:17:24.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3408482227' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:24.485 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:17:24.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:24.536 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:24.603 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:24.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:24.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:24.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:24.784 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:25.057 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:25.079 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:25.133 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 47025 47009 91.7717 59.8125 0.0128655 0.0108926 2023-09-23T15:17:25.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:25.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:17:25.247 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:25.274 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:25.314 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:25.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:25.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:25.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:25.455 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:25.723 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:25.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:25.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:25.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:25.918 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:26.018 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:26.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:26.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:26.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:26.138 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 48074 48058 90.9625 65.5625 0.0134917 0.0109887 2023-09-23T15:17:26.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:26 smithi017 ceph-mon[222878]: pgmap v652: 109 pgs: 109 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T15:17:26.170 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:26.224 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:26.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.228 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18560 5.4 5:24ff48df:::benchmark_data_smithi017_288048_object18559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 581.06243710700005, 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20308895499999999, 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18560 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163380+0000", 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163378+0000", 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163445+0000", 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6580000000000003e-05 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163462+0000", 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7041000000000002e-05 2023-09-23T15:17:26.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163469+0000", 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3359999999999997e-06 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163522+0000", 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2321000000000003e-05 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163644+0000", 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012266800000000001 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163794+0000", 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015003099999999999 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165622+0000", 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00182759 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165900+0000", 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027825799999999998 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366389+0000", 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20048898100000001 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366440+0000", 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0593000000000002e-05 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366469+0000", 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9448e-05 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.235 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18562 5.4 5:263b20e9:::benchmark_data_smithi017_288048_object18561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 581.06233243099996, 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.203517166, 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18562 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163485+0000", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163484+0000", 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:26.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163517+0000", 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2676999999999999e-05 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163526+0000", 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4999999999999999e-06 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.163531+0000", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9010000000000002e-06 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164584+0000", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001053165 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164706+0000", 2023-09-23T15:17:26.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012267800000000001 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.164859+0000", 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152922 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.165907+0000", 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001047926 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.166956+0000", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001048542 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366932+0000", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.199975919 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.366968+0000", 2023-09-23T15:17:26.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5896000000000003e-05 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.367002+0000", 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4619000000000002e-05 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:18578 5.4 5:24fb8b16:::benchmark_data_smithi017_288048_object18577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T15:17:26.242 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 581.05147571500004, 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.1939555, 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18578 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174342+0000", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174341+0000", 2023-09-23T15:17:26.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174411+0000", 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9587999999999998e-05 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174422+0000", 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1327e-05 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174427+0000", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6149999999999999e-06 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174467+0000", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0268000000000001e-05 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174552+0000", 2023-09-23T15:17:26.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4720000000000002e-05 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.174641+0000", 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9091999999999993e-05 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175376+0000", 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073561399999999999 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.175539+0000", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016279399999999999 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368206+0000", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19266702899999999 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368260+0000", 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3872999999999997e-05 2023-09-23T15:17:26.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:07:45.368297+0000", 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7327999999999999e-05 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:46777 5.4 5:2f7b1ca2:::benchmark_data_smithi017_288048_object46776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e122)", 2023-09-23T15:17:26.249 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 564.34179616400002, 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19520114399999999, 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 46777 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:17:26.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884021+0000", 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884020+0000", 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884079+0000", 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9199e-05 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884093+0000", 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3969e-05 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884099+0000", 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8220000000000003e-06 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884150+0000", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1026000000000002e-05 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884247+0000", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6985999999999996e-05 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.884345+0000", 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8879e-05 2023-09-23T15:17:26.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885695+0000", 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00134965 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:01.885893+0000", 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019828700000000001 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079139+0000", 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.193245841 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079186+0000", 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7052000000000003e-05 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:08:02.079222+0000", 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6183999999999998e-05 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.255 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106173 5.9 5:9fa66f55:::benchmark_data_smithi017_288048_object106172:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 505.51125486000001, 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26060794199999998, 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106173 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.256 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714563+0000", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714562+0000", 2023-09-23T15:17:26.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714598+0000", 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5794999999999999e-05 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714607+0000", 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2240000000000001e-06 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.714612+0000", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1070000000000004e-06 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718260+0000", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036480330000000002 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718380+0000", 2023-09-23T15:17:26.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120047 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.718533+0000", 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015235699999999999 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.724954+0000", 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064215239999999996 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728531+0000", 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003576606 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975076+0000", 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24654499199999999 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975123+0000", 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7216999999999999e-05 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.975171+0000", 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7536e-05 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:106182 5.9 5:932e2a56:::benchmark_data_smithi017_288048_object106181:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 505.50426108900001, 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26155082800000001, 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.262 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 106182 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721556+0000", 2023-09-23T15:17:26.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721554+0000", 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721601+0000", 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7094e-05 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721617+0000", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.588e-05 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721625+0000", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6120000000000001e-06 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.721968+0000", 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034316400000000002 2023-09-23T15:17:26.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722089+0000", 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121103 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.722229+0000", 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139791 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728355+0000", 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061265089999999996 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.728751+0000", 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039578699999999998 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983023+0000", 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25427224599999998 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983068+0000", 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5055000000000001e-05 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:00.983107+0000", 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8952999999999998e-05 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.268 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126389 5.4 5:26968410:::benchmark_data_smithi017_288048_object14620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 479.98488047699999, 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236534309, 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126389 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240937+0000", 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240936+0000", 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240942+0000", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.212e-06 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240953+0000", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1722999999999999e-05 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.240961+0000", 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2060000000000001e-06 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241222+0000", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026150199999999999 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241334+0000", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111864 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241432+0000", 2023-09-23T15:17:26.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8293000000000002e-05 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253717+0000", 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012284982999999999 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259156+0000", 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054391600000000002 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477373+0000", 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.218216728 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477429+0000", 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5939000000000001e-05 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477471+0000", 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2169000000000002e-05 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.274 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126392 5.4 5:22950050:::benchmark_data_smithi017_288048_object14623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 479.98460894200002, 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.236365035, 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126392 2023-09-23T15:17:26.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241209+0000", 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241208+0000", 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241213+0000", 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2070000000000003e-06 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241223+0000", 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6930000000000002e-06 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.241229+0000", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8490000000000002e-06 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249366+0000", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0081375960000000004 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249518+0000", 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015237200000000001 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.249654+0000", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013526400000000001 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.253846+0000", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041924930000000003 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.259182+0000", 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053355219999999997 2023-09-23T15:17:26.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477502+0000", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21831988299999999 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477543+0000", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1448e-05 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477574+0000", 2023-09-23T15:17:26.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0459000000000001e-05 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:126405 5.4 5:2fffe5cb:::benchmark_data_smithi017_288048_object14636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e138)", 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 479.96392951299998, 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21577380400000001, 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.281 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126405 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261888+0000", 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261886+0000", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261894+0000", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0989999999999999e-06 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261912+0000", 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8020999999999999e-05 2023-09-23T15:17:26.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261918+0000", 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6069999999999998e-06 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.261956+0000", 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7832e-05 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262070+0000", 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011383399999999999 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.262173+0000", 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103539 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.267122+0000", 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049492349999999997 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.269803+0000", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002680375 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477601+0000", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.207798129 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477633+0000", 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2582000000000001e-05 2023-09-23T15:17:26.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:09:26.477662+0000", 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8342000000000001e-05 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154970 5.4 5:2dd36d32:::benchmark_data_smithi017_288048_object43201:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 443.81790922499999, 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.201665601, 2023-09-23T15:17:26.287 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154970 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407908+0000", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407907+0000", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407914+0000", 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2860000000000002e-06 2023-09-23T15:17:26.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407934+0000", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9840999999999999e-05 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407942+0000", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1790000000000008e-06 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.407997+0000", 2023-09-23T15:17:26.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5386000000000001e-05 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408150+0000", 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015222799999999999 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.408276+0000", 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012621600000000001 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414981+0000", 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067046579999999996 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414993+0000", 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2656e-05 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609483+0000", 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19448981700000001 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609532+0000", 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8523000000000003e-05 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609574+0000", 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2299999999999998e-05 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:154974 5.4 5:2790b49c:::benchmark_data_smithi017_288048_object43205:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T15:17:26.293 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 443.81528001700002, 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19913220500000001, 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 154974 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410537+0000", 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410536+0000", 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410542+0000", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.417e-06 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410554+0000", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1855000000000001e-05 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.410559+0000", 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6640000000000003e-06 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411338+0000", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077873099999999995 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411463+0000", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000124938 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.411583+0000", 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119839 2023-09-23T15:17:26.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.414997+0000", 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034144940000000001 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.417807+0000", 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028097830000000002 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609605+0000", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.191798363 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609639+0000", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3667999999999997e-05 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:02.609670+0000", 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0574999999999998e-05 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.299 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166946 5.4 5:2ea27988:::benchmark_data_smithi017_288048_object55177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 428.13922457500001, 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28508372300000001, 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.300 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166946 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086593+0000", 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086592+0000", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086600+0000", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4349999999999997e-06 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086612+0000", 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2339e-05 2023-09-23T15:17:26.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086619+0000", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5060000000000001e-06 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087518+0000", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000898762 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087616+0000", 2023-09-23T15:17:26.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8196999999999995e-05 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087683+0000", 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6988000000000003e-05 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091939+0000", 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042566660000000001 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.095384+0000", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003444866 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371564+0000", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27617948799999997 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371623+0000", 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9219999999999999e-05 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371677+0000", 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3529999999999997e-05 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166951 5.4 5:254d5c06:::benchmark_data_smithi017_288048_object55182:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 428.13415097299998, 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28024638499999999, 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.306 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166951 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091666+0000", 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091665+0000", 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091672+0000", 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091685+0000", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3312000000000001e-05 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091692+0000", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4989999999999999e-06 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091958+0000", 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00026625599999999998 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092057+0000", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9019999999999997e-05 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092171+0000", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114345 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.098893+0000", 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067218640000000001 2023-09-23T15:17:26.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100313+0000", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014193369999999999 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371840+0000", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27152736500000002 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371876+0000", 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6191e-05 2023-09-23T15:17:26.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.371913+0000", 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6745999999999997e-05 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:175544 5.4 5:26fea254:::benchmark_data_smithi017_288048_object63775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 416.50294115600002, 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18268135499999999, 2023-09-23T15:17:26.312 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 175544 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722876+0000", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722874+0000", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722882+0000", 2023-09-23T15:17:26.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6249999999999998e-06 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722903+0000", 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1369000000000002e-05 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722911+0000", 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3560000000000004e-06 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.722956+0000", 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5698000000000002e-05 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723091+0000", 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013489599999999999 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.723244+0000", 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000152387 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.728221+0000", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049771060000000002 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.732170+0000", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039488830000000003 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905471+0000", 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17330090500000001 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905520+0000", 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9104999999999998e-05 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:29.905558+0000", 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8028999999999998e-05 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:176231 5.9 5:95f16d04:::benchmark_data_smithi017_288048_object64462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e148)", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 415.54108477699998, 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17770085299999999, 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 176231 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684733+0000", 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684732+0000", 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684735+0000", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1430000000000001e-06 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684738+0000", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5710000000000002e-06 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.684742+0000", 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1459999999999998e-06 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688383+0000", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036404229999999998 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688504+0000", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121162 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.688627+0000", 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000123273 2023-09-23T15:17:26.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691418+0000", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002790837 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.691423+0000", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2460000000000003e-06 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862340+0000", 2023-09-23T15:17:26.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17091711200000001 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862391+0000", 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0390999999999999e-05 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:30.862434+0000", 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.286e-05 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.324 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187773 5.9 5:98d8c468:::benchmark_data_smithi017_288048_object76004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.88830381700001, 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20575924800000001, 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187773 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337514+0000", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337512+0000", 2023-09-23T15:17:26.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337520+0000", 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8520000000000004e-06 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337539+0000", 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9249000000000002e-05 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337547+0000", 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3399999999999998e-06 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337601+0000", 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4172000000000002e-05 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337728+0000", 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126198 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.337851+0000", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012301799999999999 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.341076+0000", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003225136 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342051+0000", 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097561399999999997 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543167+0000", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20111568099999999 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543222+0000", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4877000000000003e-05 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.543273+0000", 2023-09-23T15:17:26.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0952000000000003e-05 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:187774 5.9 5:9a77e4c2:::benchmark_data_smithi017_288048_object76005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e151)", 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 399.883276712, 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20974965300000001, 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 187774 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342541+0000", 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342539+0000", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342545+0000", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342558+0000", 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2813000000000001e-05 2023-09-23T15:17:26.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342569+0000", 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0916e-05 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342603+0000", 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4600000000000001e-05 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342700+0000", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6618000000000002e-05 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.342775+0000", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5092000000000005e-05 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.345881+0000", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031061130000000002 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.346007+0000", 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012578200000000001 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552158+0000", 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20615127 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552231+0000", 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2809e-05 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:46.552290+0000", 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9240000000000002e-05 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197695 5.9 5:9f6a849a:::benchmark_data_smithi017_288048_object85926:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 386.19511712799999, 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.204857551, 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.337 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197695 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030700+0000", 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030699+0000", 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030707+0000", 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8480000000000001e-06 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030717+0000", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414999999999999e-05 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030724+0000", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9360000000000002e-06 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031186+0000", 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046191899999999998 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031291+0000", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010539000000000001 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.031395+0000", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010343199999999999 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.034429+0000", 2023-09-23T15:17:26.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030340480000000001 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039920+0000", 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054914489999999998 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235467+0000", 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19554693500000001 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235519+0000", 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1835000000000001e-05 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.235558+0000", 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8813000000000003e-05 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:197697 5.9 5:93334a62:::benchmark_data_smithi017_288048_object85928:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e155)", 2023-09-23T15:17:26.343 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 386.195084678, 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.216589857, 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 197697 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030733+0000", 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030732+0000", 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030735+0000", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6709999999999999e-06 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030741+0000", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0909999999999998e-06 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.030745+0000", 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2579999999999997e-06 2023-09-23T15:17:26.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.036909+0000", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061638029999999998 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037024+0000", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011513 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.037142+0000", 2023-09-23T15:17:26.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000118171 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.039953+0000", 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002810707 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.040002+0000", 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8758999999999999e-05 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247229+0000", 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20722660500000001 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247277+0000", 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8078999999999998e-05 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:00.247323+0000", 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6004000000000002e-05 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.349 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:200855 5.4 5:2c8e5777:::benchmark_data_smithi017_288048_object89086:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 382.077774326, 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37976222399999998, 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 200855 2023-09-23T15:17:26.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148043+0000", 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:17:26.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148042+0000", 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148048+0000", 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2020000000000001e-06 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148058+0000", 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5910000000000002e-06 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148063+0000", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8760000000000004e-06 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148114+0000", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1851999999999998e-05 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148231+0000", 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011667699999999999 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.148336+0000", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010487799999999999 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.151117+0000", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027812620000000001 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.152434+0000", 2023-09-23T15:17:26.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013172259999999999 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527791+0000", 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.37535686699999998 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:11:04.527805+0000", 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4005999999999999e-05 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.355 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.356 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:17:26.356 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:17:26.356 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:17:26.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:26.387 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:26.445 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:26.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:26 smithi017 ceph-mon[215646]: pgmap v652: 109 pgs: 109 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T15:17:26.448 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:26 smithi027 ceph-mon[195233]: pgmap v652: 109 pgs: 109 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 76 MiB/s wr, 1.22k op/s 2023-09-23T15:17:26.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:26.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:26.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:26.585 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:26.646 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:26.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:26.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:26.750 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:26.761 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:26.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:26.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:17:27.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:17:27.100 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:27.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:27.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:27.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 49032 49016 90.0389 59.875 0.0125198 0.0111003 2023-09-23T15:17:27.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:27.356 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:27.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:27.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:27.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:27.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:27.805 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:28.142 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 49875 49859 88.9723 52.6875 0.00293065 0.0111936 2023-09-23T15:17:28.188 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T15:17:28.444 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:28] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: pgmap v653: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2879324051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: pgmap v653: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2879324051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-09-23T15:17:28.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-09-23T15:17:28.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:28.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: pgmap v653: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2879324051' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]: dispatch 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]: dispatch 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:28.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:28.624 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:28.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:28.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:28.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:28.824 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:29.145 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 50894 50878 88.264 63.6875 0.0127612 0.0113232 2023-09-23T15:17:29.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:29.206 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-09-23T15:17:29.207 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[215646]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T15:17:29.238 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:29.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:29.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:29.345 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:29.378 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:29.435 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:29.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:29.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:29.469 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-09-23T15:17:29.470 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:29 smithi017 ceph-mon[222878]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T15:17:29.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:29.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: failed to encode map e233 with expected crc 2023-09-23T15:17:29.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 5]}]': finished 2023-09-23T15:17:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.13", "id": [1, 0]}]': finished 2023-09-23T15:17:29.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:29 smithi027 ceph-mon[195233]: osdmap e233: 8 total, 8 up, 7 in 2023-09-23T15:17:29.615 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:29.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:29.677 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:29.738 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:29.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:29.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:29.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:29.934 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:30.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:30.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:30.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:30.146 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 51880 51864 87.5423 61.625 0.0106281 0.0114164 2023-09-23T15:17:30.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:30.323 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[222878]: pgmap v655: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[222878]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[222878]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[222878]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[222878]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[215646]: pgmap v655: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[215646]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[215646]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T15:17:30.324 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[215646]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:30 smithi017 ceph-mon[215646]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.327 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:30 smithi027 ceph-mon[195233]: pgmap v655: 109 pgs: 109 active+clean; 2.8 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 48 MiB/s wr, 760 op/s 2023-09-23T15:17:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:30 smithi027 ceph-mon[195233]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:30 smithi027 ceph-mon[195233]: osdmap e234: 8 total, 8 up, 7 in 2023-09-23T15:17:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:30 smithi027 ceph-mon[195233]: failed to encode map e234 with expected crc 2023-09-23T15:17:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:30 smithi027 ceph-mon[195233]: failed to encode map e234 with expected crc 2023-09-23T15:17:31.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 52789 52773 86.729 56.8125 0.0123513 0.0115244 2023-09-23T15:17:31.239 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:17:31.549 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:31.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2043276204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[215646]: pgmap v657: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T15:17:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:31.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:31.556 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2043276204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.556 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.556 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:31 smithi017 ceph-mon[222878]: pgmap v657: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T15:17:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2043276204' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:31 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:17:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:31 smithi027 ceph-mon[195233]: pgmap v657: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s 2023-09-23T15:17:31.656 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:31.686 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:31.741 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:31.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:31.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:31.848 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:31.874 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:31.918 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:31.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:31.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:32.026 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:32.062 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:32.112 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:32.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:32.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:32.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 53644 53628 85.8759 53.4375 0.00902292 0.0116201 2023-09-23T15:17:32.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:32.244 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:32.299 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:32.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:32.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:32 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:32 smithi027 ceph-mon[195233]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:32 smithi027 ceph-mon[195233]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:32 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:17:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:32 smithi027 ceph-mon[195233]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T15:17:32.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[215646]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[215646]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[215646]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:32.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[222878]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[222878]: failed to encode map e235 with expected crc 2023-09-23T15:17:32.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:17:32.432 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:32 smithi017 ceph-mon[222878]: osdmap e235: 8 total, 8 up, 7 in 2023-09-23T15:17:32.434 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:32.535 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:32.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:32.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:32.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:32.680 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:32.734 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:32.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:32.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:32.925 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:33.123 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:33.123 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:33.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:33.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:33.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:17:33.149117+0000 min lat: 0.00111397 max lat: 0.361358 avg lat: 0.0117391 2023-09-23T15:17:33.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:17:33.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 54549 54533 85.1434 56.5625 0.0314816 0.0117391 2023-09-23T15:17:33.159 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:33.199 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:33.227 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:33.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:33.281 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:33.324 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:33.334 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:33.368 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:33.416 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:33.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:33 smithi017 ceph-mon[215646]: pgmap v659: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T15:17:33.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/249243771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:33.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:33 smithi017 ceph-mon[222878]: pgmap v659: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T15:17:33.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/249243771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:33.448 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:33.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:33.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.455 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.456 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:17:33.510 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:33.511 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:33.553 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:33 smithi027 ceph-mon[195233]: pgmap v659: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 54 MiB/s wr, 863 op/s 2023-09-23T15:17:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/249243771' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:33.641 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:34.148 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 55546 55530 84.5867 62.3125 0.00305665 0.0118086 2023-09-23T15:17:34.419 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T15:17:34.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:17:34.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:17:34.794 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:34.799 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:17:34.799 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:17:34.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:34.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:17:34.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:34.976 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:35.152 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:35.152 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 56291 56275 83.6782 46.5625 0.00968988 0.011944 2023-09-23T15:17:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:35.159 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:35.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:17:35.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:35.297 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:35.405 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:35.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:35.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:35.511 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:35.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:35.519 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:35.544 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:35 smithi027 ceph-mon[195233]: pgmap v660: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:17:35.545 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:35.610 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:35.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:35.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:35.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:35.794 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:35 smithi017 ceph-mon[222878]: pgmap v660: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:17:35.803 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:35 smithi017 ceph-mon[215646]: pgmap v660: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:17:35.817 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:36.143 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:36.150 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 57170 57154 83.0101 54.9375 0.0179349 0.0120387 2023-09-23T15:17:36.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:36.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:36.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:36.292 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:36.360 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:36.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:36.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:36.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:36.555 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:37.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:17:37.150 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 58256 58240 82.6662 67.875 0.00468052 0.0120827 2023-09-23T15:17:37.793 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:37.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:37.802 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:37.902 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:37.932 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:37.979 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:37.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:37.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:38.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:37 smithi017 ceph-mon[215646]: pgmap v661: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:17:38.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:37 smithi017 ceph-mon[222878]: pgmap v661: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:17:38.086 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:38.123 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:38.150 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 58967 58951 81.8172 44.4375 0.0109985 0.0122162 2023-09-23T15:17:38.342 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:38.348 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:38.348 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:38] "GET /metrics HTTP/1.1" 200 36281 "" "Prometheus/2.43.0" 2023-09-23T15:17:38.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:37 smithi027 ceph-mon[195233]: pgmap v661: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:17:38.428 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:38.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:38.478 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:38.528 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:38.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:38.533 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:38.558 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:38.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:38.666 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:38.723 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:38.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:38.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:38.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:38.917 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:39.029 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.029 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.138 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:39.143 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:39.150 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 59812 59796 81.1869 52.8125 0.020521 0.012309 2023-09-23T15:17:39.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:39.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.287 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.289 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.289 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:39.363 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:40.087 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:17:40.151 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 60708 60692 80.6513 56 0.00530964 0.0123927 2023-09-23T15:17:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:40 smithi027 ceph-mon[195233]: pgmap v662: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T15:17:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/905194332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[222878]: pgmap v662: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T15:17:40.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/905194332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[215646]: pgmap v662: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 58 MiB/s wr, 924 op/s 2023-09-23T15:17:40.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/905194332' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:17:40.543 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:40.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:40.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:40.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:40.695 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:40.741 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:40.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:40.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:40.845 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:40.871 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:40.916 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:40.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:41.022 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:41.083 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:41.083 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.083 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:41.083 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T15:17:41.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/895518787' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:41.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:41.085 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:17:41.097 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:41.151 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 61514 61498 80.0208 50.375 0.0119996 0.0124717 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/895518787' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: osdmap e236: 8 total, 8 up, 7 in 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: failed to encode map e236 with expected crc 2023-09-23T15:17:41.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/895518787' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:41.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:17:42.010 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:42.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:42.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:42.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:42.151 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 62469 62453 79.6056 59.6875 0.00446125 0.0125556 2023-09-23T15:17:42.152 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:42.207 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:42.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:42.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: pgmap v664: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T15:17:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:17:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T15:17:42.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:42 smithi027 ceph-mon[195233]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: pgmap v664: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T15:17:42.371 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:17:42.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T15:17:42.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[222878]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: pgmap v664: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 61 MiB/s wr, 969 op/s 2023-09-23T15:17:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:17:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T15:17:42.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:42 smithi017 ceph-mon[215646]: failed to encode map e237 with expected crc 2023-09-23T15:17:42.380 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:42.538 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:42.538 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:42.953 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:42.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:42.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:43.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:43.089 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:43.132 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:43.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:43.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:43.151 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 63152 63136 78.8676 42.6875 0.00265102 0.0126554 2023-09-23T15:17:43.236 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:43.269 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:43.317 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:43.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:43.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:43.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:43.456 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:43.511 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:43.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:43.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:43.529 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:17:43.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:17:43.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:43.649 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:43.702 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:43.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:43.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:43.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:17:43.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:43.837 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:43.888 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:43.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:43.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:43.898 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:43.954 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:43.994 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:44.008 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:44.025 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:44.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:44.077 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:44.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:44.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:44.106 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:44.139 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:44.151 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 63897 63881 78.2344 46.5625 0.0259701 0.012776 2023-09-23T15:17:44.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:44.188 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:44.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[215646]: pgmap v666: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T15:17:44.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:44.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:44.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2195203218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:44.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[222878]: pgmap v666: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T15:17:44.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:44.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:44.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2195203218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:44.211 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:44.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:44.263 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:44.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.285 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.286 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:17:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:44 smithi027 ceph-mon[195233]: pgmap v666: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 54 MiB/s wr, 860 op/s 2023-09-23T15:17:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2195203218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:44.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:44.402 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:44.465 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:44.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:44.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:44.643 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:45.153 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 15 64589 64574 77.5607 43.3125 0.0153198 0.0128874 2023-09-23T15:17:45.434 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:45.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:45.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:45.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:45.639 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:45.659 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:45.672 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:45.711 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:45.853 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:45.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:45.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:45.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:46.003 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:46.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:46.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:46.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:46.153 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 65529 65513 77.2048 58.6875 0.0205822 0.0129464 2023-09-23T15:17:46.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:46.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:46 smithi027 ceph-mon[195233]: pgmap v667: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T15:17:46.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:46 smithi017 ceph-mon[222878]: pgmap v667: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T15:17:46.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:46 smithi017 ceph-mon[215646]: pgmap v667: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 50 MiB/s wr, 796 op/s 2023-09-23T15:17:46.589 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:46.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:46.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:46.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:46.733 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:46.790 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:46.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:46.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:46.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:46.953 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:47.155 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 66449 66433 76.8376 57.5 0.0213602 0.013009 2023-09-23T15:17:47.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:17:48.157 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 67402 67386 76.5235 59.5625 0.018409 0.013063 2023-09-23T15:17:48.294 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:48.296 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:48 smithi017 ceph-mon[215646]: pgmap v668: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:17:48.296 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:48] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-09-23T15:17:48.297 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:48 smithi017 ceph-mon[222878]: pgmap v668: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:17:48.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:48.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:48.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:48.428 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:48 smithi027 ceph-mon[195233]: pgmap v668: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T15:17:48.431 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:48.477 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:48.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:48.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:48.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:48.608 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:48.664 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:48.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:48.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:48.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:48.849 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:49.101 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:49.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:49.106 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:49.155 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 68062 68046 75.8939 41.25 0.0339015 0.0131706 2023-09-23T15:17:49.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:49.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:49.293 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:49.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:49.389 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:49.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:49.432 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:49.484 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:49.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:49.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78609 6.1 6.9e872b01 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 307.98496806899999, 2023-09-23T15:17:49.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25765822199999999, 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78609 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:17:49.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587123+0000", 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587121+0000", 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587196+0000", 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4975999999999995e-05 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587203+0000", 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8000000000000001e-06 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587210+0000", 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7819999999999996e-06 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587261+0000", 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0929e-05 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587270+0000", 2023-09-23T15:17:49.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844769+0000", 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.257499058 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844781+0000", 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1984000000000001e-05 2023-09-23T15:17:49.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78617 6.1 6.bd740781 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 307.98486769300001, 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25759161800000002, 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:17:49.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78617 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587223+0000", 2023-09-23T15:17:49.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587222+0000", 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587267+0000", 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4282000000000001e-05 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587275+0000", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9270000000000005e-06 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587283+0000", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7940000000000008e-06 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587707+0000", 2023-09-23T15:17:49.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042393500000000001 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.587715+0000", 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7619999999999995e-06 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844807+0000", 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25709155299999997 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844815+0000", 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9929999999999995e-06 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78636 6.4 6.f7e668f4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 307.959871916, 2023-09-23T15:17:49.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.238510681, 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78636 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:17:49.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612219+0000", 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612217+0000", 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612283+0000", 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6523e-05 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612306+0000", 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2892999999999999e-05 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612316+0000", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6940000000000007e-06 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612373+0000", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7399999999999999e-05 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612397+0000", 2023-09-23T15:17:49.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.37e-05 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850720+0000", 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23832351900000001 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850730+0000", 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1220000000000001e-06 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.596 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78637 6.1 6.c6e1dd41 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 307.95975957299999, 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23251106799999999, 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:49.597 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78637 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612331+0000", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612374+0000", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3646000000000002e-05 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612390+0000", 2023-09-23T15:17:49.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5828000000000002e-05 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612398+0000", 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5989999999999996e-06 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612435+0000", 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6967e-05 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612441+0000", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9039999999999997e-06 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844835+0000", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23239410299999999 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.844842+0000", 2023-09-23T15:17:49.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4850000000000003e-06 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78638 6.4 6.d9b25834 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 307.95915532800001, 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.237823757, 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.602 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78638 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:17:49.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612935+0000", 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612976+0000", 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1334000000000001e-05 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.612990+0000", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4146e-05 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613002+0000", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1308000000000001e-05 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613049+0000", 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7085000000000001e-05 2023-09-23T15:17:49.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.613057+0000", 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7849999999999997e-06 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850751+0000", 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23769393899999999 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.850759+0000", 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8979999999999996e-06 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:79630 6.4 e186/184 6:239da09b:::benchmark_data_smithi017_350392_object79629:head v 186'5007, mlcod=0'0)", 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.68546056899999, 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18234605200000001, 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.607 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886630+0000", 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886629+0000", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:44.886655+0000", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6469e-05 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048539+0000", 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161884114 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.048548+0000", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3510000000000002e-06 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.051880+0000", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033326800000000002 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.051911+0000", 2023-09-23T15:17:49.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0719999999999997e-05 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.068933+0000", 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017022036000000001 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:45.068976+0000", 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3101e-05 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81754 6.e e188/184 6:7fc47cda:::benchmark_data_smithi017_350392_object81753:head v 188'5227, mlcod=0'0)", 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 300.85264456900001, 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17966953999999999, 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719446+0000", 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719487+0000", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1573000000000003e-05 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719495+0000", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4490000000000002e-06 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.719500+0000", 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0799999999999996e-06 2023-09-23T15:17:49.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.737767+0000", 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018267255 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.737792+0000", 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.493e-05 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.899075+0000", 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.161283123 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.899116+0000", 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0494999999999998e-05 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.616 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81766 6.e e188/184 6:7bc810f6:::benchmark_data_smithi017_350392_object81765:head v 188'5228, mlcod=0'0)", 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 300.85206206700002, 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.185409134, 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720029+0000", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720028+0000", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720256+0000", 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022787799999999999 2023-09-23T15:17:49.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720267+0000", 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0808999999999999e-05 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.720273+0000", 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3899999999999998e-06 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.744243+0000", 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.023969824000000001 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.744258+0000", 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5251e-05 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905371+0000", 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.16111323899999999 2023-09-23T15:17:49.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905438+0000", 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45074.0:81770 6.e e188/184 6:75c4e65d:::benchmark_data_smithi017_350392_object81769:head v 188'5229, mlcod=0'0)", 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 300.850589012, 2023-09-23T15:17:49.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.183962289, 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721502+0000", 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721501+0000", 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721576+0000", 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5160000000000005e-05 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721592+0000", 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6030999999999999e-05 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.721600+0000", 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7919999999999993e-06 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.893038+0000", 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17143856499999999 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.893063+0000", 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4309000000000001e-05 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905444+0000", 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012381647000000001 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:48.905464+0000", 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9684e-05 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:8634 7.b 7:d1abafb9:::benchmark_data_smithi017_422824_object8633:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:56.546489+0000", 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 53.025601934999997, 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.180856515, 2023-09-23T15:17:49.626 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8634 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546489+0000", 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546489+0000", 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546487+0000", 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546594+0000", 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106424 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546604+0000", 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6919999999999997e-06 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546610+0000", 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6619999999999999e-06 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546667+0000", 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7256000000000002e-05 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546805+0000", 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013792899999999999 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546998+0000", 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019225899999999999 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548617+0000", 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001619068 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.549499+0000", 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088226400000000001 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727255+0000", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17775616399999999 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727308+0000", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2641e-05 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727345+0000", 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7580999999999998e-05 2023-09-23T15:17:49.632 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:8635 7.b 7:da3bdf2f:::benchmark_data_smithi017_422824_object8634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:56.546626+0000", 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 53.025464886000002, 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18088955600000001, 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.633 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8635 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546626+0000", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546626+0000", 2023-09-23T15:17:49.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546625+0000", 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546647+0000", 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1175e-05 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546653+0000", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7920000000000004e-06 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.546659+0000", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2039999999999996e-06 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.547880+0000", 2023-09-23T15:17:49.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001221689 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548013+0000", 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013308100000000001 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548158+0000", 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144547 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.549424+0000", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001265651 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.549660+0000", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000236741 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727420+0000", 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.177759999 2023-09-23T15:17:49.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727471+0000", 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1150999999999999e-05 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727515+0000", 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4066999999999997e-05 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:8636 7.b 7:d558f7dc:::benchmark_data_smithi017_422824_object8635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e229)", 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:56.548621+0000", 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 53.023469972999997, 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17910538100000001, 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 8636 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.640 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548621+0000", 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548621+0000", 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548620+0000", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548649+0000", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8717999999999998e-05 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548660+0000", 2023-09-23T15:17:49.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1286e-05 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548667+0000", 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0849999999999999e-06 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548710+0000", 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3133999999999997e-05 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548843+0000", 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013285799999999999 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.548977+0000", 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134213 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.549859+0000", 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088212299999999996 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.550782+0000", 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00092234900000000004 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727643+0000", 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17686133900000001 2023-09-23T15:17:49.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727691+0000", 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8401e-05 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:56.727726+0000", 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4752000000000002e-05 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26351 7.b 7:d1d584bc:::benchmark_data_smithi017_422824_object26350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.816808+0000", 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.755282680999997, 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22337122500000001, 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.647 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26351 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816808+0000", 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816808+0000", 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816807+0000", 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816827+0000", 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0085000000000002e-05 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816841+0000", 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4562999999999999e-05 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816850+0000", 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6230000000000003e-06 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816998+0000", 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014793900000000001 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.817332+0000", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00033419100000000001 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.817584+0000", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000252472 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.820088+0000", 2023-09-23T15:17:49.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025036099999999999 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.821473+0000", 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013845190000000001 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040104+0000", 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21863170500000001 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040149+0000", 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4631e-05 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040179+0000", 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0437999999999999e-05 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26354 7.b 7:d8520770:::benchmark_data_smithi017_422824_object26353:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.816864+0000", 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.755226901, 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.22345636099999999, 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26354 2023-09-23T15:17:49.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816864+0000", 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816864+0000", 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816863+0000", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816885+0000", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1297999999999998e-05 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816893+0000", 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3420000000000008e-06 2023-09-23T15:17:49.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.816900+0000", 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6680000000000004e-06 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.818683+0000", 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017830319999999999 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.818825+0000", 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014194399999999999 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.818962+0000", 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013690499999999999 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.821479+0000", 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025172200000000001 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823270+0000", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001791265 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040228+0000", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21695755999999999 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040279+0000", 2023-09-23T15:17:49.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1291000000000001e-05 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.040320+0000", 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1306000000000001e-05 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26359 7.b 7:d5280ec6:::benchmark_data_smithi017_422824_object26358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.823028+0000", 2023-09-23T15:17:49.660 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.749063040999999, 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21960998400000001, 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26359 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823028+0000", 2023-09-23T15:17:49.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823028+0000", 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823026+0000", 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823048+0000", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2345999999999999e-05 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823059+0000", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0729e-05 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823066+0000", 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9360000000000002e-06 2023-09-23T15:17:49.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823303+0000", 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023663499999999999 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823442+0000", 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013959000000000001 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823578+0000", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013555800000000001 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.826011+0000", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024337220000000001 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.826046+0000", 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4609e-05 2023-09-23T15:17:49.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042548+0000", 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.216502375 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042601+0000", 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2473999999999999e-05 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042638+0000", 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6872000000000002e-05 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26360 7.b 7:d36f181d:::benchmark_data_smithi017_422824_object26359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.823072+0000", 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.749018366000001, 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21966474599999999, 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.667 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26360 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823072+0000", 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823072+0000", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823072+0000", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823092+0000", 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0361e-05 2023-09-23T15:17:49.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823100+0000", 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7989999999999994e-06 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.823106+0000", 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9399999999999999e-06 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.824776+0000", 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001670438 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.824902+0000", 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012560800000000001 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.825027+0000", 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124716 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.826362+0000", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00133541 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827339+0000", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00097670599999999993 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042672+0000", 2023-09-23T15:17:49.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.215332991 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042708+0000", 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5716999999999997e-05 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042737+0000", 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9589000000000001e-05 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.673 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26370 7.b 7:d399cc7c:::benchmark_data_smithi017_422824_object26369:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.827802+0000", 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.744288505, 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21502633700000001, 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.674 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26370 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827802+0000", 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827802+0000", 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827801+0000", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827827+0000", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5837e-05 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827839+0000", 2023-09-23T15:17:49.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2011000000000001e-05 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827845+0000", 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8699999999999997e-06 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.827927+0000", 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2389e-05 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828057+0000", 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012969199999999999 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828147+0000", 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0165000000000002e-05 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.830622+0000", 2023-09-23T15:17:49.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002474773 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.830667+0000", 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5460000000000002e-05 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042767+0000", 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21209977099999999 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042800+0000", 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3417999999999998e-05 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.042829+0000", 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8408999999999999e-05 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:26372 7.b 7:d3ba7bdb:::benchmark_data_smithi017_422824_object26371:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:07.828049+0000", 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.744042194000002, 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.21582268600000001, 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 26372 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.681 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828049+0000", 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828049+0000", 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828048+0000", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828067+0000", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9378e-05 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828076+0000", 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7169999999999997e-06 2023-09-23T15:17:49.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.828083+0000", 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9240000000000001e-06 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.829283+0000", 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001200071 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.829463+0000", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000180347 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.829621+0000", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015751900000000001 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.830775+0000", 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001154125 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:07.831917+0000", 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011422380000000001 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.043785+0000", 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.211867848 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.043835+0000", 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9583000000000003e-05 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:08.043871+0000", 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6628999999999998e-05 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:28654 7.b 7:dfd6e4ba:::benchmark_data_smithi017_422824_object28653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.687 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:09.647954+0000", 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 39.924136908000001, 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18822349199999999, 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 28654 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.647954+0000", 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.647954+0000", 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.647952+0000", 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.647981+0000", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8632e-05 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.648004+0000", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3227999999999999e-05 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.648015+0000", 2023-09-23T15:17:49.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1165000000000001e-05 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.648080+0000", 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5091000000000004e-05 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.648232+0000", 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015143099999999999 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.648379+0000", 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014728500000000001 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.649758+0000", 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00137963 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.649769+0000", 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0614e-05 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836087+0000", 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.186317809 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836137+0000", 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9906e-05 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836177+0000", 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0636000000000002e-05 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:28660 7.b 7:d02c0f37:::benchmark_data_smithi017_422824_object28659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e230)", 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:09.652892+0000", 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 39.919198729999998, 2023-09-23T15:17:49.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.18339876999999999, 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 28660 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652892+0000", 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652892+0000", 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652891+0000", 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652918+0000", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7647000000000001e-05 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652938+0000", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0183e-05 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.652949+0000", 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0302999999999999e-05 2023-09-23T15:17:49.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.653005+0000", 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5897999999999999e-05 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.653152+0000", 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014732100000000001 2023-09-23T15:17:49.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.653298+0000", 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014660300000000001 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.654520+0000", 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001221364 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:17:49.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.654699+0000", 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017959000000000001 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836216+0000", 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.181516236 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836255+0000", 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9122000000000003e-05 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:17:49.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:09.836291+0000", 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6056999999999997e-05 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:17:49.701 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:17:49.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:49.731 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:49.784 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:49.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:49.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:49.889 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:49.920 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:49.985 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:49.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:49.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:50.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:50.122 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:50.155 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 68898 68882 75.4793 52.25 0.0131859 0.0132432 2023-09-23T15:17:50.174 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:50.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:50.180 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:50.281 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:50.314 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:50.374 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:50.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:50.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:50.381 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:50 smithi027 ceph-mon[195233]: pgmap v669: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s 2023-09-23T15:17:50.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:50 smithi017 ceph-mon[222878]: pgmap v669: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s 2023-09-23T15:17:50.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:50 smithi017 ceph-mon[215646]: pgmap v669: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 32 MiB/s wr, 512 op/s 2023-09-23T15:17:50.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:50.569 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:51.162 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 69728 69712 75.0643 51.875 0.0443101 0.0133163 2023-09-23T15:17:51.218 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3803311977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.219 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.220 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3803311977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.229 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:17:51.276 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:51.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:51.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:17:51.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:51.422 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:51.475 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:51.481 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3803311977' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.481 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:17:51.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:51.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:51.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:17:51.662 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:52.162 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 70507 70491 74.6173 48.6875 0.00641479 0.0133949 2023-09-23T15:17:52.290 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: pgmap v670: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1902682914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.316 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: pgmap v670: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T15:17:52.316 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1902682914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:17:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: pgmap v670: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 51 MiB/s wr, 821 op/s 2023-09-23T15:17:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: failed to encode map e238 with expected crc 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1902682914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:17:52.650 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:17:52.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:52.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:17:52.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:52.783 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:52.832 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:52.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:52.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:52.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:52.963 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:53.015 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:53.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:53.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:53.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:53.162 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:17:53.163416+0000 min lat: 0.00111397 max lat: 0.361358 avg lat: 0.013477 2023-09-23T15:17:53.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:17:53.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 71270 71254 74.1678 47.6875 0.0137045 0.013477 2023-09-23T15:17:53.200 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:53.367 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:53.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:17:53.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:53.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:53.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:17:53.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:53 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:53.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:53.559 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:17:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: failed to encode map e239 with expected crc 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:17:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:53 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:17:53.730 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:53.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:53.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:53.808 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:53.813 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:53.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:53.865 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:53.917 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:53.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:54.022 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:17:54.032 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:17:54.053 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:54.074 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:17:54.107 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:17:54.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:17:54.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:54.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:17:54.149 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:17:54.162 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 72086 72070 73.7882 51 0.0154836 0.0135457 2023-09-23T15:17:54.185 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:17:54.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:54.246 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:17:54.291 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:17:54.330 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:54.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:17:54.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:17:54.393 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:54.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:54.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:54.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:54 smithi017 ceph-mon[215646]: pgmap v673: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T15:17:54.501 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/557436763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:54.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:54 smithi017 ceph-mon[222878]: pgmap v673: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T15:17:54.501 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/557436763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:54.523 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:54 smithi027 ceph-mon[195233]: pgmap v673: 109 pgs: 109 active+clean; 4.2 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 803 op/s 2023-09-23T15:17:54.523 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/557436763' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:17:54.527 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:54.588 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:54.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:54.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:54.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:54.777 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:55.162 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 72908 72892 73.4268 51.375 0.00338824 0.0136065 2023-09-23T15:17:55.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:17:55.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:55 smithi017 ceph-mon[215646]: pgmap v674: 109 pgs: 109 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T15:17:55.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:55 smithi017 ceph-mon[222878]: pgmap v674: 109 pgs: 109 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T15:17:55.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:55 smithi027 ceph-mon[195233]: pgmap v674: 109 pgs: 109 active+clean; 4.3 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 48 MiB/s wr, 773 op/s 2023-09-23T15:17:55.669 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:55.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:55.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:55.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:55.808 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:55.864 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:55.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:55.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:55.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:56.006 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:56.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:56.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:56.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:56.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 73659 73643 73.0066 46.9375 0.0121179 0.0136922 2023-09-23T15:17:56.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:56.263 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:56.384 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:56.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:56.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:56.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:17:56.536 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:56.598 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:17:56.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:56.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:17:56.704 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:17:56.771 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:57.062 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:17:57.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:57.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:17:57.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 74449 74433 72.6375 49.375 0.0119309 0.0137608 2023-09-23T15:17:57.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:57.244 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:57.393 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:57.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:57.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:57.452 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:17:57.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:17:57.581 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:57.625 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:17:57.626 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:17:58.032 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:17:58.092 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:17:58.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:58.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:17:58.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 75213 75197 72.2547 47.75 0.00410384 0.0138292 2023-09-23T15:17:58.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:17:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:17:58] "GET /metrics HTTP/1.1" 200 36286 "" "Prometheus/2.43.0" 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[215646]: pgmap v675: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2218509683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[222878]: pgmap v675: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2218509683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.204 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:58.277 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:58 smithi027 ceph-mon[195233]: pgmap v675: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 62 MiB/s wr, 992 op/s 2023-09-23T15:17:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2218509683' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:17:58.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:17:58.969 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:58.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:58.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:17:59.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:59.140 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:59.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 75995 75979 71.9006 48.875 0.00665988 0.0138939 2023-09-23T15:17:59.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[215646]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[215646]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[215646]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[222878]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[222878]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[222878]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:17:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:59.194 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:59.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:59.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:17:59.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:17:59.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:59 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:17:59.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:59 smithi027 ceph-mon[195233]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T15:17:59.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:59 smithi027 ceph-mon[195233]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:59 smithi027 ceph-mon[195233]: failed to encode map e240 with expected crc 2023-09-23T15:17:59.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:17:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:17:59.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:59.386 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:17:59.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:59.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:17:59.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:17:59.527 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:59.590 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:17:59.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:59.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:17:59.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:17:59.715 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:17:59.782 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:17:59.911 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:17:59.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:17:59.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:00.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:00.042 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:00.095 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:00.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:00.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:00.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 76705 76689 71.49 44.375 0.00998059 0.0139841 2023-09-23T15:18:00.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:00 smithi017 ceph-mon[215646]: pgmap v677: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s 2023-09-23T15:18:00.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:00 smithi017 ceph-mon[222878]: pgmap v677: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s 2023-09-23T15:18:00.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:00.230 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:00.290 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:00.297 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:00 smithi027 ceph-mon[195233]: pgmap v677: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 37 MiB/s wr, 589 op/s 2023-09-23T15:18:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:00.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:00.399 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:00.433 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:00.487 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:00.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:00.594 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:00.624 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:00.634 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:00.665 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:00.676 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:00.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:00.707 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:00.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:00.920 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:01.089 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:01.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:01.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:01.163 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 77504 77488 71.1731 49.9375 0.00924386 0.0140443 2023-09-23T15:18:01.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:01.308 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:02.167 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 78299 78283 70.8582 49.6875 0.0131562 0.0141079 2023-09-23T15:18:02.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:02 smithi017 ceph-mon[222878]: pgmap v678: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T15:18:02.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:02 smithi017 ceph-mon[215646]: pgmap v678: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T15:18:02.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:02 smithi027 ceph-mon[195233]: pgmap v678: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 58 MiB/s wr, 931 op/s 2023-09-23T15:18:02.692 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:02.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:02.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:02.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:02.833 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:02.885 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:02.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:02.890 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:02.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:03.097 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/283404137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.105 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:18:03.167 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 79114 79098 70.5737 50.9375 0.0170848 0.014163 2023-09-23T15:18:03.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/283404137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.404 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/283404137' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.405 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:03.631 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:03.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:03.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:03.734 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:03.813 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: pgmap v679: 109 pgs: 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: failed to encode map e241 with expected crc 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:04.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[222878]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T15:18:04.170 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 79985 79969 70.3437 54.4375 0.00657843 0.0142098 2023-09-23T15:18:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: pgmap v679: 109 pgs: 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T15:18:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:04.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:18:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: failed to encode map e241 with expected crc 2023-09-23T15:18:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:04.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:04 smithi027 ceph-mon[195233]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: pgmap v679: 109 pgs: 109 active+clean; 4.7 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 49 MiB/s wr, 791 op/s 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: failed to encode map e241 with expected crc 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:04.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:04 smithi017 ceph-mon[215646]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T15:18:04.750 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:04.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:04.766 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:04.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:04.975 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:05.170 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 80854 80838 70.1209 54.3125 0.0260136 0.0142537 2023-09-23T15:18:05.223 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:05.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:05.234 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:05.241 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:18:05.241 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:18:05.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:05.364 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:05.420 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:05.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:05.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:05.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:05.589 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:06.090 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:06.090 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:06.091 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:06.111 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:06.151 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:06.170 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 81626 81610 69.8215 48.25 0.0174775 0.0143167 2023-09-23T15:18:06.190 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:06.225 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:06.260 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.287 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.288 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:18:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:06 smithi027 ceph-mon[195233]: pgmap v681: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T15:18:06.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3871466915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:06.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:06 smithi017 ceph-mon[222878]: pgmap v681: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T15:18:06.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3871466915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:06.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:06 smithi017 ceph-mon[215646]: pgmap v681: 109 pgs: 109 active+clean; 4.8 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 46 MiB/s wr, 738 op/s 2023-09-23T15:18:06.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3871466915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:06.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:18:07.068 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:07.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:07.110 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:18:07.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:18:07.170 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 82404 82388 69.5351 48.625 0.0269369 0.0143753 2023-09-23T15:18:07.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:07.214 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:07.273 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:07.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2670283129' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:07 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:07.408 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:07.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2670283129' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:07 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.415 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2670283129' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.416 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:07 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:18:07.459 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:07.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:07.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:07.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:07.628 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:08.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:08] "GET /metrics HTTP/1.1" 200 36283 "" "Prometheus/2.43.0" 2023-09-23T15:18:08.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:18:08.091 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:08.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:08.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:08.173 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 83210 83194 69.2772 50.375 0.0186707 0.0144296 2023-09-23T15:18:08.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:08.229 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:08.236 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:08.291 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:08.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:08.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:08.348 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: pgmap v682: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-09-23T15:18:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:18:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T15:18:08.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:08 smithi027 ceph-mon[195233]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: pgmap v682: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[215646]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: pgmap v682: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 57 MiB/s wr, 905 op/s 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:08 smithi017 ceph-mon[222878]: failed to encode map e242 with expected crc 2023-09-23T15:18:08.500 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:08.716 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:08.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:08.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:08.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:08.915 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:09.173 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 83948 83932 68.9727 46.125 0.0167612 0.0144931 2023-09-23T15:18:09.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:18:10.173 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 84674 84658 68.6664 45.375 0.010106 0.0145566 2023-09-23T15:18:10.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:10 smithi017 ceph-mon[215646]: pgmap v684: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s 2023-09-23T15:18:10.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:10 smithi017 ceph-mon[222878]: pgmap v684: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s 2023-09-23T15:18:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:10 smithi027 ceph-mon[195233]: pgmap v684: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 34 MiB/s wr, 539 op/s 2023-09-23T15:18:10.537 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:10.547 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:10.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:10.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:10.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:10.746 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:10.860 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:10.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:10.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:11.059 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:11.178 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 85400 85384 68.3636 45.375 0.0248404 0.014621 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.276 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106188 5.3 5:cc2b8235:::benchmark_data_smithi017_288048_object106187:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 550.54424467299998, 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25023266199999999, 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106188 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728962+0000", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728961+0000", 2023-09-23T15:18:11.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728985+0000", 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4165999999999999e-05 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729002+0000", 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6997e-05 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729010+0000", 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2709999999999992e-06 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729326+0000", 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00031631699999999999 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729469+0000", 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014238 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.729612+0000", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014267699999999999 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732809+0000", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031979479999999999 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732951+0000", 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014197299999999999 2023-09-23T15:18:11.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979128+0000", 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.246176479 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979168+0000", 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0108000000000003e-05 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979195+0000", 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6879000000000001e-05 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106190 5.5 5:a890e663:::benchmark_data_smithi017_288048_object106189:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 550.54023375600002, 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24679059, 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.284 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106190 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732973+0000", 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732972+0000", 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.732993+0000", 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1047e-05 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733006+0000", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2432999999999999e-05 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733011+0000", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733051+0000", 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9368e-05 2023-09-23T15:18:11.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733147+0000", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6447999999999995e-05 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733242+0000", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5328000000000006e-05 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.735970+0000", 2023-09-23T15:18:11.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027275289999999998 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979622+0000", 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243651849 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979677+0000", 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5585999999999999e-05 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.979764+0000", 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6321000000000002e-05 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166954 5.3 5:c22244f8:::benchmark_data_smithi017_288048_object55185:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.18117809, 2023-09-23T15:18:11.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.279674321, 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166954 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092029+0000", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092027+0000", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092034+0000", 2023-09-23T15:18:11.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8050000000000001e-06 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092053+0000", 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9315000000000001e-05 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092061+0000", 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9319999999999996e-06 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092120+0000", 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8522e-05 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092275+0000", 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015473200000000001 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:18:11.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092403+0000", 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128739 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096986+0000", 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045829709999999999 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.099294+0000", 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023073640000000001 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371598+0000", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27230407499999998 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371657+0000", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9521000000000002e-05 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371703+0000", 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5927000000000003e-05 2023-09-23T15:18:11.296 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166955 5.5 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 473.18071680600002, 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41495601799999998, 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:11.297 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166955 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092490+0000", 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092489+0000", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092494+0000", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2989999999999998e-06 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092512+0000", 2023-09-23T15:18:11.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7952999999999999e-05 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092521+0000", 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1449999999999997e-06 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092566+0000", 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5461999999999999e-05 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092677+0000", 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011045200000000001 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4", 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092782+0000", 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096249+0000", 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00346767 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507308+0000", 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.41105842100000001 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507348+0000", 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0608999999999997e-05 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.507446+0000", 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7705000000000001e-05 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147635 6.9 6:99198402:::benchmark_data_smithi017_350392_object47244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.917946+0000", 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 205.35526086799999, 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27232970699999998, 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147635 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917946+0000", 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917946+0000", 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917945+0000", 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917950+0000", 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1399999999999999e-06 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917961+0000", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0902e-05 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.917969+0000", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2649999999999999e-06 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.920941+0000", 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029726119999999999 2023-09-23T15:18:11.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.921070+0000", 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012885199999999999 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.921185+0000", 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114806 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926402+0000", 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052168240000000001 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930403+0000", 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040014409999999997 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.190182+0000", 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.259778442 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.190237+0000", 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5198999999999998e-05 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.190276+0000", 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9051999999999998e-05 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.309 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147641 6.0 6:096cbb93:::benchmark_data_smithi017_350392_object47250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.930497+0000", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 205.34270944799999, 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27495693100000002, 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147641 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930497+0000", 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930497+0000", 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930496+0000", 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930502+0000", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0990000000000004e-06 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930511+0000", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7620000000000001e-06 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930516+0000", 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2390000000000002e-06 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930561+0000", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5389000000000002e-05 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930696+0000", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134938 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.930822+0000", 2023-09-23T15:18:11.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125866 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.936890+0000", 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060678449999999997 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937162+0000", 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027174300000000002 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205352+0000", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26819045800000002 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205415+0000", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2883000000000001e-05 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205454+0000", 2023-09-23T15:18:11.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9356000000000001e-05 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147644 6.9 6:969a51cc:::benchmark_data_smithi017_350392_object47253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.933427+0000", 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 205.33977955399999, 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266190131, 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:11.316 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147644 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933427+0000", 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933427+0000", 2023-09-23T15:18:11.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933426+0000", 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933433+0000", 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4440000000000002e-06 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933445+0000", 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2017e-05 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933453+0000", 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8299999999999996e-06 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933499+0000", 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5639000000000001e-05 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933620+0000", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012129 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.933734+0000", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011380299999999999 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.936777+0000", 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003043661 2023-09-23T15:18:11.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937206+0000", 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042879300000000001 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.199531+0000", 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26232444799999999 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.199580+0000", 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.952e-05 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.199618+0000", 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7345999999999998e-05 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:3401 7.6 7:643df5bf:::benchmark_data_smithi017_422824_object3400:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e228)", 2023-09-23T15:18:11.322 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:16:54.427489+0000", 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 76.845717785000005, 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.241966869, 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 3401 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427489+0000", 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427489+0000", 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427487+0000", 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:18:11.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427540+0000", 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2679000000000002e-05 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427553+0000", 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3553999999999999e-05 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427562+0000", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5590000000000006e-06 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427717+0000", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015550300000000001 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.427888+0000", 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017085000000000001 2023-09-23T15:18:11.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.428043+0000", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015544800000000001 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.429427+0000", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013835329999999999 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.431474+0000", 2023-09-23T15:18:11.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020473430000000001 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.669311+0000", 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23783711799999999 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.669401+0000", 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9915999999999998e-05 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:16:54.669456+0000", 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4623999999999997e-05 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64524 7.4 7:2625c760:::benchmark_data_smithi017_422824_object64523:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.842074+0000", 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.431133025000001, 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24518707000000001, 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.329 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64524 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842074+0000", 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842074+0000", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842072+0000", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842097+0000", 2023-09-23T15:18:11.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4879e-05 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842109+0000", 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2843000000000001e-05 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842118+0000", 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9190000000000005e-06 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842182+0000", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4084999999999994e-05 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842327+0000", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014456399999999999 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.842494+0000", 2023-09-23T15:18:11.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166508 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847457+0000", 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049637090000000002 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.849940+0000", 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024827920000000002 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.087190+0000", 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23725025999999999 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.087232+0000", 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2117999999999999e-05 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.087261+0000", 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8563000000000001e-05 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64538 7.6 7:6082800e:::benchmark_data_smithi017_422824_object64537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851313+0000", 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.421893808, 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25428551700000002, 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.336 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64538 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851313+0000", 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851313+0000", 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851313+0000", 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851333+0000", 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0775999999999999e-05 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851341+0000", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2289999999999997e-06 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851347+0000", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.331e-06 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851397+0000", 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0033e-05 2023-09-23T15:18:11.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851522+0000", 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124564 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851666+0000", 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144182 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.854161+0000", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002495298 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.854177+0000", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5682999999999999e-05 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.105512+0000", 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25133506799999999 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.105568+0000", 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6416999999999999e-05 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.105599+0000", 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0352e-05 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.342 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:84281 7.9 7:973789c2:::benchmark_data_smithi017_422824_object84280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.604268+0000", 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.66893866, 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.131247793, 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84281 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604268+0000", 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604268+0000", 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604268+0000", 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604292+0000", 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4056000000000001e-05 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604300+0000", 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.028e-06 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.604306+0000", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.925e-06 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.613819+0000", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095131 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.613965+0000", 2023-09-23T15:18:11.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014620099999999999 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614133+0000", 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168314 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732190+0000", 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11805711100000001 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732223+0000", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3046000000000002e-05 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.735419+0000", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003195608 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.735472+0000", 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2769000000000002e-05 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.735516+0000", 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4347000000000001e-05 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:84277 7.e e242/223 7:7d286ca0:::benchmark_data_smithi017_422824_object84276:head v 242'5274, mlcod=242'5271)", 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.605493+0000", 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.667713639, 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12352305199999999, 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.349 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605493+0000", 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605493+0000", 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605493+0000", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605518+0000", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5471000000000001e-05 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605526+0000", 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5900000000000002e-06 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.605532+0000", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8860000000000001e-06 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.608645+0000", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003112966 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.608665+0000", 2023-09-23T15:18:11.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9871999999999999e-05 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728977+0000", 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12031207300000001 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729016+0000", 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9493999999999998e-05 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:84285 7.e e242/223 7:76872f06:::benchmark_data_smithi017_422824_object84284:head v 242'5275, mlcod=242'5271)", 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.611423+0000", 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.661783818, 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.117629173, 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611423+0000", 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611423+0000", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611422+0000", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611445+0000", 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2929e-05 2023-09-23T15:18:11.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611456+0000", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1058e-05 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611463+0000", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4189999999999999e-06 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611525+0000", 2023-09-23T15:18:11.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1840000000000004e-05 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.611544+0000", 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9419000000000002e-05 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729021+0000", 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.117476816 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729052+0000", 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1436999999999997e-05 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:84292 7.6 7:6db12391:::benchmark_data_smithi017_422824_object84291:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.614594+0000", 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.658613034, 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.118221442, 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.358 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84292 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614594+0000", 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614594+0000", 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614592+0000", 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614620+0000", 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8053999999999999e-05 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614638+0000", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7660999999999999e-05 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614647+0000", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9320000000000002e-06 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614708+0000", 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0479999999999997e-05 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614836+0000", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012820600000000001 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.615018+0000", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018227799999999999 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732340+0000", 2023-09-23T15:18:11.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.117321587 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732397+0000", 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7472000000000001e-05 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732725+0000", 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032780400000000001 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732777+0000", 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2062999999999999e-05 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732815+0000", 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8408000000000002e-05 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.364 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:84293 7.6 7:6d81f6a1:::benchmark_data_smithi017_422824_object84292:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.614657+0000", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.658550223, 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.12855338999999999, 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84293 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614657+0000", 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614657+0000", 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614656+0000", 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614676+0000", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0262e-05 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614693+0000", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6841e-05 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.614699+0000", 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0410000000000003e-06 2023-09-23T15:18:11.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729084+0000", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11438443299999999 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729230+0000", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000146718 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729366+0000", 2023-09-23T15:18:11.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013595299999999999 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.736184+0000", 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068176230000000001 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.736195+0000", 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0541e-05 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.743133+0000", 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069386969999999997 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.743176+0000", 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2369e-05 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.743210+0000", 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4462000000000003e-05 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.370 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:84289 7.7 e242/223 7:ee571661:::benchmark_data_smithi017_422824_object84288:head v 242'5155, mlcod=242'5152)", 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.616419+0000", 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.6567879730000001, 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112648102, 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616419+0000", 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616419+0000", 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616418+0000", 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616460+0000", 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1628999999999997e-05 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616472+0000", 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2065999999999999e-05 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616480+0000", 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.072e-06 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616622+0000", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014190599999999999 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616645+0000", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3030000000000001e-05 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729026+0000", 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112380997 2023-09-23T15:18:11.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729067+0000", 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1221999999999999e-05 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:84294 7.d e242/223 7:b9ce2ade:::benchmark_data_smithi017_422824_object84293:head v 242'5385, mlcod=242'5383)", 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.616493+0000", 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.656713525, 2023-09-23T15:18:11.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.124857222, 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616493+0000", 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616493+0000", 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616493+0000", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616525+0000", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2137000000000001e-05 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616531+0000", 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1739999999999997e-06 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.616536+0000", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6609999999999999e-06 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728964+0000", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.112428023 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728991+0000", 2023-09-23T15:18:11.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6543e-05 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.741306+0000", 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01231485 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.741351+0000", 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5127999999999999e-05 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:84281 7.9 e242/223 ondisk, result = 0)", 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.618887+0000", 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.6543198669999999, 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11330612, 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.618887+0000", 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.618887+0000", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.618886+0000", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.618893+0000", 2023-09-23T15:18:11.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4230000000000002e-06 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728945+0000", 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.110051881 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728956+0000", 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1848000000000001e-05 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732177+0000", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032204270000000001 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732184+0000", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4359999999999996e-06 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732193+0000", 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9339999999999995e-06 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.383 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:84292 7.6 e242/223 ondisk, result = 0)", 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.619256+0000", 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.6539513700000001, 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.11314529299999999, 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.619256+0000", 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.619256+0000", 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.619255+0000", 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.619264+0000", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7150000000000004e-06 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728937+0000", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.109673176 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728951+0000", 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3808999999999999e-05 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732368+0000", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034177180000000001 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732390+0000", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1441e-05 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732401+0000", 2023-09-23T15:18:11.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1004e-05 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:84299 7.9 7:9f74d759:::benchmark_data_smithi017_422824_object84298:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e242)", 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:18:09.620774+0000", 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 1.6524325389999999, 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.115813188, 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:11.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84299 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.620774+0000", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.620774+0000", 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:11.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.620773+0000", 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.620800+0000", 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6903e-05 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:11.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728943+0000", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.10814325 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.728955+0000", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2646000000000001e-05 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729011+0000", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5322000000000003e-05 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:11.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729206+0000", 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019516299999999999 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.729366+0000", 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016034299999999999 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.732260+0000", 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002894074 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.735368+0000", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003107706 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.736512+0000", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011440510000000001 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.736554+0000", 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2067999999999998e-05 2023-09-23T15:18:11.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:18:09.736588+0000", 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3374999999999998e-05 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:11.394 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:18:11.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:11.395 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:11.395 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:11.395 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:11.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:11.588 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:12.110 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:18:12.179 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 86105 86089 68.0561 44.0625 0.00779286 0.0146751 2023-09-23T15:18:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:12 smithi017 ceph-mon[215646]: pgmap v685: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T15:18:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:12.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:12 smithi017 ceph-mon[222878]: pgmap v685: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T15:18:12.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:12.543 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:12.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:12.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:12 smithi027 ceph-mon[195233]: pgmap v685: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 60 MiB/s wr, 956 op/s 2023-09-23T15:18:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:12.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:12.684 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:12.746 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:12.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:12.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:12.879 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:12.927 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:12.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:12.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:13.034 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:13.071 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:13.131 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:13.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:13.139 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:13.180 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:18:13.181048+0000 min lat: 0.00111397 max lat: 0.361358 avg lat: 0.0147359 2023-09-23T15:18:13.180 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:18:13.180 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 86916 86900 67.8379 50.6875 0.0228749 0.0147359 2023-09-23T15:18:13.240 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.269 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:13 smithi027 ceph-mon[195233]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.271 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:13.325 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:13.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:13.331 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:13.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:18:13.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:13.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.407 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[215646]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/114697632' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:13 smithi017 ceph-mon[222878]: failed to encode map e243 with expected crc 2023-09-23T15:18:13.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:13.517 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:14.189 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 87608 87592 67.5276 43.25 0.0315207 0.0148017 2023-09-23T15:18:14.240 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:14.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:14.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:14.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:14.385 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:14 smithi027 ceph-mon[195233]: pgmap v687: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T15:18:14.385 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:14.388 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:14.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:14 smithi017 ceph-mon[215646]: pgmap v687: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T15:18:14.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:14.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:14 smithi017 ceph-mon[222878]: pgmap v687: 109 pgs: 109 active+clean; 5.1 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 48 MiB/s wr, 764 op/s 2023-09-23T15:18:14.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:14.451 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:14.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:14.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:14.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:14.594 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:14.652 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:14.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:14.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:14.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:14.849 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:15.029 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:15.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:15.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:15.099 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:15.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:15.166 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:15.192 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 88403 88387 67.3086 49.6875 0.0169636 0.0148512 2023-09-23T15:18:15.219 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:15.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:15.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:15.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:15.433 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:15.640 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:15.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:15.660 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:15.761 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:15.791 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:15.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:15.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:15.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:15.907 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:15.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:15.956 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:16.001 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:16.020 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:16.052 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:16.086 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:16.132 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:16.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:16.154 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:16.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[215646]: pgmap v688: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T15:18:16.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2513150752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:16.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/266105477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.161 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.171 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:16.171 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:18:16.190 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 89061 89045 66.9934 41.125 0.00489104 0.0149186 2023-09-23T15:18:16.202 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:16.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.204 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:18:16.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:16.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:18:16.364 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:16.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[222878]: pgmap v688: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T15:18:16.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2513150752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:16.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/266105477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:16 smithi027 ceph-mon[195233]: pgmap v688: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 716 op/s 2023-09-23T15:18:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2513150752' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/266105477' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:18:16.921 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:16.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:17.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:17.077 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:17.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:18:17.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[215646]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T15:18:17.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[215646]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.182 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[215646]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.195 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 89730 89714 66.6899 41.8125 0.0128014 0.0149892 2023-09-23T15:18:17.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:18:17.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[222878]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T15:18:17.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[222878]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:17 smithi017 ceph-mon[222878]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:18:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:17 smithi027 ceph-mon[195233]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T15:18:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:17 smithi027 ceph-mon[195233]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:17 smithi027 ceph-mon[195233]: failed to encode map e244 with expected crc 2023-09-23T15:18:17.629 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:18:17.680 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:17.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:17.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:17.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:17.819 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:17.881 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:17.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:17.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:17.990 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:18.021 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:18.081 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:18.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:18.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:18.199 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 90546 90530 66.5045 51 0.013932 0.015031 2023-09-23T15:18:18.275 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:18.295 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:18] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-09-23T15:18:18.295 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:18 smithi017 ceph-mon[215646]: pgmap v690: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T15:18:18.296 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:18 smithi017 ceph-mon[222878]: pgmap v690: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T15:18:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:18 smithi027 ceph-mon[195233]: pgmap v690: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 910 op/s 2023-09-23T15:18:18.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:18.621 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:18.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:18.642 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:18.697 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:18:18.697 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:18:18.740 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:18.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:18.772 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:18.826 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:18.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:18.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:18.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:19.030 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:19.199 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 91232 91216 66.2282 42.875 0.0905659 0.0150874 2023-09-23T15:18:19.711 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:19.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:19.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:19.789 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:18:19.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:19.961 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:20.199 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 91914 91898 65.9571 42.625 0.0288805 0.015155 2023-09-23T15:18:20.252 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:20.256 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:20 smithi017 ceph-mon[215646]: pgmap v691: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 31 MiB/s wr, 493 op/s 2023-09-23T15:18:20.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:20.259 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:20 smithi017 ceph-mon[222878]: pgmap v691: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 31 MiB/s wr, 493 op/s 2023-09-23T15:18:20.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:20.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:18:20.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:20.499 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:20.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:20 smithi027 ceph-mon[195233]: pgmap v691: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 31 MiB/s wr, 493 op/s 2023-09-23T15:18:21.199 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 92633 92617 65.7183 44.9375 0.00761867 0.0152073 2023-09-23T15:18:21.680 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:21.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:21.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:21.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:21.824 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:21.835 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:21.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:21.878 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:21.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:21.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:22.013 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:22.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:22.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:22.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:22.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:22.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 93253 93237 65.4111 38.75 0.0206357 0.0152825 2023-09-23T15:18:22.279 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:22 smithi017 ceph-mon[215646]: pgmap v692: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T15:18:22.281 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:22 smithi017 ceph-mon[222878]: pgmap v692: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T15:18:22.288 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:22.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:22 smithi027 ceph-mon[195233]: pgmap v692: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 53 MiB/s wr, 843 op/s 2023-09-23T15:18:22.533 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:22.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:22.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:22.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:22.676 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:22.729 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:22.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:22.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:22.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:22.867 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:22.920 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:22.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:22.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:23.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:23.055 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:23.109 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:23.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:23.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0144 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 93867 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 65.175 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 26.364 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 168.188 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 38.75 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1042 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 421.823 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 2691 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 620 2023-09-23T15:18:23.206 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0153394 2023-09-23T15:18:23.207 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.017764 2023-09-23T15:18:23.207 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.361358 2023-09-23T15:18:23.207 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00111397 2023-09-23T15:18:23.211 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:18:23.216 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:23.246 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:23.270 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:18:23.278 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:18:23.302 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:23.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:23.409 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:23.457 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3224807077' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:23 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:23.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:23.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3224807077' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:23.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3224807077' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:23.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:23 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: pgmap v693: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: failed to encode map e245 with expected crc 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T15:18:24.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[222878]: failed to encode map e245 with expected crc 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: pgmap v693: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: failed to encode map e245 with expected crc 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T15:18:24.390 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:24 smithi017 ceph-mon[215646]: failed to encode map e245 with expected crc 2023-09-23T15:18:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: pgmap v693: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 46 MiB/s wr, 729 op/s 2023-09-23T15:18:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: failed to encode map e245 with expected crc 2023-09-23T15:18:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1103293090' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:18:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T15:18:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:24 smithi027 ceph-mon[195233]: failed to encode map e245 with expected crc 2023-09-23T15:18:25.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:18:25.419 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:25.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:25.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:25.531 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:25.642 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:25.839 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:25.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:25.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:25.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:26.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:26.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:26.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:26.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:26.233 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:26.281 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:26.331 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:26.372 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:26.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:26.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:26.420 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:26.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:26 smithi017 ceph-mon[215646]: pgmap v695: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 122 B/s rd, 37 MiB/s wr, 631 op/s 2023-09-23T15:18:26.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4045803199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:26.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:26 smithi017 ceph-mon[222878]: pgmap v695: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 122 B/s rd, 37 MiB/s wr, 631 op/s 2023-09-23T15:18:26.454 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4045803199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:26.462 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:26.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:26.499 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:26.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:26 smithi027 ceph-mon[195233]: pgmap v695: 109 pgs: 109 active+clean; 5.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 122 B/s rd, 37 MiB/s wr, 631 op/s 2023-09-23T15:18:26.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4045803199' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:26.509 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:26.524 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:18:26.525 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:18:26.525 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:18:26.525 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:18:26.525 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:18:26.534 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:26.542 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:26.562 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:26.569 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:26.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.582 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:18:26.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:26.747 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:27.062 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:27.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:27.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:27.227 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:27.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:18:27.808 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:27.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:27.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:27.916 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:28.016 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:28.086 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:28 smithi017 ceph-mon[215646]: pgmap v696: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 695 op/s 2023-09-23T15:18:28.087 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:28 smithi017 ceph-mon[222878]: pgmap v696: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 695 op/s 2023-09-23T15:18:28.087 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:28] "GET /metrics HTTP/1.1" 200 36288 "" "Prometheus/2.43.0" 2023-09-23T15:18:28.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:18:28.281 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:28.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:28.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:28 smithi027 ceph-mon[195233]: pgmap v696: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 38 MiB/s wr, 695 op/s 2023-09-23T15:18:28.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:28.520 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:28.752 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:28.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:28.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:28.863 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:28.895 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:28.938 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:28.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:28.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:29.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:29.072 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:29.136 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:29.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:29.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:29.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:29.278 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:29.281 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:29.341 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:29.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:29.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:29.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:29.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:29.452 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:29.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:29.485 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:29.536 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:29.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:29.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:29.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:29.673 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:29.766 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:30.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:18:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:30 smithi027 ceph-mon[195233]: pgmap v697: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 690 op/s 2023-09-23T15:18:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:30 smithi017 ceph-mon[215646]: pgmap v697: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 690 op/s 2023-09-23T15:18:30.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:30 smithi017 ceph-mon[222878]: pgmap v697: 109 pgs: 109 active+clean; 5.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 37 MiB/s wr, 690 op/s 2023-09-23T15:18:30.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:30.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:30.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:30.858 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:31.087 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:18:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:18:31.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:18:31.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:18:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:32 smithi027 ceph-mon[195233]: pgmap v698: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s 2023-09-23T15:18:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:32 smithi027 ceph-mon[195233]: failed to encode map e246 with expected crc 2023-09-23T15:18:32.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:18:32.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:32 smithi027 ceph-mon[195233]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T15:18:32.432 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:32.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:32.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[215646]: pgmap v698: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[215646]: failed to encode map e246 with expected crc 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[215646]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[222878]: pgmap v698: 109 pgs: 109 active+clean; 5.5 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 102 B/s rd, 16 MiB/s wr, 672 op/s 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[222878]: failed to encode map e246 with expected crc 2023-09-23T15:18:32.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/257084091' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:18:32.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:32 smithi017 ceph-mon[222878]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T15:18:32.521 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:18:32.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:18:32.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:32.572 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:32.631 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:32.637 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:32.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:32.770 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:32.824 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:32.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:32.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:32.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:33.017 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:33.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:33 smithi017 ceph-mon[215646]: failed to encode map e246 with expected crc 2023-09-23T15:18:33.105 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:33.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:33.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:33.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:33.286 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:33.340 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:33.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:33.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:33.344 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:33 smithi027 ceph-mon[195233]: failed to encode map e246 with expected crc 2023-09-23T15:18:33.383 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:33 smithi017 ceph-mon[222878]: failed to encode map e246 with expected crc 2023-09-23T15:18:33.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:33.542 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:33.687 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:33.708 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:33.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.742 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106171 5.a 5:591ee17c:::benchmark_data_smithi017_288048_object106170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.711513+0000", 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 573.02744543599999, 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27237841299999999, 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106171 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711513+0000", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711513+0000", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711512+0000", 2023-09-23T15:18:33.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711536+0000", 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3844000000000001e-05 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711556+0000", 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.031e-05 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.711564+0000", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1049999999999995e-06 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.714105+0000", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002540853 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.714255+0000", 2023-09-23T15:18:33.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001493 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.714420+0000", 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000165923 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.717209+0000", 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027886590000000002 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.717533+0000", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00032422400000000002 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.983777+0000", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26624330800000001 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.983840+0000", 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3763000000000006e-05 2023-09-23T15:18:33.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.983892+0000", 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1266e-05 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106177 5.a 5:561ec9dc:::benchmark_data_smithi017_288048_object106176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.720283+0000", 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 573.01867538600004, 2023-09-23T15:18:33.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26373635699999998, 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106177 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720283+0000", 2023-09-23T15:18:33.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720283+0000", 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720282+0000", 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720305+0000", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2622000000000001e-05 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720321+0000", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6154000000000002e-05 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720329+0000", 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9340000000000006e-06 2023-09-23T15:18:33.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720380+0000", 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1743000000000002e-05 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720542+0000", 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000161485 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720709+0000", 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000167214 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.727902+0000", 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071932530000000001 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.727958+0000", 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5760000000000001e-05 2023-09-23T15:18:33.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.983932+0000", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25597333799999999 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.983984+0000", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2030999999999997e-05 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.984020+0000", 2023-09-23T15:18:33.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6140000000000003e-05 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106178 5.a 5:56b4f3e4:::benchmark_data_smithi017_288048_object106177:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.720341+0000", 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 573.01861787899998, 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26378721300000002, 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.756 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106178 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720341+0000", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720341+0000", 2023-09-23T15:18:33.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720340+0000", 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720359+0000", 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9080999999999999e-05 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720367+0000", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1470000000000004e-06 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.720372+0000", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4249999999999997e-06 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.724197+0000", 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038252170000000001 2023-09-23T15:18:33.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.724327+0000", 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130166 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.724497+0000", 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016971900000000001 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728117+0000", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036200899999999999 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.728125+0000", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8809999999999996e-06 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.984056+0000", 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25593129399999998 2023-09-23T15:18:33.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.984094+0000", 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7939999999999999e-05 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.984128+0000", 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3785000000000003e-05 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:106191 5.a 5:5d3118dd:::benchmark_data_smithi017_288048_object106190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e134)", 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:09:00.733009+0000", 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 573.00594991100002, 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25460040900000003, 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 106191 2023-09-23T15:18:33.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733009+0000", 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733009+0000", 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733007+0000", 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733033+0000", 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6169999999999998e-05 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733050+0000", 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6796999999999999e-05 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733060+0000", 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6199999999999994e-06 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733096+0000", 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6046e-05 2023-09-23T15:18:33.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733191+0000", 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5616000000000001e-05 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.733282+0000", 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0198000000000006e-05 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.736004+0000", 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027220289999999999 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.736085+0000", 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0988000000000005e-05 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.987511+0000", 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25142630500000002 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.987568+0000", 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6660000000000003e-05 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:09:00.987609+0000", 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1637999999999997e-05 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166948 5.b 5:dfd8a27b:::benchmark_data_smithi017_288048_object55179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.086662+0000", 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 495.65229713399998, 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28522272500000001, 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166948 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086662+0000", 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086662+0000", 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086660+0000", 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086667+0000", 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9260000000000003e-06 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086686+0000", 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8658999999999999e-05 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086693+0000", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2699999999999999e-06 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086729+0000", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6275000000000001e-05 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086819+0000", 2023-09-23T15:18:33.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9795000000000003e-05 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.086919+0000", 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9600999999999997e-05 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097351+0000", 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010432853000000001 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097426+0000", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4869000000000004e-05 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371798+0000", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27437133899999999 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371846+0000", 2023-09-23T15:18:33.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7916e-05 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371884+0000", 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8689999999999997e-05 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166952 5.b 5:da70994f:::benchmark_data_smithi017_288048_object55183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.089706+0000", 2023-09-23T15:18:33.776 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 495.64925301199997, 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28232127000000001, 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166952 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089706+0000", 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089706+0000", 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089704+0000", 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089712+0000", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9999999999999996e-06 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089731+0000", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9012e-05 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.089740+0000", 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4089999999999992e-06 2023-09-23T15:18:33.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.091927+0000", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021868399999999998 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092050+0000", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012294899999999999 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.092166+0000", 2023-09-23T15:18:33.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011615799999999999 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097417+0000", 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052503899999999997 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097440+0000", 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3073000000000001e-05 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371962+0000", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27452247099999999 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371999+0000", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6667e-05 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.372027+0000", 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8019999999999999e-05 2023-09-23T15:18:33.782 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:166956 5.d 5:bebbfc4b:::benchmark_data_smithi017_288048_object55187:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:18.096860+0000", 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 495.64209908499998, 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27504527600000001, 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166956 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096860+0000", 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096860+0000", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096858+0000", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096864+0000", 2023-09-23T15:18:33.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0170000000000002e-06 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096878+0000", 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3867e-05 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096885+0000", 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2940000000000002e-06 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.096941+0000", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6823999999999997e-05 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097097+0000", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015604400000000001 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.097174+0000", 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7034999999999997e-05 2023-09-23T15:18:33.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.101032+0000", 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038574669999999998 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.104349+0000", 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003316879 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371820+0000", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26747126799999998 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371867+0000", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7299999999999998e-05 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:18.371905+0000", 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7548e-05 2023-09-23T15:18:33.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.34764.0:168114 5.d 5:bef6c055:::benchmark_data_smithi017_288048_object56345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:10:20.026518+0000", 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 493.71244070400002, 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25419382800000001, 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.790 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.34764", 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 168114 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026518+0000", 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026518+0000", 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026517+0000", 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026520+0000", 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9399999999999998e-06 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026528+0000", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1040000000000001e-06 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026533+0000", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2449999999999998e-06 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026785+0000", 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025210899999999998 2023-09-23T15:18:33.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026884+0000", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8672999999999995e-05 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.026981+0000", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7646999999999995e-05 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.042144+0000", 2023-09-23T15:18:33.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015162474 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.279970+0000", 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23782589200000001 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.280593+0000", 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00062352800000000004 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.280654+0000", 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0968999999999999e-05 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:10:20.280712+0000", 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7778000000000001e-05 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78609 6.1 6:80d4e179:::benchmark_data_smithi017_350392_object78608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.89064999300001, 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3694808949999999, 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78609 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848307+0000", 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848333+0000", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5845000000000001e-05 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848345+0000", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2347000000000001e-05 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848356+0000", 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0528e-05 2023-09-23T15:18:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849744+0000", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001388467 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849757+0000", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2284e-05 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.198127+0000", 2023-09-23T15:18:33.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3483700409999999 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.200339+0000", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022119430000000001 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.200633+0000", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029397499999999998 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214375+0000", 2023-09-23T15:18:33.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013742694 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214502+0000", 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012641799999999999 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214546+0000", 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4276000000000001e-05 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214607+0000", 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0668999999999998e-05 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.217790+0000", 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031828379999999999 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78617 6.1 6:81e02ebd:::benchmark_data_smithi017_350392_object78616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:18:33.803 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.89058811400002, 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3832878639999999, 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78617 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848370+0000", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848402+0000", 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2126999999999999e-05 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848411+0000", 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2410000000000001e-06 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848420+0000", 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8629999999999997e-06 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849784+0000", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013640639999999999 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849792+0000", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7810000000000005e-06 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204128+0000", 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.354335651 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204325+0000", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197455 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204530+0000", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020487500000000001 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214389+0000", 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098588370000000005 2023-09-23T15:18:33.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218413+0000", 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040238640000000003 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228773+0000", 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010360163 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228831+0000", 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8516e-05 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231658+0000", 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028269829999999999 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.810 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78637 6.1 6:82bb8763:::benchmark_data_smithi017_350392_object78636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.89053201299998, 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3859898829999999, 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78637 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848426+0000", 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848450+0000", 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4318e-05 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848458+0000", 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4340000000000003e-06 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848464+0000", 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1079999999999998e-06 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849813+0000", 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001349329 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849820+0000", 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0110000000000003e-06 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208302+0000", 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3584813790000001 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208482+0000", 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017982600000000001 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:18:33.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208669+0000", 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018774899999999999 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214462+0000", 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057931709999999997 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218460+0000", 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039972669999999997 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231725+0000", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013265279 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231798+0000", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2589000000000006e-05 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.234417+0000", 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026190129999999999 2023-09-23T15:18:33.817 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147640 6.f 6:f5e90299:::benchmark_data_smithi017_350392_object47249:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 227.81240779699999, 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27895413200000002, 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:33.818 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147640 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:18:33.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926549+0000", 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926557+0000", 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.559e-06 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926571+0000", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3725e-05 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926580+0000", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5640000000000002e-06 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926626+0000", 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5689000000000003e-05 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926756+0000", 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129863 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926870+0000", 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114127 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.931667+0000", 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047970349999999998 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.931726+0000", 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8600000000000001e-05 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205421+0000", 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27369536900000002 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205468+0000", 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6739000000000001e-05 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205505+0000", 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7401000000000003e-05 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147645 6.f 6:ffe1d62f:::benchmark_data_smithi017_350392_object47254:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.937771+0000", 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 227.80118760600001, 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26944058599999998, 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147645 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937771+0000", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937771+0000", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937770+0000", 2023-09-23T15:18:33.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937776+0000", 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7959999999999998e-06 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937789+0000", 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2194e-05 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937796+0000", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3130000000000001e-06 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937840+0000", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4428000000000002e-05 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.937930+0000", 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0155000000000007e-05 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.938010+0000", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9164000000000003e-05 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.941748+0000", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037382050000000001 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.143273+0000", 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.201525499 2023-09-23T15:18:33.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.207138+0000", 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063865038999999998 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.207176+0000", 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7784000000000003e-05 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.207212+0000", 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5542e-05 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64528 7.8 7:1d508351:::benchmark_data_smithi017_422824_object64527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.847611+0000", 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.891347586999998, 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27063357199999999, 2023-09-23T15:18:33.831 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64528 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847611+0000", 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847611+0000", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847609+0000", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847636+0000", 2023-09-23T15:18:33.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6889999999999998e-05 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847649+0000", 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2493e-05 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847658+0000", 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6570000000000003e-06 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847786+0000", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012864300000000001 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847916+0000", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012997599999999999 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.848046+0000", 2023-09-23T15:18:33.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013034 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.850402+0000", 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023553699999999999 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117996+0000", 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26759391100000002 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118159+0000", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016274700000000001 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118205+0000", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6306e-05 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118245+0000", 2023-09-23T15:18:33.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9892999999999999e-05 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64529 7.e 7:7d16ebd8:::benchmark_data_smithi017_422824_object64528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.891283377000001, 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.273188133, 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.838 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64529 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847696+0000", 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0910000000000001e-05 2023-09-23T15:18:33.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847703+0000", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847709+0000", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4909999999999996e-06 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847787+0000", 2023-09-23T15:18:33.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8174000000000001e-05 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847961+0000", 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017452599999999999 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.848087+0000", 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012575800000000001 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.850201+0000", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021134719999999999 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120707+0000", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.270506674 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120747+0000", 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9394999999999998e-05 2023-09-23T15:18:33.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120796+0000", 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9567000000000002e-05 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120863+0000", 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7117000000000001e-05 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.844 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64532 7.e 7:731a4c04:::benchmark_data_smithi017_422824_object64531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.887884325000002, 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28321638999999998, 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:33.845 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64532 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851073+0000", 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851107+0000", 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4767000000000001e-05 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851120+0000", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2638e-05 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851128+0000", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1519999999999992e-06 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851174+0000", 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6133e-05 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118070+0000", 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26689595700000002 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118233+0000", 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016251300000000001 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.125753+0000", 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075200090000000002 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.128398+0000", 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026455459999999999 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134191+0000", 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057925729999999996 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134247+0000", 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6369999999999997e-05 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134291+0000", 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3470999999999998e-05 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.851 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64534 7.5 7:a537e92b:::benchmark_data_smithi017_422824_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.887816774000001, 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28027905600000003, 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64534 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:18:33.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851141+0000", 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851169+0000", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7999000000000001e-05 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851177+0000", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5970000000000003e-06 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851182+0000", 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4040000000000003e-06 2023-09-23T15:18:33.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117946+0000", 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266763213 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118119+0000", 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017326 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118260+0000", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140767 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129302+0000", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01104228 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129468+0000", 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166195 2023-09-23T15:18:33.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131318+0000", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018502740000000001 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131369+0000", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0327000000000001e-05 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131421+0000", 2023-09-23T15:18:33.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2238999999999999e-05 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64539 7.5 7:aff3ef34:::benchmark_data_smithi017_422824_object64538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.887761435999998, 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28619797000000002, 2023-09-23T15:18:33.858 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:33.859 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:18:33.859 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64539 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851217+0000", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0457000000000001e-05 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117948+0000", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26673075899999998 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117957+0000", 2023-09-23T15:18:33.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9560000000000003e-06 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120513+0000", 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002556034 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120654+0000", 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014076799999999999 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120798+0000", 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144394 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129322+0000", 2023-09-23T15:18:33.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085240860000000002 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129497+0000", 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000174737 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137299+0000", 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078019229999999997 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:18:33.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137345+0000", 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6551000000000002e-05 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137395+0000", 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9852999999999999e-05 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.867 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:64528 7.8 e237/223 ondisk, result = 0)", 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.853053+0000", 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.885905538999999, 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26494495200000001, 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853053+0000", 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853053+0000", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853052+0000", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853060+0000", 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.418e-06 2023-09-23T15:18:33.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117929+0000", 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26486956900000003 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117945+0000", 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6201e-05 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117984+0000", 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8824e-05 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117991+0000", 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9870000000000002e-06 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117998+0000", 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8630000000000002e-06 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.871 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:64529 7.e e237/223 ondisk, result = 0)", 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.853903+0000", 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 48.885055919999999, 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266809089, 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:18:33.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853903+0000", 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853903+0000", 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853902+0000", 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.853908+0000", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4049999999999997e-06 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117940+0000", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26403166099999997 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117951+0000", 2023-09-23T15:18:33.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0645000000000001e-05 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120692+0000", 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027408850000000002 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120700+0000", 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3080000000000008e-06 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:18:33.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120712+0000", 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1914e-05 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:18:33.876 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:18:33.894 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:33.961 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:33.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:34.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:34.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[215646]: pgmap v700: 109 pgs: 109 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 110 B/s rd, 15 MiB/s wr, 723 op/s 2023-09-23T15:18:34.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/166518359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:34.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:34.142 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:18:34.168 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:34.303 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:34 smithi027 ceph-mon[195233]: pgmap v700: 109 pgs: 109 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 110 B/s rd, 15 MiB/s wr, 723 op/s 2023-09-23T15:18:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/166518359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:34 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[222878]: pgmap v700: 109 pgs: 109 active+clean; 5.5 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 110 B/s rd, 15 MiB/s wr, 723 op/s 2023-09-23T15:18:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/166518359' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:34.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:34 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:18:35.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[215646]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[215646]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[215646]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:35.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[222878]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[222878]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:18:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:35 smithi017 ceph-mon[222878]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T15:18:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:35 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:35 smithi027 ceph-mon[195233]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:35 smithi027 ceph-mon[195233]: failed to encode map e247 with expected crc 2023-09-23T15:18:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:35 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:18:35.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:35 smithi027 ceph-mon[195233]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T15:18:35.812 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:35.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:35.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:35.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:36.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:36.264 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:36 smithi017 ceph-mon[215646]: pgmap v702: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 400 KiB/s wr, 648 op/s 2023-09-23T15:18:36.264 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:36 smithi017 ceph-mon[222878]: pgmap v702: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 400 KiB/s wr, 648 op/s 2023-09-23T15:18:36.389 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:36.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:36.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:36.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:18:36.507 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:36.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:36 smithi027 ceph-mon[195233]: pgmap v702: 109 pgs: 109 active+clean; 5.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 400 KiB/s wr, 648 op/s 2023-09-23T15:18:36.636 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:36.786 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:36.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:36.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:36.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:36.922 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:36.982 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:36.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:36.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:37.088 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:37.177 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:37.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:37.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:37.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:37.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:37.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:37.323 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:37.323 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:37.420 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:18:37.421 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:18:38.196 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:38] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2023-09-23T15:18:38.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:38 smithi017 ceph-mon[222878]: pgmap v703: 109 pgs: 109 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-09-23T15:18:38.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3378155500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:38.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:38 smithi017 ceph-mon[215646]: pgmap v703: 109 pgs: 109 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-09-23T15:18:38.197 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3378155500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:38.290 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:38.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:38.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:38.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:38 smithi027 ceph-mon[195233]: pgmap v703: 109 pgs: 109 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 837 op/s 2023-09-23T15:18:38.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3378155500' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:38.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:18:38.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:38.546 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:38.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:38.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:38.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:38.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:39.032 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:39.065 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:39.172 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:39.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:39.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:39.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:39.297 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:39.313 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:39.367 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:39.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:39.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:39.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:39.588 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:39.606 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:18:39.646 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:18:40.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:40 smithi017 ceph-mon[215646]: pgmap v704: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:18:40.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:40 smithi017 ceph-mon[222878]: pgmap v704: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:18:40.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:40 smithi027 ceph-mon[195233]: pgmap v704: 109 pgs: 109 active+clean; 5.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:18:40.357 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:40.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:40.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:40.465 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:40.521 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:40.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:18:41.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:41 smithi027 ceph-mon[195233]: from='client.55507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:41.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:41 smithi017 ceph-mon[215646]: from='client.55507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:41.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:41 smithi017 ceph-mon[222878]: from='client.55507 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:41.726 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:41.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:41.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:41.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:41.925 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:42.067 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":704,"stamp":"2023-09-23T15:18:38.534048+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5631776715,"num_objects":85890,"num_object_clones":0,"num_object_copies":257670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85890,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":102781,"num_write_kb":6020214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4724630,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102487,"ondisk_log_size":102487,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":18693268,"kb_used_data":16550236,"kb_used_omap":30,"kb_used_meta":2142561,"kb_avail":637522796,"statfs":{"total":671965249536,"available":652823343104,"internally_reserved":0,"allocated":16947441664,"data_stored":16941119118,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":2193983064},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,7,4,5,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-247857152,"num_objects":-3782,"num_object_clones":0,"num_object_copies":-11346,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3782,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3782,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.008092"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":241,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065226+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:18:31.065226+0000","last_peered":"2023-09-23T15:18:31.065226+0000","last_clean":"2023-09-23T15:18:31.065226+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:18:31.065226+0000","last_undegraded":"2023-09-23T15:18:31.065226+0000","last_fullsized":"2023-09-23T15:18:31.065226+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":176,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073206+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:18:31.073206+0000","last_peered":"2023-09-23T15:18:31.073206+0000","last_clean":"2023-09-23T15:18:31.073206+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:18:31.073206+0000","last_undegraded":"2023-09-23T15:18:31.073206+0000","last_fullsized":"2023-09-23T15:18:31.073206+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":125,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.106958+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:18:31.106958+0000","last_peered":"2023-09-23T15:18:31.106958+0000","last_clean":"2023-09-23T15:18:31.106958+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:18:31.106958+0000","last_undegraded":"2023-09-23T15:18:31.106958+0000","last_fullsized":"2023-09-23T15:18:31.106958+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":213,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.084205+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:18:31.084205+0000","last_peered":"2023-09-23T15:18:31.084205+0000","last_clean":"2023-09-23T15:18:31.084205+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:18:31.084205+0000","last_undegraded":"2023-09-23T15:18:31.084205+0000","last_fullsized":"2023-09-23T15:18:31.084205+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":246,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.071842+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:18:31.071842+0000","last_peered":"2023-09-23T15:18:31.071842+0000","last_clean":"2023-09-23T15:18:31.071842+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:18:31.071842+0000","last_undegraded":"2023-09-23T15:18:31.071842+0000","last_fullsized":"2023-09-23T15:18:31.071842+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":221,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073749+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:18:31.073749+0000","last_peered":"2023-09-23T15:18:31.073749+0000","last_clean":"2023-09-23T15:18:31.073749+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:18:31.073749+0000","last_undegraded":"2023-09-23T15:18:31.073749+0000","last_fullsized":"2023-09-23T15:18:31.073749+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":192,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.079196+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:18:31.079196+0000","last_peered":"2023-09-23T15:18:31.079196+0000","last_clean":"2023-09-23T15:18:31.079196+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:18:31.079196+0000","last_undegraded":"2023-09-23T15:18:31.079196+0000","last_fullsized":"2023-09-23T15:18:31.079196+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":228,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.071913+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:18:31.071913+0000","last_peered":"2023-09-23T15:18:31.071913+0000","last_clean":"2023-09-23T15:18:31.071913+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:18:31.071913+0000","last_undegraded":"2023-09-23T15:18:31.071913+0000","last_fullsized":"2023-09-23T15:18:31.071913+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":544,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T15:18:23.427243+0000","last_change":"2023-09-23T15:16:55.370503+0000","last_active":"2023-09-23T15:18:23.427243+0000","last_peered":"2023-09-23T15:18:23.427243+0000","last_clean":"2023-09-23T15:18:23.427243+0000","last_became_active":"2023-09-23T15:16:46.275526+0000","last_became_peered":"2023-09-23T15:16:46.275526+0000","last_unstale":"2023-09-23T15:18:23.427243+0000","last_undegraded":"2023-09-23T15:18:23.427243+0000","last_fullsized":"2023-09-23T15:18:23.427243+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:54.310407+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":237,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074801+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:18:31.074801+0000","last_peered":"2023-09-23T15:18:31.074801+0000","last_clean":"2023-09-23T15:18:31.074801+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:18:31.074801+0000","last_undegraded":"2023-09-23T15:18:31.074801+0000","last_fullsized":"2023-09-23T15:18:31.074801+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074046+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:18:31.074046+0000","last_peered":"2023-09-23T15:18:31.074046+0000","last_clean":"2023-09-23T15:18:31.074046+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:18:31.074046+0000","last_undegraded":"2023-09-23T15:18:31.074046+0000","last_fullsized":"2023-09-23T15:18:31.074046+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":110,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073134+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:18:31.073134+0000","last_peered":"2023-09-23T15:18:31.073134+0000","last_clean":"2023-09-23T15:18:31.073134+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:18:31.073134+0000","last_undegraded":"2023-09-23T15:18:31.073134+0000","last_fullsized":"2023-09-23T15:18:31.073134+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.107657+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:18:31.107657+0000","last_peered":"2023-09-23T15:18:31.107657+0000","last_clean":"2023-09-23T15:18:31.107657+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:18:31.107657+0000","last_undegraded":"2023-09-23T15:18:31.107657+0000","last_fullsized":"2023-09-23T15:18:31.107657+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":231,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065697+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:18:31.065697+0000","last_peered":"2023-09-23T15:18:31.065697+0000","last_clean":"2023-09-23T15:18:31.065697+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:18:31.065697+0000","last_undegraded":"2023-09-23T15:18:31.065697+0000","last_fullsized":"2023-09-23T15:18:31.065697+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":232,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.067230+0000","last_change":"2023-09-23T15:17:29.222343+0000","last_active":"2023-09-23T15:18:31.067230+0000","last_peered":"2023-09-23T15:18:31.067230+0000","last_clean":"2023-09-23T15:18:31.067230+0000","last_became_active":"2023-09-23T15:17:29.221973+0000","last_became_peered":"2023-09-23T15:17:29.221973+0000","last_unstale":"2023-09-23T15:18:31.067230+0000","last_undegraded":"2023-09-23T15:18:31.067230+0000","last_fullsized":"2023-09-23T15:18:31.067230+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:45.319242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":554,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.132198+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:18:34.132198+0000","last_peered":"2023-09-23T15:18:34.132198+0000","last_clean":"2023-09-23T15:18:34.132198+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:18:34.132198+0000","last_undegraded":"2023-09-23T15:18:34.132198+0000","last_fullsized":"2023-09-23T15:18:34.132198+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":109,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.067275+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:18:31.067275+0000","last_peered":"2023-09-23T15:18:31.067275+0000","last_clean":"2023-09-23T15:18:31.067275+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:18:31.067275+0000","last_undegraded":"2023-09-23T15:18:31.067275+0000","last_fullsized":"2023-09-23T15:18:31.067275+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074436+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:18:31.074436+0000","last_peered":"2023-09-23T15:18:31.074436+0000","last_clean":"2023-09-23T15:18:31.074436+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:18:31.074436+0000","last_undegraded":"2023-09-23T15:18:31.074436+0000","last_fullsized":"2023-09-23T15:18:31.074436+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":546,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074367+0000","last_change":"2023-09-23T15:16:46.276165+0000","last_active":"2023-09-23T15:18:31.074367+0000","last_peered":"2023-09-23T15:18:31.074367+0000","last_clean":"2023-09-23T15:18:31.074367+0000","last_became_active":"2023-09-23T15:16:46.275685+0000","last_became_peered":"2023-09-23T15:16:46.275685+0000","last_unstale":"2023-09-23T15:18:31.074367+0000","last_undegraded":"2023-09-23T15:18:31.074367+0000","last_fullsized":"2023-09-23T15:18:31.074367+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:36.377771+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":212,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.078586+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:18:31.078586+0000","last_peered":"2023-09-23T15:18:31.078586+0000","last_clean":"2023-09-23T15:18:31.078586+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:18:31.078586+0000","last_undegraded":"2023-09-23T15:18:31.078586+0000","last_fullsized":"2023-09-23T15:18:31.078586+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074696+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:18:31.074696+0000","last_peered":"2023-09-23T15:18:31.074696+0000","last_clean":"2023-09-23T15:18:31.074696+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:18:31.074696+0000","last_undegraded":"2023-09-23T15:18:31.074696+0000","last_fullsized":"2023-09-23T15:18:31.074696+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":500,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.108084+0000","last_change":"2023-09-23T15:16:51.467653+0000","last_active":"2023-09-23T15:18:31.108084+0000","last_peered":"2023-09-23T15:18:31.108084+0000","last_clean":"2023-09-23T15:18:31.108084+0000","last_became_active":"2023-09-23T15:16:46.277151+0000","last_became_peered":"2023-09-23T15:16:46.277151+0000","last_unstale":"2023-09-23T15:18:31.108084+0000","last_undegraded":"2023-09-23T15:18:31.108084+0000","last_fullsized":"2023-09-23T15:18:31.108084+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:46.522265+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":93,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065159+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:18:31.065159+0000","last_peered":"2023-09-23T15:18:31.065159+0000","last_clean":"2023-09-23T15:18:31.065159+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:18:31.065159+0000","last_undegraded":"2023-09-23T15:18:31.065159+0000","last_fullsized":"2023-09-23T15:18:31.065159+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":201,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.084355+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:18:31.084355+0000","last_peered":"2023-09-23T15:18:31.084355+0000","last_clean":"2023-09-23T15:18:31.084355+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:18:31.084355+0000","last_undegraded":"2023-09-23T15:18:31.084355+0000","last_fullsized":"2023-09-23T15:18:31.084355+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":546,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.075159+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:18:31.075159+0000","last_peered":"2023-09-23T15:18:31.075159+0000","last_clean":"2023-09-23T15:18:31.075159+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:18:31.075159+0000","last_undegraded":"2023-09-23T15:18:31.075159+0000","last_fullsized":"2023-09-23T15:18:31.075159+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":213,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073000+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:18:31.073000+0000","last_peered":"2023-09-23T15:18:31.073000+0000","last_clean":"2023-09-23T15:18:31.073000+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:18:31.073000+0000","last_undegraded":"2023-09-23T15:18:31.073000+0000","last_fullsized":"2023-09-23T15:18:31.073000+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074141+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:18:31.074141+0000","last_peered":"2023-09-23T15:18:31.074141+0000","last_clean":"2023-09-23T15:18:31.074141+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:18:31.074141+0000","last_undegraded":"2023-09-23T15:18:31.074141+0000","last_fullsized":"2023-09-23T15:18:31.074141+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"247'6635","reported_seq":6668,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:36.521856+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:18:36.521856+0000","last_peered":"2023-09-23T15:18:36.521856+0000","last_clean":"2023-09-23T15:18:36.521856+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:18:36.521856+0000","last_undegraded":"2023-09-23T15:18:36.521856+0000","last_fullsized":"2023-09-23T15:18:36.521856+0000","mapping_epoch":223,"log_start":"229'800","ondisk_log_start":"229'800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5835,"log_dups_size":0,"ondisk_log_size":5835,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354615296,"num_objects":5411,"num_object_clones":0,"num_object_copies":16233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6635,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":544,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065373+0000","last_change":"2023-09-23T15:16:51.284220+0000","last_active":"2023-09-23T15:18:31.065373+0000","last_peered":"2023-09-23T15:18:31.065373+0000","last_clean":"2023-09-23T15:18:31.065373+0000","last_became_active":"2023-09-23T15:16:46.276402+0000","last_became_peered":"2023-09-23T15:16:46.276402+0000","last_unstale":"2023-09-23T15:18:31.065373+0000","last_undegraded":"2023-09-23T15:18:31.065373+0000","last_fullsized":"2023-09-23T15:18:31.065373+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.284115+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.284115+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:27.778741+0000","scrub_duration":0.001563902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":121,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.066258+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:18:31.066258+0000","last_peered":"2023-09-23T15:18:31.066258+0000","last_clean":"2023-09-23T15:18:31.066258+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:18:31.066258+0000","last_undegraded":"2023-09-23T15:18:31.066258+0000","last_fullsized":"2023-09-23T15:18:31.066258+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":198,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074477+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:18:31.074477+0000","last_peered":"2023-09-23T15:18:31.074477+0000","last_clean":"2023-09-23T15:18:31.074477+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:18:31.074477+0000","last_undegraded":"2023-09-23T15:18:31.074477+0000","last_fullsized":"2023-09-23T15:18:31.074477+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"247'6352","reported_seq":6385,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:35.152632+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:18:35.152632+0000","last_peered":"2023-09-23T15:18:35.152632+0000","last_clean":"2023-09-23T15:18:35.152632+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:18:35.152632+0000","last_undegraded":"2023-09-23T15:18:35.152632+0000","last_fullsized":"2023-09-23T15:18:35.152632+0000","mapping_epoch":223,"log_start":"229'800","ondisk_log_start":"229'800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5552,"log_dups_size":0,"ondisk_log_size":5552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6352,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":549,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074358+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:18:31.074358+0000","last_peered":"2023-09-23T15:18:31.074358+0000","last_clean":"2023-09-23T15:18:31.074358+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:18:31.074358+0000","last_undegraded":"2023-09-23T15:18:31.074358+0000","last_fullsized":"2023-09-23T15:18:31.074358+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":121,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065603+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:18:31.065603+0000","last_peered":"2023-09-23T15:18:31.065603+0000","last_clean":"2023-09-23T15:18:31.065603+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:18:31.065603+0000","last_undegraded":"2023-09-23T15:18:31.065603+0000","last_fullsized":"2023-09-23T15:18:31.065603+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065072+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:18:31.065072+0000","last_peered":"2023-09-23T15:18:31.065072+0000","last_clean":"2023-09-23T15:18:31.065072+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:18:31.065072+0000","last_undegraded":"2023-09-23T15:18:31.065072+0000","last_fullsized":"2023-09-23T15:18:31.065072+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"247'6283","reported_seq":6316,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.752950+0000","last_change":"2023-09-23T15:16:44.269684+0000","last_active":"2023-09-23T15:18:34.752950+0000","last_peered":"2023-09-23T15:18:34.752950+0000","last_clean":"2023-09-23T15:18:34.752950+0000","last_became_active":"2023-09-23T15:16:44.269411+0000","last_became_peered":"2023-09-23T15:16:44.269411+0000","last_unstale":"2023-09-23T15:18:34.752950+0000","last_undegraded":"2023-09-23T15:18:34.752950+0000","last_fullsized":"2023-09-23T15:18:34.752950+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6283,"log_dups_size":0,"ondisk_log_size":6283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:33.481012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343998464,"num_objects":5249,"num_object_clones":0,"num_object_copies":15747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6283,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":469,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073587+0000","last_change":"2023-09-23T15:16:49.361068+0000","last_active":"2023-09-23T15:18:31.073587+0000","last_peered":"2023-09-23T15:18:31.073587+0000","last_clean":"2023-09-23T15:18:31.073587+0000","last_became_active":"2023-09-23T15:16:46.276691+0000","last_became_peered":"2023-09-23T15:16:46.276691+0000","last_unstale":"2023-09-23T15:18:31.073587+0000","last_undegraded":"2023-09-23T15:18:31.073587+0000","last_fullsized":"2023-09-23T15:18:31.073587+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:16.914979+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":107,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074748+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:18:31.074748+0000","last_peered":"2023-09-23T15:18:31.074748+0000","last_clean":"2023-09-23T15:18:31.074748+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:18:31.074748+0000","last_undegraded":"2023-09-23T15:18:31.074748+0000","last_fullsized":"2023-09-23T15:18:31.074748+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":235,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073307+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:18:31.073307+0000","last_peered":"2023-09-23T15:18:31.073307+0000","last_clean":"2023-09-23T15:18:31.073307+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:18:31.073307+0000","last_undegraded":"2023-09-23T15:18:31.073307+0000","last_fullsized":"2023-09-23T15:18:31.073307+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"246'6247","reported_seq":6279,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.006709+0000","last_change":"2023-09-23T15:16:44.271841+0000","last_active":"2023-09-23T15:18:34.006709+0000","last_peered":"2023-09-23T15:18:34.006709+0000","last_clean":"2023-09-23T15:18:34.006709+0000","last_became_active":"2023-09-23T15:16:44.271547+0000","last_became_peered":"2023-09-23T15:16:44.271547+0000","last_unstale":"2023-09-23T15:18:34.006709+0000","last_undegraded":"2023-09-23T15:18:34.006709+0000","last_fullsized":"2023-09-23T15:18:34.006709+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6247,"log_dups_size":0,"ondisk_log_size":6247,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:03.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346750976,"num_objects":5291,"num_object_clones":0,"num_object_copies":15873,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5291,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6247,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":544,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065502+0000","last_change":"2023-09-23T15:16:49.264186+0000","last_active":"2023-09-23T15:18:31.065502+0000","last_peered":"2023-09-23T15:18:31.065502+0000","last_clean":"2023-09-23T15:18:31.065502+0000","last_became_active":"2023-09-23T15:16:46.275415+0000","last_became_peered":"2023-09-23T15:16:46.275415+0000","last_unstale":"2023-09-23T15:18:31.065502+0000","last_undegraded":"2023-09-23T15:18:31.065502+0000","last_fullsized":"2023-09-23T15:18:31.065502+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.264096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.264096+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:25.863800+0000","scrub_duration":0.0040937940000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":206,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074606+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:18:31.074606+0000","last_peered":"2023-09-23T15:18:31.074606+0000","last_clean":"2023-09-23T15:18:31.074606+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:18:31.074606+0000","last_undegraded":"2023-09-23T15:18:31.074606+0000","last_fullsized":"2023-09-23T15:18:31.074606+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":208,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.078688+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:18:31.078688+0000","last_peered":"2023-09-23T15:18:31.078688+0000","last_clean":"2023-09-23T15:18:31.078688+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:18:31.078688+0000","last_undegraded":"2023-09-23T15:18:31.078688+0000","last_fullsized":"2023-09-23T15:18:31.078688+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"247'6360","reported_seq":6392,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.775843+0000","last_change":"2023-09-23T15:16:44.263685+0000","last_active":"2023-09-23T15:18:34.775843+0000","last_peered":"2023-09-23T15:18:34.775843+0000","last_clean":"2023-09-23T15:18:34.775843+0000","last_became_active":"2023-09-23T15:16:44.263391+0000","last_became_peered":"2023-09-23T15:16:44.263391+0000","last_unstale":"2023-09-23T15:18:34.775843+0000","last_undegraded":"2023-09-23T15:18:34.775843+0000","last_fullsized":"2023-09-23T15:18:34.775843+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6360,"log_dups_size":0,"ondisk_log_size":6360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:33.410338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353370112,"num_objects":5392,"num_object_clones":0,"num_object_copies":16176,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5392,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6360,"num_write_kb":376064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":545,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073855+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:18:31.073855+0000","last_peered":"2023-09-23T15:18:31.073855+0000","last_clean":"2023-09-23T15:18:31.073855+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:18:31.073855+0000","last_undegraded":"2023-09-23T15:18:31.073855+0000","last_fullsized":"2023-09-23T15:18:31.073855+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073413+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:18:31.073413+0000","last_peered":"2023-09-23T15:18:31.073413+0000","last_clean":"2023-09-23T15:18:31.073413+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:18:31.073413+0000","last_undegraded":"2023-09-23T15:18:31.073413+0000","last_fullsized":"2023-09-23T15:18:31.073413+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":205,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065397+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:18:31.065397+0000","last_peered":"2023-09-23T15:18:31.065397+0000","last_clean":"2023-09-23T15:18:31.065397+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:18:31.065397+0000","last_undegraded":"2023-09-23T15:18:31.065397+0000","last_fullsized":"2023-09-23T15:18:31.065397+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"246'6406","reported_seq":6438,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.007579+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:18:34.007579+0000","last_peered":"2023-09-23T15:18:34.007579+0000","last_clean":"2023-09-23T15:18:34.007579+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:18:34.007579+0000","last_undegraded":"2023-09-23T15:18:34.007579+0000","last_fullsized":"2023-09-23T15:18:34.007579+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6406,"log_dups_size":0,"ondisk_log_size":6406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352845824,"num_objects":5384,"num_object_clones":0,"num_object_copies":16152,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5384,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6406,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":551,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074326+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:18:31.074326+0000","last_peered":"2023-09-23T15:18:31.074326+0000","last_clean":"2023-09-23T15:18:31.074326+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:18:31.074326+0000","last_undegraded":"2023-09-23T15:18:31.074326+0000","last_fullsized":"2023-09-23T15:18:31.074326+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":222,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073214+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:18:31.073214+0000","last_peered":"2023-09-23T15:18:31.073214+0000","last_clean":"2023-09-23T15:18:31.073214+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:18:31.073214+0000","last_undegraded":"2023-09-23T15:18:31.073214+0000","last_fullsized":"2023-09-23T15:18:31.073214+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065607+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:18:31.065607+0000","last_peered":"2023-09-23T15:18:31.065607+0000","last_clean":"2023-09-23T15:18:31.065607+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:18:31.065607+0000","last_undegraded":"2023-09-23T15:18:31.065607+0000","last_fullsized":"2023-09-23T15:18:31.065607+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"247'6480","reported_seq":6513,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.193572+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:18:34.193572+0000","last_peered":"2023-09-23T15:18:34.193572+0000","last_clean":"2023-09-23T15:18:34.193572+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:18:34.193572+0000","last_undegraded":"2023-09-23T15:18:34.193572+0000","last_fullsized":"2023-09-23T15:18:34.193572+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6480,"log_dups_size":0,"ondisk_log_size":6480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":361824280,"num_objects":5522,"num_object_clones":0,"num_object_copies":16566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5522,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6480,"num_write_kb":384001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":500,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.108009+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:18:31.108009+0000","last_peered":"2023-09-23T15:18:31.108009+0000","last_clean":"2023-09-23T15:18:31.108009+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:18:31.108009+0000","last_undegraded":"2023-09-23T15:18:31.108009+0000","last_fullsized":"2023-09-23T15:18:31.108009+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":230,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.064947+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:18:31.064947+0000","last_peered":"2023-09-23T15:18:31.064947+0000","last_clean":"2023-09-23T15:18:31.064947+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:18:31.064947+0000","last_undegraded":"2023-09-23T15:18:31.064947+0000","last_fullsized":"2023-09-23T15:18:31.064947+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":232,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065602+0000","last_change":"2023-09-23T15:17:29.219853+0000","last_active":"2023-09-23T15:18:31.065602+0000","last_peered":"2023-09-23T15:18:31.065602+0000","last_clean":"2023-09-23T15:18:31.065602+0000","last_became_active":"2023-09-23T15:17:29.219486+0000","last_became_peered":"2023-09-23T15:17:29.219486+0000","last_unstale":"2023-09-23T15:18:31.065602+0000","last_undegraded":"2023-09-23T15:18:31.065602+0000","last_fullsized":"2023-09-23T15:18:31.065602+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:27.120247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"247'6421","reported_seq":6453,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.248066+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:18:34.248066+0000","last_peered":"2023-09-23T15:18:34.248066+0000","last_clean":"2023-09-23T15:18:34.248066+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:18:34.248066+0000","last_undegraded":"2023-09-23T15:18:34.248066+0000","last_fullsized":"2023-09-23T15:18:34.248066+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6421,"log_dups_size":0,"ondisk_log_size":6421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270656,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6421,"num_write_kb":378944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":546,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.075099+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:18:31.075099+0000","last_peered":"2023-09-23T15:18:31.075099+0000","last_clean":"2023-09-23T15:18:31.075099+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:18:31.075099+0000","last_undegraded":"2023-09-23T15:18:31.075099+0000","last_fullsized":"2023-09-23T15:18:31.075099+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":240,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.071764+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:18:31.071764+0000","last_peered":"2023-09-23T15:18:31.071764+0000","last_clean":"2023-09-23T15:18:31.071764+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:18:31.071764+0000","last_undegraded":"2023-09-23T15:18:31.071764+0000","last_fullsized":"2023-09-23T15:18:31.071764+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":125,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.109577+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:18:31.109577+0000","last_peered":"2023-09-23T15:18:31.109577+0000","last_clean":"2023-09-23T15:18:31.109577+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:18:31.109577+0000","last_undegraded":"2023-09-23T15:18:31.109577+0000","last_fullsized":"2023-09-23T15:18:31.109577+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"247'6415","reported_seq":6447,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.352010+0000","last_change":"2023-09-23T15:16:44.271607+0000","last_active":"2023-09-23T15:18:34.352010+0000","last_peered":"2023-09-23T15:18:34.352010+0000","last_clean":"2023-09-23T15:18:34.352010+0000","last_became_active":"2023-09-23T15:16:44.271334+0000","last_became_peered":"2023-09-23T15:16:44.271334+0000","last_unstale":"2023-09-23T15:18:34.352010+0000","last_undegraded":"2023-09-23T15:18:34.352010+0000","last_fullsized":"2023-09-23T15:18:34.352010+0000","mapping_epoch":223,"log_start":"229'400","ondisk_log_start":"229'400","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6015,"log_dups_size":0,"ondisk_log_size":6015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:58.532783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354877440,"num_objects":5415,"num_object_clones":0,"num_object_copies":16245,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5415,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6415,"num_write_kb":378560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":469,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073671+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:18:31.073671+0000","last_peered":"2023-09-23T15:18:31.073671+0000","last_clean":"2023-09-23T15:18:31.073671+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:18:31.073671+0000","last_undegraded":"2023-09-23T15:18:31.073671+0000","last_fullsized":"2023-09-23T15:18:31.073671+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":424,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.107408+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:18:31.107408+0000","last_peered":"2023-09-23T15:18:31.107408+0000","last_clean":"2023-09-23T15:18:31.107408+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:18:31.107408+0000","last_undegraded":"2023-09-23T15:18:31.107408+0000","last_fullsized":"2023-09-23T15:18:31.107408+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074608+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:18:31.074608+0000","last_peered":"2023-09-23T15:18:31.074608+0000","last_clean":"2023-09-23T15:18:31.074608+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:18:31.074608+0000","last_undegraded":"2023-09-23T15:18:31.074608+0000","last_fullsized":"2023-09-23T15:18:31.074608+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":169,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T15:18:23.271394+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:18:23.271394+0000","last_peered":"2023-09-23T15:18:23.271394+0000","last_clean":"2023-09-23T15:18:23.271394+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:18:23.271394+0000","last_undegraded":"2023-09-23T15:18:23.271394+0000","last_fullsized":"2023-09-23T15:18:23.271394+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"247'6255","reported_seq":6287,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.187015+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:18:34.187015+0000","last_peered":"2023-09-23T15:18:34.187015+0000","last_clean":"2023-09-23T15:18:34.187015+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:18:34.187015+0000","last_undegraded":"2023-09-23T15:18:34.187015+0000","last_fullsized":"2023-09-23T15:18:34.187015+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6255,"log_dups_size":0,"ondisk_log_size":6255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6255,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":544,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074477+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:18:31.074477+0000","last_peered":"2023-09-23T15:18:31.074477+0000","last_clean":"2023-09-23T15:18:31.074477+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:18:31.074477+0000","last_undegraded":"2023-09-23T15:18:31.074477+0000","last_fullsized":"2023-09-23T15:18:31.074477+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":765,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.075274+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:18:31.075274+0000","last_peered":"2023-09-23T15:18:31.075274+0000","last_clean":"2023-09-23T15:18:31.075274+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:18:31.075274+0000","last_undegraded":"2023-09-23T15:18:31.075274+0000","last_fullsized":"2023-09-23T15:18:31.075274+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":206,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.067179+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:18:31.067179+0000","last_peered":"2023-09-23T15:18:31.067179+0000","last_clean":"2023-09-23T15:18:31.067179+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:18:31.067179+0000","last_undegraded":"2023-09-23T15:18:31.067179+0000","last_fullsized":"2023-09-23T15:18:31.067179+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":140,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.107095+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:18:31.107095+0000","last_peered":"2023-09-23T15:18:31.107095+0000","last_clean":"2023-09-23T15:18:31.107095+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:18:31.107095+0000","last_undegraded":"2023-09-23T15:18:31.107095+0000","last_fullsized":"2023-09-23T15:18:31.107095+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"247'6218","reported_seq":6251,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.764510+0000","last_change":"2023-09-23T15:16:44.269240+0000","last_active":"2023-09-23T15:18:34.764510+0000","last_peered":"2023-09-23T15:18:34.764510+0000","last_clean":"2023-09-23T15:18:34.764510+0000","last_became_active":"2023-09-23T15:16:44.268966+0000","last_became_peered":"2023-09-23T15:16:44.268966+0000","last_unstale":"2023-09-23T15:18:34.764510+0000","last_undegraded":"2023-09-23T15:18:34.764510+0000","last_fullsized":"2023-09-23T15:18:34.764510+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6218,"log_dups_size":0,"ondisk_log_size":6218,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:01.694002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344326144,"num_objects":5254,"num_object_clones":0,"num_object_copies":15762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6218,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":554,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.078680+0000","last_change":"2023-09-23T15:16:46.276901+0000","last_active":"2023-09-23T15:18:31.078680+0000","last_peered":"2023-09-23T15:18:31.078680+0000","last_clean":"2023-09-23T15:18:31.078680+0000","last_became_active":"2023-09-23T15:16:46.276619+0000","last_became_peered":"2023-09-23T15:16:46.276619+0000","last_unstale":"2023-09-23T15:18:31.078680+0000","last_undegraded":"2023-09-23T15:18:31.078680+0000","last_fullsized":"2023-09-23T15:18:31.078680+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:27.244809+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":860,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T15:18:23.273165+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:18:23.273165+0000","last_peered":"2023-09-23T15:18:23.273165+0000","last_clean":"2023-09-23T15:18:23.273165+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:18:23.273165+0000","last_undegraded":"2023-09-23T15:18:23.273165+0000","last_fullsized":"2023-09-23T15:18:23.273165+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.110372+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:18:31.110372+0000","last_peered":"2023-09-23T15:18:31.110372+0000","last_clean":"2023-09-23T15:18:31.110372+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:18:31.110372+0000","last_undegraded":"2023-09-23T15:18:31.110372+0000","last_fullsized":"2023-09-23T15:18:31.110372+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":144,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.109498+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:18:31.109498+0000","last_peered":"2023-09-23T15:18:31.109498+0000","last_clean":"2023-09-23T15:18:31.109498+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:18:31.109498+0000","last_undegraded":"2023-09-23T15:18:31.109498+0000","last_fullsized":"2023-09-23T15:18:31.109498+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"247'6422","reported_seq":6455,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.750998+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:18:34.750998+0000","last_peered":"2023-09-23T15:18:34.750998+0000","last_clean":"2023-09-23T15:18:34.750998+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:18:34.750998+0000","last_undegraded":"2023-09-23T15:18:34.750998+0000","last_fullsized":"2023-09-23T15:18:34.750998+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6422,"log_dups_size":0,"ondisk_log_size":6422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352976896,"num_objects":5386,"num_object_clones":0,"num_object_copies":16158,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5386,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6422,"num_write_kb":377856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":534,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074821+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:18:31.074821+0000","last_peered":"2023-09-23T15:18:31.074821+0000","last_clean":"2023-09-23T15:18:31.074821+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:18:31.074821+0000","last_undegraded":"2023-09-23T15:18:31.074821+0000","last_fullsized":"2023-09-23T15:18:31.074821+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":537,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.076086+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:18:31.076086+0000","last_peered":"2023-09-23T15:18:31.076086+0000","last_clean":"2023-09-23T15:18:31.076086+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:18:31.076086+0000","last_undegraded":"2023-09-23T15:18:31.076086+0000","last_fullsized":"2023-09-23T15:18:31.076086+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":162,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.080159+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:18:31.080159+0000","last_peered":"2023-09-23T15:18:31.080159+0000","last_clean":"2023-09-23T15:18:31.080159+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:18:31.080159+0000","last_undegraded":"2023-09-23T15:18:31.080159+0000","last_fullsized":"2023-09-23T15:18:31.080159+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073586+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:18:31.073586+0000","last_peered":"2023-09-23T15:18:31.073586+0000","last_clean":"2023-09-23T15:18:31.073586+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:18:31.073586+0000","last_undegraded":"2023-09-23T15:18:31.073586+0000","last_fullsized":"2023-09-23T15:18:31.073586+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"246'6232","reported_seq":6264,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.007693+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:18:34.007693+0000","last_peered":"2023-09-23T15:18:34.007693+0000","last_clean":"2023-09-23T15:18:34.007693+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:18:34.007693+0000","last_undegraded":"2023-09-23T15:18:34.007693+0000","last_fullsized":"2023-09-23T15:18:34.007693+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6232,"log_dups_size":0,"ondisk_log_size":6232,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352583680,"num_objects":5380,"num_object_clones":0,"num_object_copies":16140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6232,"num_write_kb":371584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":500,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.107932+0000","last_change":"2023-09-23T15:16:48.526093+0000","last_active":"2023-09-23T15:18:31.107932+0000","last_peered":"2023-09-23T15:18:31.107932+0000","last_clean":"2023-09-23T15:18:31.107932+0000","last_became_active":"2023-09-23T15:16:46.276681+0000","last_became_peered":"2023-09-23T15:16:46.276681+0000","last_unstale":"2023-09-23T15:18:31.107932+0000","last_undegraded":"2023-09-23T15:18:31.107932+0000","last_fullsized":"2023-09-23T15:18:31.107932+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:34.078052+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":331,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.110159+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:18:31.110159+0000","last_peered":"2023-09-23T15:18:31.110159+0000","last_clean":"2023-09-23T15:18:31.110159+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:18:31.110159+0000","last_undegraded":"2023-09-23T15:18:31.110159+0000","last_fullsized":"2023-09-23T15:18:31.110159+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":193,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.079110+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:18:31.079110+0000","last_peered":"2023-09-23T15:18:31.079110+0000","last_clean":"2023-09-23T15:18:31.079110+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:18:31.079110+0000","last_undegraded":"2023-09-23T15:18:31.079110+0000","last_fullsized":"2023-09-23T15:18:31.079110+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":223,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.083975+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:18:31.083975+0000","last_peered":"2023-09-23T15:18:31.083975+0000","last_clean":"2023-09-23T15:18:31.083975+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:18:31.083975+0000","last_undegraded":"2023-09-23T15:18:31.083975+0000","last_fullsized":"2023-09-23T15:18:31.083975+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"247'6321","reported_seq":6353,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.708046+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:18:34.708046+0000","last_peered":"2023-09-23T15:18:34.708046+0000","last_clean":"2023-09-23T15:18:34.708046+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:18:34.708046+0000","last_undegraded":"2023-09-23T15:18:34.708046+0000","last_fullsized":"2023-09-23T15:18:34.708046+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6321,"log_dups_size":0,"ondisk_log_size":6321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6321,"num_write_kb":372160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":469,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073482+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:18:31.073482+0000","last_peered":"2023-09-23T15:18:31.073482+0000","last_clean":"2023-09-23T15:18:31.073482+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:18:31.073482+0000","last_undegraded":"2023-09-23T15:18:31.073482+0000","last_fullsized":"2023-09-23T15:18:31.073482+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":869,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.088117+0000","last_change":"2023-09-23T15:16:38.973875+0000","last_active":"2023-09-23T15:18:31.088117+0000","last_peered":"2023-09-23T15:18:31.088117+0000","last_clean":"2023-09-23T15:18:31.088117+0000","last_became_active":"2023-09-23T15:16:38.973269+0000","last_became_peered":"2023-09-23T15:16:38.973269+0000","last_unstale":"2023-09-23T15:18:31.088117+0000","last_undegraded":"2023-09-23T15:18:31.088117+0000","last_fullsized":"2023-09-23T15:18:31.088117+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:17:00.522554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.107275+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:18:31.107275+0000","last_peered":"2023-09-23T15:18:31.107275+0000","last_clean":"2023-09-23T15:18:31.107275+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:18:31.107275+0000","last_undegraded":"2023-09-23T15:18:31.107275+0000","last_fullsized":"2023-09-23T15:18:31.107275+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":198,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.078922+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:18:31.078922+0000","last_peered":"2023-09-23T15:18:31.078922+0000","last_clean":"2023-09-23T15:18:31.078922+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:18:31.078922+0000","last_undegraded":"2023-09-23T15:18:31.078922+0000","last_fullsized":"2023-09-23T15:18:31.078922+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"247'6497","reported_seq":6530,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:35.141680+0000","last_change":"2023-09-23T15:16:44.260057+0000","last_active":"2023-09-23T15:18:35.141680+0000","last_peered":"2023-09-23T15:18:35.141680+0000","last_clean":"2023-09-23T15:18:35.141680+0000","last_became_active":"2023-09-23T15:16:44.259815+0000","last_became_peered":"2023-09-23T15:16:44.259815+0000","last_unstale":"2023-09-23T15:18:35.141680+0000","last_undegraded":"2023-09-23T15:18:35.141680+0000","last_fullsized":"2023-09-23T15:18:35.141680+0000","mapping_epoch":223,"log_start":"229'1000","ondisk_log_start":"229'1000","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5497,"log_dups_size":0,"ondisk_log_size":5497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:50.147826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357105664,"num_objects":5449,"num_object_clones":0,"num_object_copies":16347,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5449,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6497,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":544,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065459+0000","last_change":"2023-09-23T15:16:52.317545+0000","last_active":"2023-09-23T15:18:31.065459+0000","last_peered":"2023-09-23T15:18:31.065459+0000","last_clean":"2023-09-23T15:18:31.065459+0000","last_became_active":"2023-09-23T15:16:46.275975+0000","last_became_peered":"2023-09-23T15:16:46.275975+0000","last_unstale":"2023-09-23T15:18:31.065459+0000","last_undegraded":"2023-09-23T15:18:31.065459+0000","last_fullsized":"2023-09-23T15:18:31.065459+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:52.317424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:52.317424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:29.771978+0000","scrub_duration":0.001669299,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":224,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074051+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:18:31.074051+0000","last_peered":"2023-09-23T15:18:31.074051+0000","last_clean":"2023-09-23T15:18:31.074051+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:18:31.074051+0000","last_undegraded":"2023-09-23T15:18:31.074051+0000","last_fullsized":"2023-09-23T15:18:31.074051+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":708,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.109861+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:18:31.109861+0000","last_peered":"2023-09-23T15:18:31.109861+0000","last_clean":"2023-09-23T15:18:31.109861+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:18:31.109861+0000","last_undegraded":"2023-09-23T15:18:31.109861+0000","last_fullsized":"2023-09-23T15:18:31.109861+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":229,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.071984+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:18:31.071984+0000","last_peered":"2023-09-23T15:18:31.071984+0000","last_clean":"2023-09-23T15:18:31.071984+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:18:31.071984+0000","last_undegraded":"2023-09-23T15:18:31.071984+0000","last_fullsized":"2023-09-23T15:18:31.071984+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"247'6330","reported_seq":6363,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:35.118504+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:18:35.118504+0000","last_peered":"2023-09-23T15:18:35.118504+0000","last_clean":"2023-09-23T15:18:35.118504+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:18:35.118504+0000","last_undegraded":"2023-09-23T15:18:35.118504+0000","last_fullsized":"2023-09-23T15:18:35.118504+0000","mapping_epoch":223,"log_start":"229'800","ondisk_log_start":"229'800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352190464,"num_objects":5374,"num_object_clones":0,"num_object_copies":16122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6330,"num_write_kb":374528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":545,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.076539+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:18:31.076539+0000","last_peered":"2023-09-23T15:18:31.076539+0000","last_clean":"2023-09-23T15:18:31.076539+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:18:31.076539+0000","last_undegraded":"2023-09-23T15:18:31.076539+0000","last_fullsized":"2023-09-23T15:18:31.076539+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":723,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073652+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:18:31.073652+0000","last_peered":"2023-09-23T15:18:31.073652+0000","last_clean":"2023-09-23T15:18:31.073652+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:18:31.073652+0000","last_undegraded":"2023-09-23T15:18:31.073652+0000","last_fullsized":"2023-09-23T15:18:31.073652+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":211,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.084067+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:18:31.084067+0000","last_peered":"2023-09-23T15:18:31.084067+0000","last_clean":"2023-09-23T15:18:31.084067+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:18:31.084067+0000","last_undegraded":"2023-09-23T15:18:31.084067+0000","last_fullsized":"2023-09-23T15:18:31.084067+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":188,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065008+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:18:31.065008+0000","last_peered":"2023-09-23T15:18:31.065008+0000","last_clean":"2023-09-23T15:18:31.065008+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:18:31.065008+0000","last_undegraded":"2023-09-23T15:18:31.065008+0000","last_fullsized":"2023-09-23T15:18:31.065008+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":200,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.078838+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:18:31.078838+0000","last_peered":"2023-09-23T15:18:31.078838+0000","last_clean":"2023-09-23T15:18:31.078838+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:18:31.078838+0000","last_undegraded":"2023-09-23T15:18:31.078838+0000","last_fullsized":"2023-09-23T15:18:31.078838+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":247,"reported_epoch":245,"state":"active+clean","last_fresh":"2023-09-23T15:18:23.427593+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:18:23.427593+0000","last_peered":"2023-09-23T15:18:23.427593+0000","last_clean":"2023-09-23T15:18:23.427593+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:18:23.427593+0000","last_undegraded":"2023-09-23T15:18:23.427593+0000","last_fullsized":"2023-09-23T15:18:23.427593+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":225,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074659+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:18:31.074659+0000","last_peered":"2023-09-23T15:18:31.074659+0000","last_clean":"2023-09-23T15:18:31.074659+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:18:31.074659+0000","last_undegraded":"2023-09-23T15:18:31.074659+0000","last_fullsized":"2023-09-23T15:18:31.074659+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":239,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.067295+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:18:31.067295+0000","last_peered":"2023-09-23T15:18:31.067295+0000","last_clean":"2023-09-23T15:18:31.067295+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:18:31.067295+0000","last_undegraded":"2023-09-23T15:18:31.067295+0000","last_fullsized":"2023-09-23T15:18:31.067295+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":191,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.074598+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:18:31.074598+0000","last_peered":"2023-09-23T15:18:31.074598+0000","last_clean":"2023-09-23T15:18:31.074598+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:18:31.074598+0000","last_undegraded":"2023-09-23T15:18:31.074598+0000","last_fullsized":"2023-09-23T15:18:31.074598+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":222,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.065112+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:18:31.065112+0000","last_peered":"2023-09-23T15:18:31.065112+0000","last_clean":"2023-09-23T15:18:31.065112+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:18:31.065112+0000","last_undegraded":"2023-09-23T15:18:31.065112+0000","last_fullsized":"2023-09-23T15:18:31.065112+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":220,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073629+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:18:31.073629+0000","last_peered":"2023-09-23T15:18:31.073629+0000","last_clean":"2023-09-23T15:18:31.073629+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:18:31.073629+0000","last_undegraded":"2023-09-23T15:18:31.073629+0000","last_fullsized":"2023-09-23T15:18:31.073629+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":234,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.073974+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:18:31.073974+0000","last_peered":"2023-09-23T15:18:31.073974+0000","last_clean":"2023-09-23T15:18:31.073974+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:18:31.073974+0000","last_undegraded":"2023-09-23T15:18:31.073974+0000","last_fullsized":"2023-09-23T15:18:31.073974+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":196,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.079283+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:18:31.079283+0000","last_peered":"2023-09-23T15:18:31.079283+0000","last_clean":"2023-09-23T15:18:31.079283+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:18:31.079283+0000","last_undegraded":"2023-09-23T15:18:31.079283+0000","last_fullsized":"2023-09-23T15:18:31.079283+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":215,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.064911+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:18:31.064911+0000","last_peered":"2023-09-23T15:18:31.064911+0000","last_clean":"2023-09-23T15:18:31.064911+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:18:31.064911+0000","last_undegraded":"2023-09-23T15:18:31.064911+0000","last_fullsized":"2023-09-23T15:18:31.064911+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5626986520,"num_objects":85862,"num_object_clones":0,"num_object_copies":257586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85862,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":101874,"num_write_kb":6007489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16853053440,"data_stored":16853041224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":98074,"ondisk_log_size":98074,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201566,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463202,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1963464,"kb_used_data":1752128,"kb_used_omap":0,"kb_used_meta":211328,"kb_avail":91781688,"statfs":{"total":95995035648,"available":93984448512,"internally_reserved":0,"allocated":1794179072,"data_stored":1793431655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":216399872},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.407,"15min":2.7879999999999998},"min":{"1min":0.22800000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":7.5099999999999998,"5min":12.926,"15min":108.39400000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.5780000000000001,"5min":2.016,"15min":3.0659999999999998},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":7.7279999999999998,"5min":12.634,"15min":108.452},"last":6.5179999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:17:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.214,"15min":3.3180000000000001},"min":{"1min":0.27600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.236999999999995,"5min":86.236999999999995,"15min":86.236999999999995},"last":1.3380000000000001},{"interface":"front","average":{"1min":5.1609999999999996,"5min":4.0999999999999996,"15min":3.7709999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":5.2809999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":6.157,"5min":6.1630000000000003,"15min":6.5309999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.255},"max":{"1min":86.569000000000003,"5min":112.352,"15min":180.374},"last":0.999},{"interface":"front","average":{"1min":5.3170000000000002,"5min":5.8200000000000003,"15min":6.4199999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":88.269000000000005,"5min":117.667,"15min":181.685},"last":0.75600000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:17:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":7.2699999999999996,"15min":3.2280000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":6.7549999999999999,"5min":102.254,"15min":102.254},"last":3.149},{"interface":"front","average":{"1min":1.7649999999999999,"5min":7.4059999999999997,"15min":3.2410000000000001},"min":{"1min":0.42399999999999999,"5min":0.374,"15min":0.29799999999999999},"max":{"1min":8.1479999999999997,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.55700000000000005,"15min":1.0309999999999999},"min":{"1min":0.28899999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.70199999999999996,"5min":6.5119999999999996,"15min":36.290999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.60899999999999999,"15min":1.121},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":1.444,"5min":4.7590000000000003,"15min":36.758000000000003},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.68999999999999995,"15min":1.411},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":0.91300000000000003,"5min":5.5880000000000001,"15min":131.90600000000001},"last":0.371},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.64600000000000002,"15min":1.4550000000000001},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":2.0070000000000001,"5min":4.7859999999999996,"15min":131.61000000000001},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:17:41 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.81999999999999995,"15min":1.0309999999999999},"min":{"1min":0.36799999999999999,"5min":0.32600000000000001,"15min":0.28999999999999998},"max":{"1min":2.306,"5min":5.125,"15min":17},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.74399999999999999,"15min":0.98699999999999999},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":1.327,"5min":3.754,"15min":7.7809999999999997},"last":0.56100000000000005}]}]},{"osd":1,"up_from":183,"seq":785979015240,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2635556,"kb_used_data":2373788,"kb_used_omap":6,"kb_used_meta":261753,"kb_avail":91109596,"statfs":{"total":95995035648,"available":93296226304,"internally_reserved":0,"allocated":2430758912,"data_stored":2429649516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":268035147},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":2.7719999999999998,"15min":3.46},"min":{"1min":0.26000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.4619999999999997,"5min":16.626000000000001,"15min":16.626000000000001},"last":2.1309999999999998},{"interface":"front","average":{"1min":2.2229999999999999,"5min":3.968,"15min":4.3959999999999999},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.3260000000000005,"5min":19.044,"15min":19.044},"last":2}]},{"osd":2,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.0620000000000003,"15min":4.1059999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.4900000000000002,"5min":33.880000000000003,"15min":33.880000000000003},"last":5.6269999999999998},{"interface":"front","average":{"1min":1.627,"5min":3.2130000000000001,"15min":3.8860000000000001},"min":{"1min":0.28699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0589999999999993,"5min":16.882000000000001,"15min":16.882000000000001},"last":3.5470000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":7.5949999999999998,"15min":9.7289999999999992},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.1240000000000006,"5min":211.405,"15min":211.405},"last":2.2639999999999998},{"interface":"front","average":{"1min":2.363,"5min":7.8319999999999999,"15min":10.988},"min":{"1min":0.378,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.2850000000000001,"5min":210.988,"15min":210.988},"last":1.823}]},{"osd":4,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":8.0980000000000008,"5min":3.8279999999999998,"15min":3.073},"min":{"1min":0.40400000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":68.805999999999997,"5min":68.805999999999997,"15min":68.805999999999997},"last":1.8580000000000001},{"interface":"front","average":{"1min":6.9299999999999997,"5min":3.5619999999999998,"15min":3.5590000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":68.861000000000004,"5min":68.861000000000004,"15min":68.861000000000004},"last":2.1720000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.907,"15min":3.488},"min":{"1min":0.49299999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11,"5min":16.166,"15min":16.166},"last":1.756},{"interface":"front","average":{"1min":3.1539999999999999,"5min":3.1080000000000001,"15min":2.9670000000000001},"min":{"1min":0.44600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.776,"5min":11.487,"15min":11.487},"last":1.8839999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":8.5670000000000002,"5min":4.0439999999999996,"15min":3.8879999999999999},"min":{"1min":0.86899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":70.602000000000004,"5min":70.602000000000004,"15min":70.602000000000004},"last":1.796},{"interface":"front","average":{"1min":9.0879999999999992,"5min":4.3200000000000003,"15min":3.2719999999999998},"min":{"1min":0.47099999999999997,"5min":0.377,"15min":0.377},"max":{"1min":72.239999999999995,"5min":72.239999999999995,"15min":72.239999999999995},"last":1.9119999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.0049999999999999,"15min":3.1059999999999999},"min":{"1min":0.496,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":8.3499999999999996,"5min":11.561,"15min":11.561},"last":1.9430000000000001},{"interface":"front","average":{"1min":1.234,"5min":2.516,"15min":3.0739999999999998},"min":{"1min":0.42699999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.0329999999999995,"5min":16.841000000000001,"15min":16.841000000000001},"last":2.23}]}]},{"osd":0,"up_from":95,"seq":408021893289,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3143404,"kb_used_data":2759588,"kb_used_omap":8,"kb_used_meta":383799,"kb_avail":90601748,"statfs":{"total":95995035648,"available":92776189952,"internally_reserved":0,"allocated":2825818112,"data_stored":2824809556,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":393010772},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:17:48 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":2.903,"15min":3.8290000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":44.075000000000003,"5min":44.075000000000003,"15min":44.075000000000003},"last":0.40699999999999997},{"interface":"front","average":{"1min":4.6609999999999996,"5min":3.1850000000000001,"15min":4.2009999999999996},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":43.593000000000004,"5min":43.593000000000004,"15min":43.593000000000004},"last":1.5940000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":5.8710000000000004,"15min":3.7519999999999998},"min":{"1min":0.33600000000000002,"5min":0.24099999999999999,"15min":0.17000000000000001},"max":{"1min":15.209,"5min":93.873999999999995,"15min":122.20099999999999},"last":7.8920000000000003},{"interface":"front","average":{"1min":2.3340000000000001,"5min":5.3760000000000003,"15min":3.3399999999999999},"min":{"1min":0.24099999999999999,"5min":0.23499999999999999,"15min":0.18099999999999999},"max":{"1min":23.719999999999999,"5min":94.652000000000001,"15min":122.071},"last":0.59499999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:17:35 2023","interfaces":[{"interface":"back","average":{"1min":1.845,"5min":8.8629999999999995,"15min":4.0380000000000003},"min":{"1min":0.309,"5min":0.26200000000000001,"15min":0.20000000000000001},"max":{"1min":5.4660000000000002,"5min":146.315,"15min":146.315},"last":8.1899999999999995},{"interface":"front","average":{"1min":1.641,"5min":10.147,"15min":4.3730000000000002},"min":{"1min":0.34499999999999997,"5min":0.307,"15min":0.252},"max":{"1min":4.4400000000000004,"5min":145.517,"15min":145.517},"last":8.1329999999999991}]},{"osd":4,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":2.399,"15min":1.776},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":7.0199999999999996,"5min":31.341999999999999,"15min":31.341999999999999},"last":1.6299999999999999},{"interface":"front","average":{"1min":1.454,"5min":2.2450000000000001,"15min":1.8129999999999999},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.27000000000000002},"max":{"1min":8.0559999999999992,"5min":31.059999999999999,"15min":99.168999999999997},"last":7.8639999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.968,"15min":1.8129999999999999},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.2949999999999999,"5min":31.25,"15min":109.369},"last":0.44},{"interface":"front","average":{"1min":1.825,"5min":2.1019999999999999,"15min":1.972},"min":{"1min":0.41899999999999998,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":6.9779999999999998,"5min":31.295000000000002,"15min":109.387},"last":1.6799999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":2.0419999999999998,"15min":1.871},"min":{"1min":0.42799999999999999,"5min":0.35899999999999999,"15min":0.309},"max":{"1min":8.1050000000000004,"5min":16.893999999999998,"15min":99.114999999999995},"last":7.8140000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.994,"15min":1.7390000000000001},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":4.2489999999999997,"5min":31.431000000000001,"15min":98.921999999999997},"last":0.54200000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":2.1669999999999998,"15min":1.861},"min":{"1min":0.41599999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.3280000000000003,"5min":31.472000000000001,"15min":99.549999999999997},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.871,"5min":2.2759999999999998,"15min":1.6819999999999999},"min":{"1min":0.498,"5min":0.46700000000000003,"15min":0.36399999999999999},"max":{"1min":7.0579999999999998,"5min":31.225999999999999,"15min":31.225999999999999},"last":1.7070000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566236,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2827936,"kb_used_data":2426904,"kb_used_omap":11,"kb_used_meta":400820,"kb_avail":90917216,"statfs":{"total":95995035648,"available":93099229184,"internally_reserved":0,"allocated":2485149696,"data_stored":2484155721,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":410440464},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":5.2169999999999996,"15min":2.492},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":15.15,"5min":87.132999999999996,"15min":87.132999999999996},"last":6.2770000000000001},{"interface":"front","average":{"1min":2.3370000000000002,"5min":4.1890000000000001,"15min":2.097},"min":{"1min":0.246,"5min":0.246,"15min":0.23100000000000001},"max":{"1min":14.82,"5min":84.820999999999998,"15min":84.820999999999998},"last":6.7610000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":4.883,"15min":4.9870000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.625,"5min":84.641000000000005,"15min":84.641000000000005},"last":6.6070000000000002},{"interface":"front","average":{"1min":2.1560000000000001,"5min":3.9470000000000001,"15min":3.0150000000000001},"min":{"1min":0.313,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.742,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.29699999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:17:33 2023","interfaces":[{"interface":"back","average":{"1min":1.51,"5min":7.5250000000000004,"15min":3.2509999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":5.032,"5min":87.108000000000004,"15min":87.108000000000004},"last":0.24199999999999999},{"interface":"front","average":{"1min":1.7390000000000001,"5min":7.0350000000000001,"15min":3.1259999999999999},"min":{"1min":0.28799999999999998,"5min":0.24399999999999999,"15min":0.20300000000000001},"max":{"1min":6.6970000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":6.5430000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.8450000000000002,"15min":1.982},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":5.0330000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":6.7350000000000003},{"interface":"front","average":{"1min":1.8360000000000001,"5min":3.7610000000000001,"15min":2.6030000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7080000000000002,"5min":87.019999999999996,"15min":87.019999999999996},"last":0.497}]},{"osd":5,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":3.427,"15min":2.2549999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.1269999999999998,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.64400000000000002},{"interface":"front","average":{"1min":1.222,"5min":2.984,"15min":2.0720000000000001},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":5.0949999999999998,"5min":84.664000000000001,"15min":84.664000000000001},"last":6.6769999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":3.0979999999999999,"15min":2.1890000000000001},"min":{"1min":0.47399999999999998,"5min":0.35999999999999999,"15min":0.32100000000000001},"max":{"1min":5.0659999999999998,"5min":84.715999999999994,"15min":84.715999999999994},"last":6.6340000000000003},{"interface":"front","average":{"1min":2.121,"5min":3.8879999999999999,"15min":2.3279999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.9779999999999998,"5min":87.055000000000007,"15min":87.055000000000007},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":3.52,"15min":2.1099999999999999},"min":{"1min":0.42399999999999999,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":5.4089999999999998,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.288,"5min":3.0310000000000001,"15min":1.956},"min":{"1min":0.49199999999999999,"5min":0.41299999999999998,"15min":0.34200000000000003},"max":{"1min":5.1550000000000002,"5min":84.783000000000001,"15min":84.783000000000001},"last":6.7089999999999996}]}]},{"osd":3,"up_from":110,"seq":472446402710,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3832340,"kb_used_data":3440652,"kb_used_omap":3,"kb_used_meta":391484,"kb_avail":89912812,"statfs":{"total":95995035648,"available":92070719488,"internally_reserved":0,"allocated":3523227648,"data_stored":3522246623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":400879785},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":3.1099999999999999,"15min":2.2749999999999999},"min":{"1min":0.20499999999999999,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":6.0499999999999998,"5min":25.111999999999998,"15min":79.611000000000004},"last":10.586},{"interface":"front","average":{"1min":1.3799999999999999,"5min":3.3109999999999999,"15min":2.391},"min":{"1min":0.32200000000000001,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":4.8099999999999996,"5min":18.800000000000001,"15min":79.656000000000006},"last":0.78100000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":8.0640000000000001,"5min":5.2300000000000004,"15min":4.7999999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":119.343,"5min":119.343,"15min":119.343},"last":0.24299999999999999},{"interface":"front","average":{"1min":8.3369999999999997,"5min":5.0250000000000004,"15min":4.9050000000000002},"min":{"1min":0.249,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":119.505,"5min":119.505,"15min":119.505},"last":9.2949999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.647,"5min":4.4770000000000003,"15min":4.0800000000000001},"min":{"1min":0.36699999999999999,"5min":0.255,"15min":0.22},"max":{"1min":6.0140000000000002,"5min":61.732999999999997,"15min":109.077},"last":9.1609999999999996},{"interface":"front","average":{"1min":1.643,"5min":4.6529999999999996,"15min":4.3250000000000002},"min":{"1min":0.28399999999999997,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":5.9199999999999999,"5min":60.305999999999997,"15min":119.72799999999999},"last":4.2450000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.175,"5min":3.0030000000000001,"15min":1.675},"min":{"1min":0.32100000000000001,"5min":0.312,"15min":0.27600000000000002},"max":{"1min":3.8580000000000001,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.093,"5min":2.8849999999999998,"15min":1.8140000000000001},"min":{"1min":0.38200000000000001,"5min":0.29199999999999998,"15min":0.26400000000000001},"max":{"1min":3.9540000000000002,"5min":52.228999999999999,"15min":52.228999999999999},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.5550000000000002,"15min":1.8160000000000001},"min":{"1min":0.29599999999999999,"5min":0.28100000000000003,"15min":0.25600000000000001},"max":{"1min":3.8340000000000001,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.1319999999999999,"5min":2.5070000000000001,"15min":2.012},"min":{"1min":0.32300000000000001,"5min":0.315,"15min":0.315},"max":{"1min":4.0010000000000003,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.4239999999999999,"5min":3.371,"15min":1.9610000000000001},"min":{"1min":0.34699999999999998,"5min":0.32300000000000001,"15min":0.32200000000000001},"max":{"1min":4.782,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.876},{"interface":"front","average":{"1min":1.169,"5min":3.2330000000000001,"15min":2.1520000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.32300000000000001},"max":{"1min":4.1059999999999999,"5min":78.028000000000006,"15min":78.028000000000006},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":1.2010000000000001,"5min":2.552,"15min":1.9379999999999999},"min":{"1min":0.438,"5min":0.42599999999999999,"15min":0.28299999999999997},"max":{"1min":4.4400000000000004,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.647,"5min":2.7469999999999999,"15min":1.623},"min":{"1min":0.48199999999999998,"5min":0.376,"15min":0.34799999999999998},"max":{"1min":5.9550000000000001,"5min":18.696000000000002,"15min":18.696000000000002},"last":9.0079999999999991}]}]},{"osd":4,"up_from":31,"seq":133143986475,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2308300,"kb_used_data":2061508,"kb_used_omap":0,"kb_used_meta":246784,"kb_avail":91436852,"statfs":{"total":95995035648,"available":93631336448,"internally_reserved":0,"allocated":2110984192,"data_stored":2110240838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":252706816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:03 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.1520000000000001,"15min":2.7749999999999999},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":14.212999999999999,"5min":65.988,"15min":122.69199999999999},"last":1.966},{"interface":"front","average":{"1min":2.274,"5min":2.4910000000000001,"15min":2.9159999999999999},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.0060000000000002,"5min":65.396000000000001,"15min":122.745},"last":1.2589999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.3959999999999999,"15min":3.831},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.6299999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":1.073},{"interface":"front","average":{"1min":1.2250000000000001,"5min":3.0110000000000001,"15min":4.3879999999999999},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4890000000000008,"5min":53.078000000000003,"15min":53.078000000000003},"last":3.9889999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:18:03 2023","interfaces":[{"interface":"back","average":{"1min":5.109,"5min":3.0990000000000002,"15min":3.468},"min":{"1min":0.34999999999999998,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":78.751000000000005,"5min":78.751000000000005,"15min":148.08000000000001},"last":3.3999999999999999},{"interface":"front","average":{"1min":5.0830000000000002,"5min":3.5009999999999999,"15min":3.722},"min":{"1min":0.317,"5min":0.317,"15min":0.25700000000000001},"max":{"1min":77.930999999999997,"5min":77.930999999999997,"15min":147.88900000000001},"last":0.98499999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:17:33 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":7.7169999999999996,"15min":4.0460000000000003},"min":{"1min":0.38200000000000001,"5min":0.28899999999999998,"15min":0.252},"max":{"1min":1.6379999999999999,"5min":135.607,"15min":163.97499999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.999,"5min":7.7770000000000001,"15min":3.8660000000000001},"min":{"1min":0.373,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":3.9350000000000001,"5min":135.33600000000001,"15min":164.447},"last":3.581}]},{"osd":5,"last update":"Sat Sep 23 15:17:45 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":1.4450000000000001,"15min":1.5149999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":1.2430000000000001,"5min":87.912999999999997,"15min":87.912999999999997},"last":0.30599999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":1.5780000000000001,"15min":1.395},"min":{"1min":0.28199999999999997,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":1.292,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 15:18:32 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.6910000000000001,"15min":2.097},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.1699999999999999,"5min":86.091999999999999,"15min":210.44300000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.503,"5min":1.7869999999999999,"15min":2.0009999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5350000000000001,"5min":86.048000000000002,"15min":210.78899999999999},"last":0.33900000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:17:33 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":0.60199999999999998,"15min":1.1279999999999999},"min":{"1min":0.34599999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.0979999999999999,"5min":4.5869999999999997,"15min":18.23},"last":0.437},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.82199999999999995,"15min":1.198},"min":{"1min":0.39600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":1.1200000000000001,"5min":6.1399999999999997,"15min":14.653},"last":0.40000000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724838,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1982268,"kb_used_data":1735668,"kb_used_omap":0,"kb_used_meta":246592,"kb_avail":91762884,"statfs":{"total":95995035648,"available":93965193216,"internally_reserved":0,"allocated":1777324032,"data_stored":1776585209,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":252510208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":5.7199999999999998,"5min":2.4769999999999999,"15min":1.798},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.20599999999999999},"max":{"1min":84.266000000000005,"5min":84.266000000000005,"15min":84.266000000000005},"last":1.341},{"interface":"front","average":{"1min":6.2199999999999998,"5min":2.9089999999999998,"15min":1.865},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.222},"max":{"1min":84.558000000000007,"5min":84.558000000000007,"15min":84.558000000000007},"last":0.89200000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":2.27,"15min":1.8720000000000001},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2389999999999999,"5min":18.082999999999998,"15min":18.082999999999998},"last":6.1100000000000003},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.3380000000000001,"15min":2.2919999999999998},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.3780000000000001,"5min":17.963999999999999,"15min":17.963999999999999},"last":5.9299999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.855,"15min":2.6600000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23599999999999999},"max":{"1min":22.065999999999999,"5min":33.68,"15min":97.087999999999994},"last":1.2350000000000001},{"interface":"front","average":{"1min":2.7719999999999998,"5min":3.2090000000000001,"15min":2.6739999999999999},"min":{"1min":0.34300000000000003,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":22.081,"5min":33.707999999999998,"15min":97.305999999999997},"last":2.1259999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":5.8730000000000002,"15min":3.1520000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":4.484,"5min":118.208,"15min":118.208},"last":1.4390000000000001},{"interface":"front","average":{"1min":1.2210000000000001,"5min":5.6340000000000003,"15min":2.9399999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":3.1150000000000002,"5min":118.42,"15min":118.42},"last":1.5509999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":2.3570000000000002,"15min":1.772},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":12.202,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.3149999999999999,"15min":1.8380000000000001},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.26000000000000001},"max":{"1min":12.481,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:18:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.66500000000000004,"15min":1.127},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.3859999999999999,"5min":5.8760000000000003,"15min":15.196},"last":0.378},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59799999999999998,"15min":1.008},"min":{"1min":0.35499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.97399999999999998,"5min":2.9990000000000001,"15min":14.965999999999999},"last":0.442}]},{"osd":7,"last update":"Sat Sep 23 15:17:37 2023","interfaces":[{"interface":"back","average":{"1min":0.63700000000000001,"5min":0.84999999999999998,"15min":1.2869999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":2.1869999999999998,"5min":8.2200000000000006,"15min":15.301},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.80800000000000005,"15min":1.2629999999999999},"min":{"1min":0.38900000000000001,"5min":0.34999999999999998,"15min":0.311},"max":{"1min":2.29,"5min":5.6559999999999997,"15min":15.762},"last":0.47699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2802913280,"data_stored":2802909208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2421755904,"data_stored":2421751832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2475884544,"data_stored":2475884544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3501260800,"data_stored":3501260800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2105147392,"data_stored":2105147392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1762787328,"data_stored":1762787328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1783304192,"data_stored":1783300120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:18:42.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:18:42.111 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:42.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:42.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:42.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:42.252 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:42.303 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:42.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:42.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:42.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:42 smithi017 ceph-mon[215646]: pgmap v705: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:18:42.337 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:42 smithi017 ceph-mon[222878]: pgmap v705: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:18:42.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:42 smithi027 ceph-mon[195233]: pgmap v705: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:18:42.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:42.437 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:42.491 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:42.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:42.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:42.570 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:42.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:42.713 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:42.753 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:18:42.754 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:18:42.830 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:42.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:42.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:42.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:42.970 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:43.023 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:43.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:43.032 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:43.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:43.163 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:43.219 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:43.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:43.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:43.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:43.360 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:43.416 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:43.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:43.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:43.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:43.559 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:43.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:43.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:43.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:43.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:43.800 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:44.025 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:44.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:44.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:18:44.108 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:44.115 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:18:44.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:44.238 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:44 smithi027 ceph-mon[195233]: pgmap v706: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T15:18:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:44.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1028059588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:44.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[222878]: pgmap v706: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T15:18:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1028059588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:44.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:44.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[215646]: pgmap v706: 109 pgs: 109 active+clean; 5.0 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 711 op/s 2023-09-23T15:18:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1028059588' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:44.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T15:18:45.104 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[215646]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.385 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:45.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:45.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:18:45.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.386 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:45.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T15:18:45.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:45 smithi017 ceph-mon[222878]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:45.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:45.505 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:45.536 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:45.536 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:18:45.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:18:45.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:18:45.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T15:18:45.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:45 smithi027 ceph-mon[195233]: failed to encode map e248 with expected crc 2023-09-23T15:18:45.590 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:45.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:45.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:45.727 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:45.779 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:45.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:45.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:45.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:18:45.925 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:45.981 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:18:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:45.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:18:46.089 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:46.186 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:46 smithi017 ceph-mon[222878]: pgmap v708: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:46.186 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:46 smithi017 ceph-mon[215646]: pgmap v708: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:46.196 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:46.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:46 smithi027 ceph-mon[195233]: pgmap v708: 109 pgs: 109 active+clean; 4.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:46.673 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:46.680 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:46.712 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:46.739 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:46.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:46.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:46.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:46.801 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:46.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:46.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:46.878 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:46.894 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:46.933 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:46.935 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:46.944 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.960 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.961 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:18:46.961 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:18:47.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:47.075 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:47.132 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:47.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:47.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:47.141 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4164047473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:47.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:47.268 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4164047473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:47.269 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4164047473' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:47.341 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:47.477 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:18:47.789 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:47.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:47.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:47.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:47.948 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:48.066 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:48] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2023-09-23T15:18:48.067 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:48.343 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:48 smithi017 ceph-mon[215646]: pgmap v709: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:48.343 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:48 smithi017 ceph-mon[222878]: pgmap v709: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:48.503 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:48.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:48.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:48.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:48 smithi027 ceph-mon[195233]: pgmap v709: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 697 op/s 2023-09-23T15:18:48.614 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:48.703 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:48.805 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:48.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:48.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:48.889 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:18:48.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:48.922 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:18:48.944 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:49.012 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:49.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:49.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:49.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:49.208 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:49.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:18:50.022 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:50.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:50.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:50.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:50.208 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:50.229 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":710,"stamp":"2023-09-23T15:18:48.539747+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5177612235,"num_objects":78960,"num_object_clones":0,"num_object_copies":236880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78960,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":109711,"num_write_kb":6020214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":4724630,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106917,"ondisk_log_size":106917,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":17345492,"kb_used_data":15206556,"kb_used_omap":30,"kb_used_meta":2138657,"kb_avail":638870572,"statfs":{"total":671965249536,"available":654203465728,"internally_reserved":0,"allocated":15571513344,"data_stored":15565190542,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":2189985368},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-454164480,"num_objects":-6930,"num_object_clones":0,"num_object_copies":-20790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6930,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005698"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":242,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135936+0000","last_change":"2023-09-23T15:12:41.858094+0000","last_active":"2023-09-23T15:18:34.135936+0000","last_peered":"2023-09-23T15:18:34.135936+0000","last_clean":"2023-09-23T15:18:34.135936+0000","last_became_active":"2023-09-23T15:12:41.857602+0000","last_became_peered":"2023-09-23T15:12:41.857602+0000","last_unstale":"2023-09-23T15:18:34.135936+0000","last_undegraded":"2023-09-23T15:18:34.135936+0000","last_fullsized":"2023-09-23T15:18:34.135936+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:21:53.479228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":178,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.096550+0000","last_change":"2023-09-23T15:06:16.134796+0000","last_active":"2023-09-23T15:18:44.096550+0000","last_peered":"2023-09-23T15:18:44.096550+0000","last_clean":"2023-09-23T15:18:44.096550+0000","last_became_active":"2023-09-23T15:06:16.134296+0000","last_became_peered":"2023-09-23T15:06:16.134296+0000","last_unstale":"2023-09-23T15:18:44.096550+0000","last_undegraded":"2023-09-23T15:18:44.096550+0000","last_fullsized":"2023-09-23T15:18:44.096550+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:47:16.868168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":126,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136171+0000","last_change":"2023-09-23T15:12:41.920940+0000","last_active":"2023-09-23T15:18:34.136171+0000","last_peered":"2023-09-23T15:18:34.136171+0000","last_clean":"2023-09-23T15:18:34.136171+0000","last_became_active":"2023-09-23T15:12:41.920591+0000","last_became_peered":"2023-09-23T15:12:41.920591+0000","last_unstale":"2023-09-23T15:18:34.136171+0000","last_undegraded":"2023-09-23T15:18:34.136171+0000","last_fullsized":"2023-09-23T15:18:34.136171+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:42.409672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":215,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:39.778389+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:18:39.778389+0000","last_peered":"2023-09-23T15:18:39.778389+0000","last_clean":"2023-09-23T15:18:39.778389+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:18:39.778389+0000","last_undegraded":"2023-09-23T15:18:39.778389+0000","last_fullsized":"2023-09-23T15:18:39.778389+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":247,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148409+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:18:34.148409+0000","last_peered":"2023-09-23T15:18:34.148409+0000","last_clean":"2023-09-23T15:18:34.148409+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:18:34.148409+0000","last_undegraded":"2023-09-23T15:18:34.148409+0000","last_fullsized":"2023-09-23T15:18:34.148409+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134413+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:18:34.134413+0000","last_peered":"2023-09-23T15:18:34.134413+0000","last_clean":"2023-09-23T15:18:34.134413+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:18:34.134413+0000","last_undegraded":"2023-09-23T15:18:34.134413+0000","last_fullsized":"2023-09-23T15:18:34.134413+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":193,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131374+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:18:34.131374+0000","last_peered":"2023-09-23T15:18:34.131374+0000","last_clean":"2023-09-23T15:18:34.131374+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:18:34.131374+0000","last_undegraded":"2023-09-23T15:18:34.131374+0000","last_fullsized":"2023-09-23T15:18:34.131374+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":229,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148484+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:18:34.148484+0000","last_peered":"2023-09-23T15:18:34.148484+0000","last_clean":"2023-09-23T15:18:34.148484+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:18:34.148484+0000","last_undegraded":"2023-09-23T15:18:34.148484+0000","last_fullsized":"2023-09-23T15:18:34.148484+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":545,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.085600+0000","last_change":"2023-09-23T15:16:55.370503+0000","last_active":"2023-09-23T15:18:31.085600+0000","last_peered":"2023-09-23T15:18:31.085600+0000","last_clean":"2023-09-23T15:18:31.085600+0000","last_became_active":"2023-09-23T15:16:46.275526+0000","last_became_peered":"2023-09-23T15:16:46.275526+0000","last_unstale":"2023-09-23T15:18:31.085600+0000","last_undegraded":"2023-09-23T15:18:31.085600+0000","last_fullsized":"2023-09-23T15:18:31.085600+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:54.310407+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":238,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128694+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:18:34.128694+0000","last_peered":"2023-09-23T15:18:34.128694+0000","last_clean":"2023-09-23T15:18:34.128694+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:18:34.128694+0000","last_undegraded":"2023-09-23T15:18:34.128694+0000","last_fullsized":"2023-09-23T15:18:34.128694+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":172,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148445+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:18:34.148445+0000","last_peered":"2023-09-23T15:18:34.148445+0000","last_clean":"2023-09-23T15:18:34.148445+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:18:34.148445+0000","last_undegraded":"2023-09-23T15:18:34.148445+0000","last_fullsized":"2023-09-23T15:18:34.148445+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":111,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.145686+0000","last_change":"2023-09-23T15:11:34.149709+0000","last_active":"2023-09-23T15:18:34.145686+0000","last_peered":"2023-09-23T15:18:34.145686+0000","last_clean":"2023-09-23T15:18:34.145686+0000","last_became_active":"2023-09-23T15:11:34.142940+0000","last_became_peered":"2023-09-23T15:11:34.142940+0000","last_unstale":"2023-09-23T15:18:34.145686+0000","last_undegraded":"2023-09-23T15:18:34.145686+0000","last_fullsized":"2023-09-23T15:18:34.145686+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:10.960926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136313+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:18:34.136313+0000","last_peered":"2023-09-23T15:18:34.136313+0000","last_clean":"2023-09-23T15:18:34.136313+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:18:34.136313+0000","last_undegraded":"2023-09-23T15:18:34.136313+0000","last_fullsized":"2023-09-23T15:18:34.136313+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":232,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134233+0000","last_change":"2023-09-23T15:11:34.165491+0000","last_active":"2023-09-23T15:18:34.134233+0000","last_peered":"2023-09-23T15:18:34.134233+0000","last_clean":"2023-09-23T15:18:34.134233+0000","last_became_active":"2023-09-23T15:11:34.154712+0000","last_became_peered":"2023-09-23T15:11:34.154712+0000","last_unstale":"2023-09-23T15:18:34.134233+0000","last_undegraded":"2023-09-23T15:18:34.134233+0000","last_fullsized":"2023-09-23T15:18:34.134233+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:57.334982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134792+0000","last_change":"2023-09-23T15:17:29.222343+0000","last_active":"2023-09-23T15:18:34.134792+0000","last_peered":"2023-09-23T15:18:34.134792+0000","last_clean":"2023-09-23T15:18:34.134792+0000","last_became_active":"2023-09-23T15:17:29.221973+0000","last_became_peered":"2023-09-23T15:17:29.221973+0000","last_unstale":"2023-09-23T15:18:34.134792+0000","last_undegraded":"2023-09-23T15:18:34.134792+0000","last_fullsized":"2023-09-23T15:18:34.134792+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:45.319242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":555,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.777342+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:18:34.777342+0000","last_peered":"2023-09-23T15:18:34.777342+0000","last_clean":"2023-09-23T15:18:34.777342+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:18:34.777342+0000","last_undegraded":"2023-09-23T15:18:34.777342+0000","last_fullsized":"2023-09-23T15:18:34.777342+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":110,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134852+0000","last_change":"2023-09-23T15:11:34.169925+0000","last_active":"2023-09-23T15:18:34.134852+0000","last_peered":"2023-09-23T15:18:34.134852+0000","last_clean":"2023-09-23T15:18:34.134852+0000","last_became_active":"2023-09-23T15:11:34.155212+0000","last_became_peered":"2023-09-23T15:11:34.155212+0000","last_unstale":"2023-09-23T15:18:34.134852+0000","last_undegraded":"2023-09-23T15:18:34.134852+0000","last_fullsized":"2023-09-23T15:18:34.134852+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:56.130326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146159+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:18:34.146159+0000","last_peered":"2023-09-23T15:18:34.146159+0000","last_clean":"2023-09-23T15:18:34.146159+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:18:34.146159+0000","last_undegraded":"2023-09-23T15:18:34.146159+0000","last_fullsized":"2023-09-23T15:18:34.146159+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":547,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.127985+0000","last_change":"2023-09-23T15:16:46.276165+0000","last_active":"2023-09-23T15:18:34.127985+0000","last_peered":"2023-09-23T15:18:34.127985+0000","last_clean":"2023-09-23T15:18:34.127985+0000","last_became_active":"2023-09-23T15:16:46.275685+0000","last_became_peered":"2023-09-23T15:16:46.275685+0000","last_unstale":"2023-09-23T15:18:34.127985+0000","last_undegraded":"2023-09-23T15:18:34.127985+0000","last_fullsized":"2023-09-23T15:18:34.127985+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:36.377771+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":214,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:39.778287+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:18:39.778287+0000","last_peered":"2023-09-23T15:18:39.778287+0000","last_clean":"2023-09-23T15:18:39.778287+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:18:39.778287+0000","last_undegraded":"2023-09-23T15:18:39.778287+0000","last_fullsized":"2023-09-23T15:18:39.778287+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128345+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:18:34.128345+0000","last_peered":"2023-09-23T15:18:34.128345+0000","last_clean":"2023-09-23T15:18:34.128345+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:18:34.128345+0000","last_undegraded":"2023-09-23T15:18:34.128345+0000","last_fullsized":"2023-09-23T15:18:34.128345+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":501,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136066+0000","last_change":"2023-09-23T15:16:51.467653+0000","last_active":"2023-09-23T15:18:34.136066+0000","last_peered":"2023-09-23T15:18:34.136066+0000","last_clean":"2023-09-23T15:18:34.136066+0000","last_became_active":"2023-09-23T15:16:46.277151+0000","last_became_peered":"2023-09-23T15:16:46.277151+0000","last_unstale":"2023-09-23T15:18:34.136066+0000","last_undegraded":"2023-09-23T15:18:34.136066+0000","last_fullsized":"2023-09-23T15:18:34.136066+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:46.522265+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":94,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135056+0000","last_change":"2023-09-23T15:11:34.153622+0000","last_active":"2023-09-23T15:18:34.135056+0000","last_peered":"2023-09-23T15:18:34.135056+0000","last_clean":"2023-09-23T15:18:34.135056+0000","last_became_active":"2023-09-23T15:11:34.153301+0000","last_became_peered":"2023-09-23T15:11:34.153301+0000","last_unstale":"2023-09-23T15:18:34.135056+0000","last_undegraded":"2023-09-23T15:18:34.135056+0000","last_fullsized":"2023-09-23T15:18:34.135056+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:02.965867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":202,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131434+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:18:34.131434+0000","last_peered":"2023-09-23T15:18:34.131434+0000","last_clean":"2023-09-23T15:18:34.131434+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:18:34.131434+0000","last_undegraded":"2023-09-23T15:18:34.131434+0000","last_fullsized":"2023-09-23T15:18:34.131434+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":547,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.129217+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:18:34.129217+0000","last_peered":"2023-09-23T15:18:34.129217+0000","last_clean":"2023-09-23T15:18:34.129217+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:18:34.129217+0000","last_undegraded":"2023-09-23T15:18:34.129217+0000","last_fullsized":"2023-09-23T15:18:34.129217+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":214,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.133769+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:18:34.133769+0000","last_peered":"2023-09-23T15:18:34.133769+0000","last_clean":"2023-09-23T15:18:34.133769+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:18:34.133769+0000","last_undegraded":"2023-09-23T15:18:34.133769+0000","last_fullsized":"2023-09-23T15:18:34.133769+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146258+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:18:34.146258+0000","last_peered":"2023-09-23T15:18:34.146258+0000","last_clean":"2023-09-23T15:18:34.146258+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:18:34.146258+0000","last_undegraded":"2023-09-23T15:18:34.146258+0000","last_fullsized":"2023-09-23T15:18:34.146258+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"248'7099","reported_seq":7133,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:46.431276+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:18:46.431276+0000","last_peered":"2023-09-23T15:18:46.431276+0000","last_clean":"2023-09-23T15:18:46.431276+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:18:46.431276+0000","last_undegraded":"2023-09-23T15:18:46.431276+0000","last_fullsized":"2023-09-23T15:18:46.431276+0000","mapping_epoch":223,"log_start":"229'1300","ondisk_log_start":"229'1300","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5799,"log_dups_size":0,"ondisk_log_size":5799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324206592,"num_objects":4947,"num_object_clones":0,"num_object_copies":14841,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4947,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7099,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":545,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134829+0000","last_change":"2023-09-23T15:16:51.284220+0000","last_active":"2023-09-23T15:18:34.134829+0000","last_peered":"2023-09-23T15:18:34.134829+0000","last_clean":"2023-09-23T15:18:34.134829+0000","last_became_active":"2023-09-23T15:16:46.276402+0000","last_became_peered":"2023-09-23T15:16:46.276402+0000","last_unstale":"2023-09-23T15:18:34.134829+0000","last_undegraded":"2023-09-23T15:18:34.134829+0000","last_fullsized":"2023-09-23T15:18:34.134829+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.284115+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.284115+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:27.778741+0000","scrub_duration":0.001563902,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":122,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134473+0000","last_change":"2023-09-23T15:12:42.924165+0000","last_active":"2023-09-23T15:18:34.134473+0000","last_peered":"2023-09-23T15:18:34.134473+0000","last_clean":"2023-09-23T15:18:34.134473+0000","last_became_active":"2023-09-23T15:12:42.887943+0000","last_became_peered":"2023-09-23T15:12:42.887943+0000","last_unstale":"2023-09-23T15:18:34.134473+0000","last_undegraded":"2023-09-23T15:18:34.134473+0000","last_fullsized":"2023-09-23T15:18:34.134473+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:31.030692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":199,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128066+0000","last_change":"2023-09-23T15:11:34.158939+0000","last_active":"2023-09-23T15:18:34.128066+0000","last_peered":"2023-09-23T15:18:34.128066+0000","last_clean":"2023-09-23T15:18:34.128066+0000","last_became_active":"2023-09-23T15:11:34.158255+0000","last_became_peered":"2023-09-23T15:11:34.158255+0000","last_unstale":"2023-09-23T15:18:34.128066+0000","last_undegraded":"2023-09-23T15:18:34.128066+0000","last_fullsized":"2023-09-23T15:18:34.128066+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:13.684224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"248'6784","reported_seq":6818,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:45.141575+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:18:45.141575+0000","last_peered":"2023-09-23T15:18:45.141575+0000","last_clean":"2023-09-23T15:18:45.141575+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:18:45.141575+0000","last_undegraded":"2023-09-23T15:18:45.141575+0000","last_fullsized":"2023-09-23T15:18:45.141575+0000","mapping_epoch":223,"log_start":"229'1300","ondisk_log_start":"229'1300","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6784,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":550,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146146+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:18:34.146146+0000","last_peered":"2023-09-23T15:18:34.146146+0000","last_clean":"2023-09-23T15:18:34.146146+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:18:34.146146+0000","last_undegraded":"2023-09-23T15:18:34.146146+0000","last_fullsized":"2023-09-23T15:18:34.146146+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":122,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136277+0000","last_change":"2023-09-23T15:12:42.888536+0000","last_active":"2023-09-23T15:18:34.136277+0000","last_peered":"2023-09-23T15:18:34.136277+0000","last_clean":"2023-09-23T15:18:34.136277+0000","last_became_active":"2023-09-23T15:12:42.888149+0000","last_became_peered":"2023-09-23T15:12:42.888149+0000","last_unstale":"2023-09-23T15:18:34.136277+0000","last_undegraded":"2023-09-23T15:18:34.136277+0000","last_fullsized":"2023-09-23T15:18:34.136277+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:17.021362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134929+0000","last_change":"2023-09-23T15:12:41.858908+0000","last_active":"2023-09-23T15:18:34.134929+0000","last_peered":"2023-09-23T15:18:34.134929+0000","last_clean":"2023-09-23T15:18:34.134929+0000","last_became_active":"2023-09-23T15:12:41.858616+0000","last_became_peered":"2023-09-23T15:12:41.858616+0000","last_unstale":"2023-09-23T15:18:34.134929+0000","last_undegraded":"2023-09-23T15:18:34.134929+0000","last_fullsized":"2023-09-23T15:18:34.134929+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:11.358942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"248'6740","reported_seq":6773,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.759375+0000","last_change":"2023-09-23T15:16:44.269684+0000","last_active":"2023-09-23T15:18:44.759375+0000","last_peered":"2023-09-23T15:18:44.759375+0000","last_clean":"2023-09-23T15:18:44.759375+0000","last_became_active":"2023-09-23T15:16:44.269411+0000","last_became_peered":"2023-09-23T15:16:44.269411+0000","last_unstale":"2023-09-23T15:18:44.759375+0000","last_undegraded":"2023-09-23T15:18:44.759375+0000","last_fullsized":"2023-09-23T15:18:44.759375+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6740,"log_dups_size":0,"ondisk_log_size":6740,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:33.481012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314048512,"num_objects":4792,"num_object_clones":0,"num_object_copies":14376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6740,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134165+0000","last_change":"2023-09-23T15:16:49.361068+0000","last_active":"2023-09-23T15:18:34.134165+0000","last_peered":"2023-09-23T15:18:34.134165+0000","last_clean":"2023-09-23T15:18:34.134165+0000","last_became_active":"2023-09-23T15:16:46.276691+0000","last_became_peered":"2023-09-23T15:16:46.276691+0000","last_unstale":"2023-09-23T15:18:34.134165+0000","last_undegraded":"2023-09-23T15:18:34.134165+0000","last_fullsized":"2023-09-23T15:18:34.134165+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:16.914979+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":108,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128270+0000","last_change":"2023-09-23T15:11:34.167131+0000","last_active":"2023-09-23T15:18:34.128270+0000","last_peered":"2023-09-23T15:18:34.128270+0000","last_clean":"2023-09-23T15:18:34.128270+0000","last_became_active":"2023-09-23T15:11:34.158794+0000","last_became_peered":"2023-09-23T15:11:34.158794+0000","last_unstale":"2023-09-23T15:18:34.128270+0000","last_undegraded":"2023-09-23T15:18:34.128270+0000","last_fullsized":"2023-09-23T15:18:34.128270+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:01:13.468975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134439+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:18:34.134439+0000","last_peered":"2023-09-23T15:18:34.134439+0000","last_clean":"2023-09-23T15:18:34.134439+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:18:34.134439+0000","last_undegraded":"2023-09-23T15:18:34.134439+0000","last_fullsized":"2023-09-23T15:18:34.134439+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"247'6694","reported_seq":6725,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.010898+0000","last_change":"2023-09-23T15:16:44.271841+0000","last_active":"2023-09-23T15:18:44.010898+0000","last_peered":"2023-09-23T15:18:44.010898+0000","last_clean":"2023-09-23T15:18:44.010898+0000","last_became_active":"2023-09-23T15:16:44.271547+0000","last_became_peered":"2023-09-23T15:16:44.271547+0000","last_unstale":"2023-09-23T15:18:44.010898+0000","last_undegraded":"2023-09-23T15:18:44.010898+0000","last_fullsized":"2023-09-23T15:18:44.010898+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6694,"log_dups_size":0,"ondisk_log_size":6694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:03.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317456384,"num_objects":4844,"num_object_clones":0,"num_object_copies":14532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6694,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":545,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136040+0000","last_change":"2023-09-23T15:16:49.264186+0000","last_active":"2023-09-23T15:18:34.136040+0000","last_peered":"2023-09-23T15:18:34.136040+0000","last_clean":"2023-09-23T15:18:34.136040+0000","last_became_active":"2023-09-23T15:16:46.275415+0000","last_became_peered":"2023-09-23T15:16:46.275415+0000","last_unstale":"2023-09-23T15:18:34.136040+0000","last_undegraded":"2023-09-23T15:18:34.136040+0000","last_fullsized":"2023-09-23T15:18:34.136040+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.264096+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.264096+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:25.863800+0000","scrub_duration":0.0040937940000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128523+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:18:34.128523+0000","last_peered":"2023-09-23T15:18:34.128523+0000","last_clean":"2023-09-23T15:18:34.128523+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:18:34.128523+0000","last_undegraded":"2023-09-23T15:18:34.128523+0000","last_fullsized":"2023-09-23T15:18:34.128523+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":209,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.140534+0000","last_change":"2023-09-23T15:12:41.935235+0000","last_active":"2023-09-23T15:18:34.140534+0000","last_peered":"2023-09-23T15:18:34.140534+0000","last_clean":"2023-09-23T15:18:34.140534+0000","last_became_active":"2023-09-23T15:12:41.933990+0000","last_became_peered":"2023-09-23T15:12:41.933990+0000","last_unstale":"2023-09-23T15:18:34.140534+0000","last_undegraded":"2023-09-23T15:18:34.140534+0000","last_fullsized":"2023-09-23T15:18:34.140534+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:59.635541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"248'6804","reported_seq":6838,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.744221+0000","last_change":"2023-09-23T15:16:44.263685+0000","last_active":"2023-09-23T15:18:44.744221+0000","last_peered":"2023-09-23T15:18:44.744221+0000","last_clean":"2023-09-23T15:18:44.744221+0000","last_became_active":"2023-09-23T15:16:44.263391+0000","last_became_peered":"2023-09-23T15:16:44.263391+0000","last_unstale":"2023-09-23T15:18:44.744221+0000","last_undegraded":"2023-09-23T15:18:44.744221+0000","last_fullsized":"2023-09-23T15:18:44.744221+0000","mapping_epoch":223,"log_start":"228'100","ondisk_log_start":"228'100","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6704,"log_dups_size":0,"ondisk_log_size":6704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:33.410338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324272128,"num_objects":4948,"num_object_clones":0,"num_object_copies":14844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6804,"num_write_kb":376064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":546,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148236+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:18:34.148236+0000","last_peered":"2023-09-23T15:18:34.148236+0000","last_clean":"2023-09-23T15:18:34.148236+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:18:34.148236+0000","last_undegraded":"2023-09-23T15:18:34.148236+0000","last_fullsized":"2023-09-23T15:18:34.148236+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146837+0000","last_change":"2023-09-23T15:06:16.132545+0000","last_active":"2023-09-23T15:18:34.146837+0000","last_peered":"2023-09-23T15:18:34.146837+0000","last_clean":"2023-09-23T15:18:34.146837+0000","last_became_active":"2023-09-23T15:06:16.132244+0000","last_became_peered":"2023-09-23T15:06:16.132244+0000","last_unstale":"2023-09-23T15:18:34.146837+0000","last_undegraded":"2023-09-23T15:18:34.146837+0000","last_fullsized":"2023-09-23T15:18:34.146837+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:37.636489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134313+0000","last_change":"2023-09-23T15:06:16.137694+0000","last_active":"2023-09-23T15:18:34.134313+0000","last_peered":"2023-09-23T15:18:34.134313+0000","last_clean":"2023-09-23T15:18:34.134313+0000","last_became_active":"2023-09-23T15:06:16.137341+0000","last_became_peered":"2023-09-23T15:06:16.137341+0000","last_unstale":"2023-09-23T15:18:34.134313+0000","last_undegraded":"2023-09-23T15:18:34.134313+0000","last_fullsized":"2023-09-23T15:18:34.134313+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:08.729536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"247'6860","reported_seq":6893,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.003469+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:18:44.003469+0000","last_peered":"2023-09-23T15:18:44.003469+0000","last_clean":"2023-09-23T15:18:44.003469+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:18:44.003469+0000","last_undegraded":"2023-09-23T15:18:44.003469+0000","last_fullsized":"2023-09-23T15:18:44.003469+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6860,"log_dups_size":0,"ondisk_log_size":6860,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323092480,"num_objects":4930,"num_object_clones":0,"num_object_copies":14790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6860,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":552,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.127897+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:18:34.127897+0000","last_peered":"2023-09-23T15:18:34.127897+0000","last_clean":"2023-09-23T15:18:34.127897+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:18:34.127897+0000","last_undegraded":"2023-09-23T15:18:34.127897+0000","last_fullsized":"2023-09-23T15:18:34.127897+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":223,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134316+0000","last_change":"2023-09-23T15:12:41.863886+0000","last_active":"2023-09-23T15:18:34.134316+0000","last_peered":"2023-09-23T15:18:34.134316+0000","last_clean":"2023-09-23T15:18:34.134316+0000","last_became_active":"2023-09-23T15:12:41.863079+0000","last_became_peered":"2023-09-23T15:12:41.863079+0000","last_unstale":"2023-09-23T15:18:34.134316+0000","last_undegraded":"2023-09-23T15:18:34.134316+0000","last_fullsized":"2023-09-23T15:18:34.134316+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:02.651377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135072+0000","last_change":"2023-09-23T15:12:41.857103+0000","last_active":"2023-09-23T15:18:34.135072+0000","last_peered":"2023-09-23T15:18:34.135072+0000","last_clean":"2023-09-23T15:18:34.135072+0000","last_became_active":"2023-09-23T15:12:41.856803+0000","last_became_peered":"2023-09-23T15:12:41.856803+0000","last_unstale":"2023-09-23T15:18:34.135072+0000","last_undegraded":"2023-09-23T15:18:34.135072+0000","last_fullsized":"2023-09-23T15:18:34.135072+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:50.371251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"248'6908","reported_seq":6942,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.221411+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:18:44.221411+0000","last_peered":"2023-09-23T15:18:44.221411+0000","last_clean":"2023-09-23T15:18:44.221411+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:18:44.221411+0000","last_undegraded":"2023-09-23T15:18:44.221411+0000","last_fullsized":"2023-09-23T15:18:44.221411+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6908,"log_dups_size":0,"ondisk_log_size":6908,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333774872,"num_objects":5094,"num_object_clones":0,"num_object_copies":15282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5094,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6908,"num_write_kb":384001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":501,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135970+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:18:34.135970+0000","last_peered":"2023-09-23T15:18:34.135970+0000","last_clean":"2023-09-23T15:18:34.135970+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:18:34.135970+0000","last_undegraded":"2023-09-23T15:18:34.135970+0000","last_fullsized":"2023-09-23T15:18:34.135970+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":231,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134692+0000","last_change":"2023-09-23T15:12:42.900252+0000","last_active":"2023-09-23T15:18:34.134692+0000","last_peered":"2023-09-23T15:18:34.134692+0000","last_clean":"2023-09-23T15:18:34.134692+0000","last_became_active":"2023-09-23T15:12:42.899935+0000","last_became_peered":"2023-09-23T15:12:42.899935+0000","last_unstale":"2023-09-23T15:18:34.134692+0000","last_undegraded":"2023-09-23T15:18:34.134692+0000","last_fullsized":"2023-09-23T15:18:34.134692+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.508580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136334+0000","last_change":"2023-09-23T15:17:29.219853+0000","last_active":"2023-09-23T15:18:34.136334+0000","last_peered":"2023-09-23T15:18:34.136334+0000","last_clean":"2023-09-23T15:18:34.136334+0000","last_became_active":"2023-09-23T15:17:29.219486+0000","last_became_peered":"2023-09-23T15:17:29.219486+0000","last_unstale":"2023-09-23T15:18:34.136334+0000","last_undegraded":"2023-09-23T15:18:34.136334+0000","last_fullsized":"2023-09-23T15:18:34.136334+0000","mapping_epoch":233,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":234,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:27.120247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"248'6818","reported_seq":6851,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.203075+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:18:44.203075+0000","last_peered":"2023-09-23T15:18:44.203075+0000","last_clean":"2023-09-23T15:18:44.203075+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:18:44.203075+0000","last_undegraded":"2023-09-23T15:18:44.203075+0000","last_fullsized":"2023-09-23T15:18:44.203075+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6818,"log_dups_size":0,"ondisk_log_size":6818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329252864,"num_objects":5024,"num_object_clones":0,"num_object_copies":15072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6818,"num_write_kb":378944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":547,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.129129+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:18:34.129129+0000","last_peered":"2023-09-23T15:18:34.129129+0000","last_clean":"2023-09-23T15:18:34.129129+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:18:34.129129+0000","last_undegraded":"2023-09-23T15:18:34.129129+0000","last_fullsized":"2023-09-23T15:18:34.129129+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":241,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148323+0000","last_change":"2023-09-23T15:12:42.868743+0000","last_active":"2023-09-23T15:18:34.148323+0000","last_peered":"2023-09-23T15:18:34.148323+0000","last_clean":"2023-09-23T15:18:34.148323+0000","last_became_active":"2023-09-23T15:12:42.868395+0000","last_became_peered":"2023-09-23T15:12:42.868395+0000","last_unstale":"2023-09-23T15:18:34.148323+0000","last_undegraded":"2023-09-23T15:18:34.148323+0000","last_fullsized":"2023-09-23T15:18:34.148323+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:54.813960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":126,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135553+0000","last_change":"2023-09-23T15:12:41.923923+0000","last_active":"2023-09-23T15:18:34.135553+0000","last_peered":"2023-09-23T15:18:34.135553+0000","last_clean":"2023-09-23T15:18:34.135553+0000","last_became_active":"2023-09-23T15:12:41.923590+0000","last_became_peered":"2023-09-23T15:12:41.923590+0000","last_unstale":"2023-09-23T15:18:34.135553+0000","last_undegraded":"2023-09-23T15:18:34.135553+0000","last_fullsized":"2023-09-23T15:18:34.135553+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:21.409720+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"248'6821","reported_seq":6854,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.375256+0000","last_change":"2023-09-23T15:16:44.271607+0000","last_active":"2023-09-23T15:18:44.375256+0000","last_peered":"2023-09-23T15:18:44.375256+0000","last_clean":"2023-09-23T15:18:44.375256+0000","last_became_active":"2023-09-23T15:16:44.271334+0000","last_became_peered":"2023-09-23T15:16:44.271334+0000","last_unstale":"2023-09-23T15:18:44.375256+0000","last_undegraded":"2023-09-23T15:18:44.375256+0000","last_fullsized":"2023-09-23T15:18:44.375256+0000","mapping_epoch":223,"log_start":"229'800","ondisk_log_start":"229'800","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6021,"log_dups_size":0,"ondisk_log_size":6021,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:58.532783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328269824,"num_objects":5009,"num_object_clones":0,"num_object_copies":15027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6821,"num_write_kb":378560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134337+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:18:34.134337+0000","last_peered":"2023-09-23T15:18:34.134337+0000","last_clean":"2023-09-23T15:18:34.134337+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:18:34.134337+0000","last_undegraded":"2023-09-23T15:18:34.134337+0000","last_fullsized":"2023-09-23T15:18:34.134337+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":425,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134882+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:18:34.134882+0000","last_peered":"2023-09-23T15:18:34.134882+0000","last_clean":"2023-09-23T15:18:34.134882+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:18:34.134882+0000","last_undegraded":"2023-09-23T15:18:34.134882+0000","last_fullsized":"2023-09-23T15:18:34.134882+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":224,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128182+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:18:34.128182+0000","last_peered":"2023-09-23T15:18:34.128182+0000","last_clean":"2023-09-23T15:18:34.128182+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:18:34.128182+0000","last_undegraded":"2023-09-23T15:18:34.128182+0000","last_fullsized":"2023-09-23T15:18:34.128182+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":170,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.083797+0000","last_change":"2023-09-23T15:05:44.721967+0000","last_active":"2023-09-23T15:18:31.083797+0000","last_peered":"2023-09-23T15:18:31.083797+0000","last_clean":"2023-09-23T15:18:31.083797+0000","last_became_active":"2023-09-23T15:05:44.721668+0000","last_became_peered":"2023-09-23T15:05:44.721668+0000","last_unstale":"2023-09-23T15:18:31.083797+0000","last_undegraded":"2023-09-23T15:18:31.083797+0000","last_fullsized":"2023-09-23T15:18:31.083797+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:00.820480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"248'6670","reported_seq":6703,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.239685+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:18:44.239685+0000","last_peered":"2023-09-23T15:18:44.239685+0000","last_clean":"2023-09-23T15:18:44.239685+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:18:44.239685+0000","last_undegraded":"2023-09-23T15:18:44.239685+0000","last_fullsized":"2023-09-23T15:18:44.239685+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6670,"log_dups_size":0,"ondisk_log_size":6670,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320733184,"num_objects":4894,"num_object_clones":0,"num_object_copies":14682,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4894,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6670,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":545,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128278+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:18:34.128278+0000","last_peered":"2023-09-23T15:18:34.128278+0000","last_clean":"2023-09-23T15:18:34.128278+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:18:34.128278+0000","last_undegraded":"2023-09-23T15:18:34.128278+0000","last_fullsized":"2023-09-23T15:18:34.128278+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":766,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.129414+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:18:34.129414+0000","last_peered":"2023-09-23T15:18:34.129414+0000","last_clean":"2023-09-23T15:18:34.129414+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:18:34.129414+0000","last_undegraded":"2023-09-23T15:18:34.129414+0000","last_fullsized":"2023-09-23T15:18:34.129414+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":207,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134711+0000","last_change":"2023-09-23T15:04:40.697619+0000","last_active":"2023-09-23T15:18:34.134711+0000","last_peered":"2023-09-23T15:18:34.134711+0000","last_clean":"2023-09-23T15:18:34.134711+0000","last_became_active":"2023-09-23T15:04:40.697179+0000","last_became_peered":"2023-09-23T15:04:40.697179+0000","last_unstale":"2023-09-23T15:18:34.134711+0000","last_undegraded":"2023-09-23T15:18:34.134711+0000","last_fullsized":"2023-09-23T15:18:34.134711+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:39:19.150259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":141,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134590+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:18:34.134590+0000","last_peered":"2023-09-23T15:18:34.134590+0000","last_clean":"2023-09-23T15:18:34.134590+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:18:34.134590+0000","last_undegraded":"2023-09-23T15:18:34.134590+0000","last_fullsized":"2023-09-23T15:18:34.134590+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"248'6659","reported_seq":6693,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.777654+0000","last_change":"2023-09-23T15:16:44.269240+0000","last_active":"2023-09-23T15:18:44.777654+0000","last_peered":"2023-09-23T15:18:44.777654+0000","last_clean":"2023-09-23T15:18:44.777654+0000","last_became_active":"2023-09-23T15:16:44.268966+0000","last_became_peered":"2023-09-23T15:16:44.268966+0000","last_unstale":"2023-09-23T15:18:44.777654+0000","last_undegraded":"2023-09-23T15:18:44.777654+0000","last_fullsized":"2023-09-23T15:18:44.777654+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6659,"log_dups_size":0,"ondisk_log_size":6659,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:01.694002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315424768,"num_objects":4813,"num_object_clones":0,"num_object_copies":14439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6659,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":555,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131105+0000","last_change":"2023-09-23T15:16:46.276901+0000","last_active":"2023-09-23T15:18:34.131105+0000","last_peered":"2023-09-23T15:18:34.131105+0000","last_clean":"2023-09-23T15:18:34.131105+0000","last_became_active":"2023-09-23T15:16:46.276619+0000","last_became_peered":"2023-09-23T15:16:46.276619+0000","last_unstale":"2023-09-23T15:18:34.131105+0000","last_undegraded":"2023-09-23T15:18:34.131105+0000","last_fullsized":"2023-09-23T15:18:34.131105+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:53:27.244809+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":861,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.075358+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:18:31.075358+0000","last_peered":"2023-09-23T15:18:31.075358+0000","last_clean":"2023-09-23T15:18:31.075358+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:18:31.075358+0000","last_undegraded":"2023-09-23T15:18:31.075358+0000","last_fullsized":"2023-09-23T15:18:31.075358+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":122,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.090753+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:18:44.090753+0000","last_peered":"2023-09-23T15:18:44.090753+0000","last_clean":"2023-09-23T15:18:44.090753+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:18:44.090753+0000","last_undegraded":"2023-09-23T15:18:44.090753+0000","last_fullsized":"2023-09-23T15:18:44.090753+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":145,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135478+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:18:34.135478+0000","last_peered":"2023-09-23T15:18:34.135478+0000","last_clean":"2023-09-23T15:18:34.135478+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:18:34.135478+0000","last_undegraded":"2023-09-23T15:18:34.135478+0000","last_fullsized":"2023-09-23T15:18:34.135478+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"248'6833","reported_seq":6867,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.743613+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:18:44.743613+0000","last_peered":"2023-09-23T15:18:44.743613+0000","last_clean":"2023-09-23T15:18:44.743613+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:18:44.743613+0000","last_undegraded":"2023-09-23T15:18:44.743613+0000","last_fullsized":"2023-09-23T15:18:44.743613+0000","mapping_epoch":223,"log_start":"228'100","ondisk_log_start":"228'100","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6733,"log_dups_size":0,"ondisk_log_size":6733,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326041600,"num_objects":4975,"num_object_clones":0,"num_object_copies":14925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6833,"num_write_kb":377856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":535,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128322+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:18:34.128322+0000","last_peered":"2023-09-23T15:18:34.128322+0000","last_clean":"2023-09-23T15:18:34.128322+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:18:34.128322+0000","last_undegraded":"2023-09-23T15:18:34.128322+0000","last_fullsized":"2023-09-23T15:18:34.128322+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":538,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.130816+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:18:34.130816+0000","last_peered":"2023-09-23T15:18:34.130816+0000","last_clean":"2023-09-23T15:18:34.130816+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:18:34.130816+0000","last_undegraded":"2023-09-23T15:18:34.130816+0000","last_fullsized":"2023-09-23T15:18:34.130816+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":164,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.096382+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:18:44.096382+0000","last_peered":"2023-09-23T15:18:44.096382+0000","last_clean":"2023-09-23T15:18:44.096382+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:18:44.096382+0000","last_undegraded":"2023-09-23T15:18:44.096382+0000","last_fullsized":"2023-09-23T15:18:44.096382+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":198,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134907+0000","last_change":"2023-09-23T15:11:34.154704+0000","last_active":"2023-09-23T15:18:34.134907+0000","last_peered":"2023-09-23T15:18:34.134907+0000","last_clean":"2023-09-23T15:18:34.134907+0000","last_became_active":"2023-09-23T15:11:34.154288+0000","last_became_peered":"2023-09-23T15:11:34.154288+0000","last_unstale":"2023-09-23T15:18:34.134907+0000","last_undegraded":"2023-09-23T15:18:34.134907+0000","last_fullsized":"2023-09-23T15:18:34.134907+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:10.158242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"247'6664","reported_seq":6697,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:43.984046+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:18:43.984046+0000","last_peered":"2023-09-23T15:18:43.984046+0000","last_clean":"2023-09-23T15:18:43.984046+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:18:43.984046+0000","last_undegraded":"2023-09-23T15:18:43.984046+0000","last_fullsized":"2023-09-23T15:18:43.984046+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6664,"log_dups_size":0,"ondisk_log_size":6664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324272128,"num_objects":4948,"num_object_clones":0,"num_object_copies":14844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6664,"num_write_kb":371584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":501,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135878+0000","last_change":"2023-09-23T15:16:48.526093+0000","last_active":"2023-09-23T15:18:34.135878+0000","last_peered":"2023-09-23T15:18:34.135878+0000","last_clean":"2023-09-23T15:18:34.135878+0000","last_became_active":"2023-09-23T15:16:46.276681+0000","last_became_peered":"2023-09-23T15:16:46.276681+0000","last_unstale":"2023-09-23T15:18:34.135878+0000","last_undegraded":"2023-09-23T15:18:34.135878+0000","last_fullsized":"2023-09-23T15:18:34.135878+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:34.078052+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":333,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.090540+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:18:44.090540+0000","last_peered":"2023-09-23T15:18:44.090540+0000","last_clean":"2023-09-23T15:18:44.090540+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:18:44.090540+0000","last_undegraded":"2023-09-23T15:18:44.090540+0000","last_fullsized":"2023-09-23T15:18:44.090540+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":194,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.132107+0000","last_change":"2023-09-23T15:06:16.130853+0000","last_active":"2023-09-23T15:18:34.132107+0000","last_peered":"2023-09-23T15:18:34.132107+0000","last_clean":"2023-09-23T15:18:34.132107+0000","last_became_active":"2023-09-23T15:06:16.130570+0000","last_became_peered":"2023-09-23T15:06:16.130570+0000","last_unstale":"2023-09-23T15:18:34.132107+0000","last_undegraded":"2023-09-23T15:18:34.132107+0000","last_fullsized":"2023-09-23T15:18:34.132107+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:43.380492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":224,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131198+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:18:34.131198+0000","last_peered":"2023-09-23T15:18:34.131198+0000","last_clean":"2023-09-23T15:18:34.131198+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:18:34.131198+0000","last_undegraded":"2023-09-23T15:18:34.131198+0000","last_fullsized":"2023-09-23T15:18:34.131198+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"248'6786","reported_seq":6819,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:44.674244+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:18:44.674244+0000","last_peered":"2023-09-23T15:18:44.674244+0000","last_clean":"2023-09-23T15:18:44.674244+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:18:44.674244+0000","last_undegraded":"2023-09-23T15:18:44.674244+0000","last_fullsized":"2023-09-23T15:18:44.674244+0000","mapping_epoch":223,"log_start":"0'0","ondisk_log_start":"0'0","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6786,"log_dups_size":0,"ondisk_log_size":6786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317456384,"num_objects":4844,"num_object_clones":0,"num_object_copies":14532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6786,"num_write_kb":372160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":470,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134035+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:18:34.134035+0000","last_peered":"2023-09-23T15:18:34.134035+0000","last_clean":"2023-09-23T15:18:34.134035+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:18:34.134035+0000","last_undegraded":"2023-09-23T15:18:34.134035+0000","last_fullsized":"2023-09-23T15:18:34.134035+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":870,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.136015+0000","last_change":"2023-09-23T15:16:38.973875+0000","last_active":"2023-09-23T15:18:34.136015+0000","last_peered":"2023-09-23T15:18:34.136015+0000","last_clean":"2023-09-23T15:18:34.136015+0000","last_became_active":"2023-09-23T15:16:38.973269+0000","last_became_peered":"2023-09-23T15:16:38.973269+0000","last_unstale":"2023-09-23T15:18:34.136015+0000","last_undegraded":"2023-09-23T15:18:34.136015+0000","last_fullsized":"2023-09-23T15:18:34.136015+0000","mapping_epoch":220,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:17:00.522554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":121,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134765+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:18:34.134765+0000","last_peered":"2023-09-23T15:18:34.134765+0000","last_clean":"2023-09-23T15:18:34.134765+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:18:34.134765+0000","last_undegraded":"2023-09-23T15:18:34.134765+0000","last_fullsized":"2023-09-23T15:18:34.134765+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":199,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.140790+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:18:34.140790+0000","last_peered":"2023-09-23T15:18:34.140790+0000","last_clean":"2023-09-23T15:18:34.140790+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:18:34.140790+0000","last_undegraded":"2023-09-23T15:18:34.140790+0000","last_fullsized":"2023-09-23T15:18:34.140790+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"248'6908","reported_seq":6942,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:45.120647+0000","last_change":"2023-09-23T15:16:44.260057+0000","last_active":"2023-09-23T15:18:45.120647+0000","last_peered":"2023-09-23T15:18:45.120647+0000","last_clean":"2023-09-23T15:18:45.120647+0000","last_became_active":"2023-09-23T15:16:44.259815+0000","last_became_peered":"2023-09-23T15:16:44.259815+0000","last_unstale":"2023-09-23T15:18:45.120647+0000","last_undegraded":"2023-09-23T15:18:45.120647+0000","last_fullsized":"2023-09-23T15:18:45.120647+0000","mapping_epoch":223,"log_start":"230'1400","ondisk_log_start":"230'1400","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5508,"log_dups_size":0,"ondisk_log_size":5508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:50.147826+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330170368,"num_objects":5038,"num_object_clones":0,"num_object_copies":15114,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5038,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6908,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":545,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134928+0000","last_change":"2023-09-23T15:16:52.317545+0000","last_active":"2023-09-23T15:18:34.134928+0000","last_peered":"2023-09-23T15:18:34.134928+0000","last_clean":"2023-09-23T15:18:34.134928+0000","last_became_active":"2023-09-23T15:16:46.275975+0000","last_became_peered":"2023-09-23T15:16:46.275975+0000","last_unstale":"2023-09-23T15:18:34.134928+0000","last_undegraded":"2023-09-23T15:18:34.134928+0000","last_fullsized":"2023-09-23T15:18:34.134928+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:52.317424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:52.317424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:29.771978+0000","scrub_duration":0.001669299,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":225,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146171+0000","last_change":"2023-09-23T15:11:34.161181+0000","last_active":"2023-09-23T15:18:34.146171+0000","last_peered":"2023-09-23T15:18:34.146171+0000","last_clean":"2023-09-23T15:18:34.146171+0000","last_became_active":"2023-09-23T15:11:34.153285+0000","last_became_peered":"2023-09-23T15:11:34.153285+0000","last_unstale":"2023-09-23T15:18:34.146171+0000","last_undegraded":"2023-09-23T15:18:34.146171+0000","last_fullsized":"2023-09-23T15:18:34.146171+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:49:38.943665+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":709,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135178+0000","last_change":"2023-09-23T15:12:41.924719+0000","last_active":"2023-09-23T15:18:34.135178+0000","last_peered":"2023-09-23T15:18:34.135178+0000","last_clean":"2023-09-23T15:18:34.135178+0000","last_became_active":"2023-09-23T15:12:41.924021+0000","last_became_peered":"2023-09-23T15:12:41.924021+0000","last_unstale":"2023-09-23T15:18:34.135178+0000","last_undegraded":"2023-09-23T15:18:34.135178+0000","last_fullsized":"2023-09-23T15:18:34.135178+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:11.053760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":230,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148564+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:18:34.148564+0000","last_peered":"2023-09-23T15:18:34.148564+0000","last_clean":"2023-09-23T15:18:34.148564+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:18:34.148564+0000","last_undegraded":"2023-09-23T15:18:34.148564+0000","last_fullsized":"2023-09-23T15:18:34.148564+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"248'6756","reported_seq":6790,"reported_epoch":248,"state":"active+clean","last_fresh":"2023-09-23T15:18:45.108308+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:18:45.108308+0000","last_peered":"2023-09-23T15:18:45.108308+0000","last_clean":"2023-09-23T15:18:45.108308+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:18:45.108308+0000","last_undegraded":"2023-09-23T15:18:45.108308+0000","last_fullsized":"2023-09-23T15:18:45.108308+0000","mapping_epoch":223,"log_start":"229'1300","ondisk_log_start":"229'1300","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5456,"log_dups_size":0,"ondisk_log_size":5456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324272128,"num_objects":4948,"num_object_clones":0,"num_object_copies":14844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6756,"num_write_kb":374528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":546,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146067+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:18:34.146067+0000","last_peered":"2023-09-23T15:18:34.146067+0000","last_clean":"2023-09-23T15:18:34.146067+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:18:34.146067+0000","last_undegraded":"2023-09-23T15:18:34.146067+0000","last_fullsized":"2023-09-23T15:18:34.146067+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":724,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134254+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:18:34.134254+0000","last_peered":"2023-09-23T15:18:34.134254+0000","last_clean":"2023-09-23T15:18:34.134254+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:18:34.134254+0000","last_undegraded":"2023-09-23T15:18:34.134254+0000","last_fullsized":"2023-09-23T15:18:34.134254+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":212,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131248+0000","last_change":"2023-09-23T15:12:41.935481+0000","last_active":"2023-09-23T15:18:34.131248+0000","last_peered":"2023-09-23T15:18:34.131248+0000","last_clean":"2023-09-23T15:18:34.131248+0000","last_became_active":"2023-09-23T15:12:41.935097+0000","last_became_peered":"2023-09-23T15:12:41.935097+0000","last_unstale":"2023-09-23T15:18:34.131248+0000","last_undegraded":"2023-09-23T15:18:34.131248+0000","last_fullsized":"2023-09-23T15:18:34.131248+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:03:09.949209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":189,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134859+0000","last_change":"2023-09-23T14:56:31.360757+0000","last_active":"2023-09-23T15:18:34.134859+0000","last_peered":"2023-09-23T15:18:34.134859+0000","last_clean":"2023-09-23T15:18:34.134859+0000","last_became_active":"2023-09-23T14:56:31.360448+0000","last_became_peered":"2023-09-23T14:56:31.360448+0000","last_unstale":"2023-09-23T15:18:34.134859+0000","last_undegraded":"2023-09-23T15:18:34.134859+0000","last_fullsized":"2023-09-23T15:18:34.134859+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:14.904140+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":201,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.140712+0000","last_change":"2023-09-23T15:12:43.161934+0000","last_active":"2023-09-23T15:18:34.140712+0000","last_peered":"2023-09-23T15:18:34.140712+0000","last_clean":"2023-09-23T15:18:34.140712+0000","last_became_active":"2023-09-23T15:12:43.161560+0000","last_became_peered":"2023-09-23T15:12:43.161560+0000","last_unstale":"2023-09-23T15:18:34.140712+0000","last_undegraded":"2023-09-23T15:18:34.140712+0000","last_fullsized":"2023-09-23T15:18:34.140712+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:08.217557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":248,"reported_epoch":246,"state":"active+clean","last_fresh":"2023-09-23T15:18:31.085847+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:18:31.085847+0000","last_peered":"2023-09-23T15:18:31.085847+0000","last_clean":"2023-09-23T15:18:31.085847+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:18:31.085847+0000","last_undegraded":"2023-09-23T15:18:31.085847+0000","last_fullsized":"2023-09-23T15:18:31.085847+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":226,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128603+0000","last_change":"2023-09-23T15:12:41.853520+0000","last_active":"2023-09-23T15:18:34.128603+0000","last_peered":"2023-09-23T15:18:34.128603+0000","last_clean":"2023-09-23T15:18:34.128603+0000","last_became_active":"2023-09-23T15:12:41.853162+0000","last_became_peered":"2023-09-23T15:12:41.853162+0000","last_unstale":"2023-09-23T15:18:34.128603+0000","last_undegraded":"2023-09-23T15:18:34.128603+0000","last_fullsized":"2023-09-23T15:18:34.128603+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:30.139253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134895+0000","last_change":"2023-09-23T15:12:41.860693+0000","last_active":"2023-09-23T15:18:34.134895+0000","last_peered":"2023-09-23T15:18:34.134895+0000","last_clean":"2023-09-23T15:18:34.134895+0000","last_became_active":"2023-09-23T15:12:41.860228+0000","last_became_peered":"2023-09-23T15:12:41.860228+0000","last_unstale":"2023-09-23T15:18:34.134895+0000","last_undegraded":"2023-09-23T15:18:34.134895+0000","last_fullsized":"2023-09-23T15:18:34.134895+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:54.608606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":192,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.128088+0000","last_change":"2023-09-23T14:56:31.348512+0000","last_active":"2023-09-23T15:18:34.128088+0000","last_peered":"2023-09-23T15:18:34.128088+0000","last_clean":"2023-09-23T15:18:34.128088+0000","last_became_active":"2023-09-23T14:56:31.348270+0000","last_became_peered":"2023-09-23T14:56:31.348270+0000","last_unstale":"2023-09-23T15:18:34.128088+0000","last_undegraded":"2023-09-23T15:18:34.128088+0000","last_fullsized":"2023-09-23T15:18:34.128088+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:19:32.764591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":223,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.135841+0000","last_change":"2023-09-23T15:06:16.131672+0000","last_active":"2023-09-23T15:18:34.135841+0000","last_peered":"2023-09-23T15:18:34.135841+0000","last_clean":"2023-09-23T15:18:34.135841+0000","last_became_active":"2023-09-23T15:06:16.130831+0000","last_became_peered":"2023-09-23T15:06:16.130831+0000","last_unstale":"2023-09-23T15:18:34.135841+0000","last_undegraded":"2023-09-23T15:18:34.135841+0000","last_fullsized":"2023-09-23T15:18:34.135841+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:46:57.602767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":221,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.146475+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:18:34.146475+0000","last_peered":"2023-09-23T15:18:34.146475+0000","last_clean":"2023-09-23T15:18:34.146475+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:18:34.146475+0000","last_undegraded":"2023-09-23T15:18:34.146475+0000","last_fullsized":"2023-09-23T15:18:34.146475+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":235,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.148363+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:18:34.148363+0000","last_peered":"2023-09-23T15:18:34.148363+0000","last_clean":"2023-09-23T15:18:34.148363+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:18:34.148363+0000","last_undegraded":"2023-09-23T15:18:34.148363+0000","last_fullsized":"2023-09-23T15:18:34.148363+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":197,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.131466+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:18:34.131466+0000","last_peered":"2023-09-23T15:18:34.131466+0000","last_clean":"2023-09-23T15:18:34.131466+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:18:34.131466+0000","last_undegraded":"2023-09-23T15:18:34.131466+0000","last_fullsized":"2023-09-23T15:18:34.131466+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":216,"reported_epoch":247,"state":"active+clean","last_fresh":"2023-09-23T15:18:34.134043+0000","last_change":"2023-09-23T15:05:44.716758+0000","last_active":"2023-09-23T15:18:34.134043+0000","last_peered":"2023-09-23T15:18:34.134043+0000","last_clean":"2023-09-23T15:18:34.134043+0000","last_became_active":"2023-09-23T15:05:44.716326+0000","last_became_peered":"2023-09-23T15:05:44.716326+0000","last_unstale":"2023-09-23T15:18:34.134043+0000","last_undegraded":"2023-09-23T15:18:34.134043+0000","last_fullsized":"2023-09-23T15:18:34.134043+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:26.591613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":5172822040,"num_objects":78932,"num_object_clones":0,"num_object_copies":236796,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78932,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":108804,"num_write_kb":6007489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15491346432,"data_stored":15491334216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102504,"ondisk_log_size":102504,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4950,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201568,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463204,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1815632,"kb_used_data":1611144,"kb_used_omap":0,"kb_used_meta":204480,"kb_avail":91929520,"statfs":{"total":95995035648,"available":94135828480,"internally_reserved":0,"allocated":1649811456,"data_stored":1649064007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":209387520},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":1.4870000000000001,"5min":1.407,"15min":2.7879999999999998},"min":{"1min":0.22800000000000001,"5min":0.20699999999999999,"15min":0.19800000000000001},"max":{"1min":7.5099999999999998,"5min":12.926,"15min":108.39400000000001},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.5780000000000001,"5min":2.016,"15min":3.0659999999999998},"min":{"1min":0.24099999999999999,"5min":0.23400000000000001,"15min":0.223},"max":{"1min":7.7279999999999998,"5min":12.634,"15min":108.452},"last":0.53600000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:17:49 2023","interfaces":[{"interface":"back","average":{"1min":4.6970000000000001,"5min":3.214,"15min":3.3180000000000001},"min":{"1min":0.27600000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.236999999999995,"5min":86.236999999999995,"15min":86.236999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":5.1609999999999996,"5min":4.0999999999999996,"15min":3.7709999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":86.158000000000001,"5min":86.158000000000001,"15min":86.158000000000001},"last":0.41699999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":6.157,"5min":6.1630000000000003,"15min":6.5309999999999997},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.255},"max":{"1min":86.569000000000003,"5min":112.352,"15min":180.374},"last":0.49199999999999999},{"interface":"front","average":{"1min":5.3170000000000002,"5min":5.8200000000000003,"15min":6.4199999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":88.269000000000005,"5min":117.667,"15min":181.685},"last":5.9640000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:17:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":7.2699999999999996,"15min":3.2280000000000002},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":6.7549999999999999,"5min":102.254,"15min":102.254},"last":0.92900000000000005},{"interface":"front","average":{"1min":1.7649999999999999,"5min":7.4059999999999997,"15min":3.2410000000000001},"min":{"1min":0.42399999999999999,"5min":0.374,"15min":0.29799999999999999},"max":{"1min":8.1479999999999997,"5min":98.941999999999993,"15min":98.941999999999993},"last":0.86499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.41799999999999998,"5min":0.55700000000000005,"15min":1.0309999999999999},"min":{"1min":0.28899999999999998,"5min":0.248,"15min":0.23000000000000001},"max":{"1min":0.70199999999999996,"5min":6.5119999999999996,"15min":36.290999999999997},"last":0.37},{"interface":"front","average":{"1min":0.504,"5min":0.60899999999999999,"15min":1.121},"min":{"1min":0.30399999999999999,"5min":0.27600000000000002,"15min":0.26300000000000001},"max":{"1min":1.444,"5min":4.7590000000000003,"15min":36.758000000000003},"last":0.45400000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.68999999999999995,"15min":1.411},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.28100000000000003},"max":{"1min":0.91300000000000003,"5min":5.5880000000000001,"15min":131.90600000000001},"last":0.435},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.64600000000000002,"15min":1.4550000000000001},"min":{"1min":0.34499999999999997,"5min":0.317,"15min":0.29299999999999998},"max":{"1min":2.0070000000000001,"5min":4.7859999999999996,"15min":131.61000000000001},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:17:41 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.81999999999999995,"15min":1.0309999999999999},"min":{"1min":0.36799999999999999,"5min":0.32600000000000001,"15min":0.28999999999999998},"max":{"1min":2.306,"5min":5.125,"15min":17},"last":0.504},{"interface":"front","average":{"1min":0.65100000000000002,"5min":0.74399999999999999,"15min":0.98699999999999999},"min":{"1min":0.34999999999999998,"5min":0.33500000000000002,"15min":0.25800000000000001},"max":{"1min":1.327,"5min":3.754,"15min":7.7809999999999997},"last":0.51900000000000002}]}]},{"osd":1,"up_from":183,"seq":785979015242,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2435116,"kb_used_data":2177444,"kb_used_omap":6,"kb_used_meta":257657,"kb_avail":91310036,"statfs":{"total":95995035648,"available":93501476864,"internally_reserved":0,"allocated":2229702656,"data_stored":2228593228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":263840843},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":1.784,"5min":2.7719999999999998,"15min":3.46},"min":{"1min":0.26000000000000001,"5min":0.19900000000000001,"15min":0.19900000000000001},"max":{"1min":9.4619999999999997,"5min":16.626000000000001,"15min":16.626000000000001},"last":6.4589999999999996},{"interface":"front","average":{"1min":2.2229999999999999,"5min":3.968,"15min":4.3959999999999999},"min":{"1min":0.28199999999999997,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":9.3260000000000005,"5min":19.044,"15min":19.044},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":4.0620000000000003,"15min":4.1059999999999999},"min":{"1min":0.33400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":8.4900000000000002,"5min":33.880000000000003,"15min":33.880000000000003},"last":30.898},{"interface":"front","average":{"1min":1.627,"5min":3.2130000000000001,"15min":3.8860000000000001},"min":{"1min":0.28699999999999998,"5min":0.253,"15min":0.253},"max":{"1min":8.0589999999999993,"5min":16.882000000000001,"15min":16.882000000000001},"last":6.4320000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.105,"5min":7.5949999999999998,"15min":9.7289999999999992},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":8.1240000000000006,"5min":211.405,"15min":211.405},"last":6.2919999999999998},{"interface":"front","average":{"1min":2.363,"5min":7.8319999999999999,"15min":10.988},"min":{"1min":0.378,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.2850000000000001,"5min":210.988,"15min":210.988},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":8.0980000000000008,"5min":3.8279999999999998,"15min":3.073},"min":{"1min":0.40400000000000003,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":68.805999999999997,"5min":68.805999999999997,"15min":68.805999999999997},"last":10.381},{"interface":"front","average":{"1min":6.9299999999999997,"5min":3.5619999999999998,"15min":3.5590000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":68.861000000000004,"5min":68.861000000000004,"15min":68.861000000000004},"last":6.3550000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":2.907,"15min":3.488},"min":{"1min":0.49299999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":11,"5min":16.166,"15min":16.166},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.1539999999999999,"5min":3.1080000000000001,"15min":2.9670000000000001},"min":{"1min":0.44600000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":10.776,"5min":11.487,"15min":11.487},"last":10.266}]},{"osd":6,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":8.5670000000000002,"5min":4.0439999999999996,"15min":3.8879999999999999},"min":{"1min":0.86899999999999999,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":70.602000000000004,"5min":70.602000000000004,"15min":70.602000000000004},"last":0.59199999999999997},{"interface":"front","average":{"1min":9.0879999999999992,"5min":4.3200000000000003,"15min":3.2719999999999998},"min":{"1min":0.47099999999999997,"5min":0.377,"15min":0.377},"max":{"1min":72.239999999999995,"5min":72.239999999999995,"15min":72.239999999999995},"last":10.321999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:17:51 2023","interfaces":[{"interface":"back","average":{"1min":2.633,"5min":3.0049999999999999,"15min":3.1059999999999999},"min":{"1min":0.496,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":8.3499999999999996,"5min":11.561,"15min":11.561},"last":10.352},{"interface":"front","average":{"1min":1.234,"5min":2.516,"15min":3.0739999999999998},"min":{"1min":0.42699999999999999,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":8.0329999999999995,"5min":16.841000000000001,"15min":16.841000000000001},"last":6.3840000000000003}]}]},{"osd":0,"up_from":95,"seq":408021893291,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2914108,"kb_used_data":2534388,"kb_used_omap":8,"kb_used_meta":379703,"kb_avail":90831044,"statfs":{"total":95995035648,"available":93010989056,"internally_reserved":0,"allocated":2595213312,"data_stored":2594204692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":388816468},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:17:48 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":2.903,"15min":3.8290000000000002},"min":{"1min":0.18099999999999999,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":44.075000000000003,"5min":44.075000000000003,"15min":44.075000000000003},"last":1.986},{"interface":"front","average":{"1min":4.6609999999999996,"5min":3.1850000000000001,"15min":4.2009999999999996},"min":{"1min":0.222,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":43.593000000000004,"5min":43.593000000000004,"15min":43.593000000000004},"last":8.3580000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":5.8710000000000004,"15min":3.7519999999999998},"min":{"1min":0.33600000000000002,"5min":0.24099999999999999,"15min":0.17000000000000001},"max":{"1min":15.209,"5min":93.873999999999995,"15min":122.20099999999999},"last":2.121},{"interface":"front","average":{"1min":2.3340000000000001,"5min":5.3760000000000003,"15min":3.3399999999999999},"min":{"1min":0.24099999999999999,"5min":0.23499999999999999,"15min":0.18099999999999999},"max":{"1min":23.719999999999999,"5min":94.652000000000001,"15min":122.071},"last":2.3100000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:18:35 2023","interfaces":[{"interface":"back","average":{"1min":3.2989999999999999,"5min":6.9909999999999997,"15min":4.2270000000000003},"min":{"1min":0.32900000000000001,"5min":0.26200000000000001,"15min":0.20000000000000001},"max":{"1min":8.6110000000000007,"5min":146.315,"15min":146.315},"last":2.5830000000000002},{"interface":"front","average":{"1min":6.1390000000000002,"5min":8.8960000000000008,"15min":4.7149999999999999},"min":{"1min":0.42999999999999999,"5min":0.307,"15min":0.252},"max":{"1min":64.441000000000003,"5min":145.517,"15min":145.517},"last":2.3809999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":2.399,"15min":1.776},"min":{"1min":0.34200000000000003,"5min":0.32400000000000001,"15min":0.28100000000000003},"max":{"1min":7.0199999999999996,"5min":31.341999999999999,"15min":31.341999999999999},"last":2.1659999999999999},{"interface":"front","average":{"1min":1.454,"5min":2.2450000000000001,"15min":1.8129999999999999},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.27000000000000002},"max":{"1min":8.0559999999999992,"5min":31.059999999999999,"15min":99.168999999999997},"last":2.032}]},{"osd":5,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.968,"15min":1.8129999999999999},"min":{"1min":0.31900000000000001,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.2949999999999999,"5min":31.25,"15min":109.369},"last":2.181},{"interface":"front","average":{"1min":1.825,"5min":2.1019999999999999,"15min":1.972},"min":{"1min":0.41899999999999998,"5min":0.36399999999999999,"15min":0.28599999999999998},"max":{"1min":6.9779999999999998,"5min":31.295000000000002,"15min":109.387},"last":2.6259999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.5269999999999999,"5min":2.0419999999999998,"15min":1.871},"min":{"1min":0.42799999999999999,"5min":0.35899999999999999,"15min":0.309},"max":{"1min":8.1050000000000004,"5min":16.893999999999998,"15min":99.114999999999995},"last":2.0830000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.994,"15min":1.7390000000000001},"min":{"1min":0.35899999999999999,"5min":0.313,"15min":0.313},"max":{"1min":4.2489999999999997,"5min":31.431000000000001,"15min":98.921999999999997},"last":2.2810000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":2.1669999999999998,"15min":1.861},"min":{"1min":0.41599999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":4.3280000000000003,"5min":31.472000000000001,"15min":99.549999999999997},"last":2.222},{"interface":"front","average":{"1min":1.871,"5min":2.2759999999999998,"15min":1.6819999999999999},"min":{"1min":0.498,"5min":0.46700000000000003,"15min":0.36399999999999999},"max":{"1min":7.0579999999999998,"5min":31.225999999999999,"15min":31.225999999999999},"last":2.1960000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566238,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2641768,"kb_used_data":2232736,"kb_used_omap":11,"kb_used_meta":409012,"kb_avail":91103384,"statfs":{"total":95995035648,"available":93289865216,"internally_reserved":0,"allocated":2286321664,"data_stored":2285327657,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":418829072},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":3.2530000000000001,"5min":5.2169999999999996,"15min":2.492},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":15.15,"5min":87.132999999999996,"15min":87.132999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":2.3370000000000002,"5min":4.1890000000000001,"15min":2.097},"min":{"1min":0.246,"5min":0.246,"15min":0.23100000000000001},"max":{"1min":14.82,"5min":84.820999999999998,"15min":84.820999999999998},"last":0.36799999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":4.883,"15min":4.9870000000000001},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":5.625,"5min":84.641000000000005,"15min":84.641000000000005},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.1560000000000001,"5min":3.9470000000000001,"15min":3.0150000000000001},"min":{"1min":0.313,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.742,"5min":84.751999999999995,"15min":84.751999999999995},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 15:18:34 2023","interfaces":[{"interface":"back","average":{"1min":5.6120000000000001,"5min":7.8700000000000001,"15min":3.5990000000000002},"min":{"1min":0.24199999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":28.617999999999999,"5min":87.108000000000004,"15min":87.108000000000004},"last":1.1599999999999999},{"interface":"front","average":{"1min":6.5339999999999998,"5min":7.5490000000000004,"15min":3.5350000000000001},"min":{"1min":0.38300000000000001,"5min":0.252,"15min":0.20300000000000001},"max":{"1min":30.742999999999999,"5min":86.158000000000001,"15min":86.158000000000001},"last":0.98199999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":2.8450000000000002,"15min":1.982},"min":{"1min":0.32700000000000001,"5min":0.30599999999999999,"15min":0.27200000000000002},"max":{"1min":5.0330000000000004,"5min":84.608000000000004,"15min":84.608000000000004},"last":0.5},{"interface":"front","average":{"1min":1.8360000000000001,"5min":3.7610000000000001,"15min":2.6030000000000002},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":5.7080000000000002,"5min":87.019999999999996,"15min":87.019999999999996},"last":1.083}]},{"osd":5,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":3.427,"15min":2.2549999999999999},"min":{"1min":0.31900000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.1269999999999998,"5min":84.498000000000005,"15min":84.498000000000005},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.222,"5min":2.984,"15min":2.0720000000000001},"min":{"1min":0.44500000000000001,"5min":0.30399999999999999,"15min":0.30199999999999999},"max":{"1min":5.0949999999999998,"5min":84.664000000000001,"15min":84.664000000000001},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.242,"5min":3.0979999999999999,"15min":2.1890000000000001},"min":{"1min":0.47399999999999998,"5min":0.35999999999999999,"15min":0.32100000000000001},"max":{"1min":5.0659999999999998,"5min":84.715999999999994,"15min":84.715999999999994},"last":0.59299999999999997},{"interface":"front","average":{"1min":2.121,"5min":3.8879999999999999,"15min":2.3279999999999998},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":6.9779999999999998,"5min":87.055000000000007,"15min":87.055000000000007},"last":1.1299999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:18:12 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":3.52,"15min":2.1099999999999999},"min":{"1min":0.42399999999999999,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":5.4089999999999998,"5min":84.596000000000004,"15min":84.596000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.288,"5min":3.0310000000000001,"15min":1.956},"min":{"1min":0.49199999999999999,"5min":0.41299999999999998,"15min":0.34200000000000003},"max":{"1min":5.1550000000000002,"5min":84.783000000000001,"15min":84.783000000000001},"last":0.61899999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402712,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3570652,"kb_used_data":3166676,"kb_used_omap":3,"kb_used_meta":403772,"kb_avail":90174500,"statfs":{"total":95995035648,"available":92338688000,"internally_reserved":0,"allocated":3242676224,"data_stored":3241695167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":413462697},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":5.375,"5min":3.6459999999999999,"15min":2.6080000000000001},"min":{"1min":0.24299999999999999,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":33.220999999999997,"5min":33.220999999999997,"15min":79.611000000000004},"last":3.4420000000000002},{"interface":"front","average":{"1min":6.1109999999999998,"5min":3.923,"15min":2.7730000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.17599999999999999},"max":{"1min":33.084000000000003,"5min":33.084000000000003,"15min":79.656000000000006},"last":2.161}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":8.0640000000000001,"5min":5.2300000000000004,"15min":4.7999999999999998},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":119.343,"5min":119.343,"15min":119.343},"last":1.9890000000000001},{"interface":"front","average":{"1min":8.3369999999999997,"5min":5.0250000000000004,"15min":4.9050000000000002},"min":{"1min":0.249,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":119.505,"5min":119.505,"15min":119.505},"last":3.355}]},{"osd":2,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":2.9449999999999998,"5min":4.4699999999999998,"15min":4.2350000000000003},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.22},"max":{"1min":9.1609999999999996,"5min":61.732999999999997,"15min":109.077},"last":3.7050000000000001},{"interface":"front","average":{"1min":8.9890000000000008,"5min":5.6340000000000003,"15min":4.8959999999999999},"min":{"1min":0.28499999999999998,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":119.449,"5min":119.449,"15min":119.72799999999999},"last":2.8860000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":2.1669999999999998,"5min":2.8580000000000001,"15min":1.7789999999999999},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.27600000000000002},"max":{"1min":6.0019999999999998,"5min":54.051000000000002,"15min":54.051000000000002},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.909,"5min":2.8300000000000001,"15min":1.891},"min":{"1min":0.39500000000000002,"5min":0.32300000000000001,"15min":0.26400000000000001},"max":{"1min":5.452,"5min":52.228999999999999,"15min":52.228999999999999},"last":2.0230000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.375,"15min":1.921},"min":{"1min":0.40300000000000002,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":6.0350000000000001,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.71399999999999997},{"interface":"front","average":{"1min":1.891,"5min":2.4390000000000001,"15min":2.0840000000000001},"min":{"1min":0.434,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":5.5140000000000002,"5min":12.667999999999999,"15min":56.981999999999999},"last":2.0590000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":2.3279999999999998,"5min":3.202,"15min":2.0710000000000002},"min":{"1min":0.42899999999999999,"5min":0.32300000000000001,"15min":0.32200000000000001},"max":{"1min":6.0590000000000002,"5min":78.102000000000004,"15min":78.102000000000004},"last":0.746},{"interface":"front","average":{"1min":1.9139999999999999,"5min":3.1629999999999998,"15min":2.2240000000000002},"min":{"1min":0.42399999999999999,"5min":0.35599999999999998,"15min":0.32300000000000001},"max":{"1min":5.4880000000000004,"5min":78.028000000000006,"15min":78.028000000000006},"last":2.149}]},{"osd":7,"last update":"Sat Sep 23 15:18:41 2023","interfaces":[{"interface":"back","average":{"1min":1.9399999999999999,"5min":2.4910000000000001,"15min":2.0139999999999998},"min":{"1min":0.45100000000000001,"5min":0.42599999999999999,"15min":0.28299999999999997},"max":{"1min":5.5410000000000004,"5min":12.701000000000001,"15min":14.885999999999999},"last":2.1739999999999999},{"interface":"front","average":{"1min":1.8140000000000001,"5min":2.548,"15min":1.7010000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":9.0079999999999991,"5min":18.696000000000002,"15min":18.696000000000002},"last":1.8109999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986477,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2134548,"kb_used_data":1894348,"kb_used_omap":0,"kb_used_meta":240192,"kb_avail":91610604,"statfs":{"total":95995035648,"available":93809258496,"internally_reserved":0,"allocated":1939812352,"data_stored":1939068966,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":245956608},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:03 2023","interfaces":[{"interface":"back","average":{"1min":2.198,"5min":2.1520000000000001,"15min":2.7749999999999999},"min":{"1min":0.22700000000000001,"5min":0.219,"15min":0.219},"max":{"1min":14.212999999999999,"5min":65.988,"15min":122.69199999999999},"last":2.4009999999999998},{"interface":"front","average":{"1min":2.274,"5min":2.4910000000000001,"15min":2.9159999999999999},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.0060000000000002,"5min":65.396000000000001,"15min":122.745},"last":1.083}]},{"osd":1,"last update":"Sat Sep 23 15:17:52 2023","interfaces":[{"interface":"back","average":{"1min":1.419,"5min":2.3959999999999999,"15min":3.831},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.6299999999999999,"5min":53.832000000000001,"15min":53.832000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.2250000000000001,"5min":3.0110000000000001,"15min":4.3879999999999999},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":9.4890000000000008,"5min":53.078000000000003,"15min":53.078000000000003},"last":0.51300000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:18:03 2023","interfaces":[{"interface":"back","average":{"1min":5.109,"5min":3.0990000000000002,"15min":3.468},"min":{"1min":0.34999999999999998,"5min":0.27200000000000002,"15min":0.255},"max":{"1min":78.751000000000005,"5min":78.751000000000005,"15min":148.08000000000001},"last":2.2509999999999999},{"interface":"front","average":{"1min":5.0830000000000002,"5min":3.5009999999999999,"15min":3.722},"min":{"1min":0.317,"5min":0.317,"15min":0.25700000000000001},"max":{"1min":77.930999999999997,"5min":77.930999999999997,"15min":147.88900000000001},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:18:36 2023","interfaces":[{"interface":"back","average":{"1min":1.8740000000000001,"5min":3.8940000000000001,"15min":4.133},"min":{"1min":0.45500000000000002,"5min":0.32400000000000001,"15min":0.252},"max":{"1min":11.031000000000001,"5min":96.447000000000003,"15min":163.97499999999999},"last":1.498},{"interface":"front","average":{"1min":2.8069999999999999,"5min":4.181,"15min":4.0129999999999999},"min":{"1min":0.40000000000000002,"5min":0.29899999999999999,"15min":0.27800000000000002},"max":{"1min":11.609999999999999,"5min":95.944999999999993,"15min":164.447},"last":0.48299999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:17:45 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":1.4450000000000001,"15min":1.5149999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.23499999999999999},"max":{"1min":1.2430000000000001,"5min":87.912999999999997,"15min":87.912999999999997},"last":1.0369999999999999},{"interface":"front","average":{"1min":0.48099999999999998,"5min":1.5780000000000001,"15min":1.395},"min":{"1min":0.28199999999999997,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":1.292,"5min":88.602999999999994,"15min":88.602999999999994},"last":0.28399999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:18:32 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":1.6910000000000001,"15min":2.097},"min":{"1min":0.315,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.1699999999999999,"5min":86.091999999999999,"15min":210.44300000000001},"last":1.069},{"interface":"front","average":{"1min":0.503,"5min":1.7869999999999999,"15min":2.0009999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.5350000000000001,"5min":86.048000000000002,"15min":210.78899999999999},"last":0.32100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.53700000000000003,"15min":1.115},"min":{"1min":0.32400000000000001,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":1.111,"5min":3.0979999999999999,"15min":18.23},"last":0.376},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.79700000000000004,"15min":1.194},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.314},"max":{"1min":3.9929999999999999,"5min":6.1399999999999997,"15min":14.653},"last":1.0980000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724840,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1833668,"kb_used_data":1589820,"kb_used_omap":0,"kb_used_meta":243840,"kb_avail":91911484,"statfs":{"total":95995035648,"available":94117359616,"internally_reserved":0,"allocated":1627975680,"data_stored":1627236825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":249692160},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":5.7199999999999998,"5min":2.4769999999999999,"15min":1.798},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.20599999999999999},"max":{"1min":84.266000000000005,"5min":84.266000000000005,"15min":84.266000000000005},"last":0.26700000000000002},{"interface":"front","average":{"1min":6.2199999999999998,"5min":2.9089999999999998,"15min":1.865},"min":{"1min":0.25700000000000001,"5min":0.247,"15min":0.222},"max":{"1min":84.558000000000007,"5min":84.558000000000007,"15min":84.558000000000007},"last":0.28699999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":2.27,"15min":1.8720000000000001},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":4.2389999999999999,"5min":18.082999999999998,"15min":18.082999999999998},"last":1.8069999999999999},{"interface":"front","average":{"1min":0.90100000000000002,"5min":2.3380000000000001,"15min":2.2919999999999998},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":3.3780000000000001,"5min":17.963999999999999,"15min":17.963999999999999},"last":1.7989999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:18:05 2023","interfaces":[{"interface":"back","average":{"1min":3.012,"5min":2.855,"15min":2.6600000000000001},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.23599999999999999},"max":{"1min":22.065999999999999,"5min":33.68,"15min":97.087999999999994},"last":1.929},{"interface":"front","average":{"1min":2.7719999999999998,"5min":3.2090000000000001,"15min":2.6739999999999999},"min":{"1min":0.34300000000000003,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":22.081,"5min":33.707999999999998,"15min":97.305999999999997},"last":9.2330000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:18:40 2023","interfaces":[{"interface":"back","average":{"1min":8.8490000000000002,"5min":7.173,"15min":3.7010000000000001},"min":{"1min":0.30599999999999999,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":74.599000000000004,"5min":118.208,"15min":118.208},"last":0.48999999999999999},{"interface":"front","average":{"1min":8.9320000000000004,"5min":6.8019999999999996,"15min":3.4940000000000002},"min":{"1min":0.439,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":74.578000000000003,"5min":118.42,"15min":118.42},"last":1.5880000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:17:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1379999999999999,"5min":2.3570000000000002,"15min":1.772},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26200000000000001},"max":{"1min":12.202,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.2050000000000001,"5min":2.3149999999999999,"15min":1.8380000000000001},"min":{"1min":0.32400000000000001,"5min":0.27200000000000002,"15min":0.26000000000000001},"max":{"1min":12.481,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:18:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.66500000000000004,"15min":1.127},"min":{"1min":0.311,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.3859999999999999,"5min":5.8760000000000003,"15min":15.196},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59799999999999998,"15min":1.008},"min":{"1min":0.35499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.97399999999999998,"5min":2.9990000000000001,"15min":14.965999999999999},"last":0.56100000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:18:40 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.73999999999999999,"15min":1.282},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":2.113,"5min":8.2200000000000006,"15min":15.301},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.71899999999999997,"15min":1.254},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":0.92700000000000005,"5min":5.6559999999999997,"15min":15.762},"last":0.51200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2573012992,"data_stored":2573008920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2222592000,"data_stored":2222587928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2277441536,"data_stored":2277441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3223519232,"data_stored":3223519232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1935278080,"data_stored":1935278080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1615462400,"data_stored":1615462400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1644040192,"data_stored":1644036120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:18:50.236 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:18:50.236 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:18:50.236 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:18:50.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T15:18:50.278 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:50.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:50 smithi017 ceph-mon[222878]: pgmap v710: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T15:18:50.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:50 smithi017 ceph-mon[222878]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:50.319 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:50 smithi017 ceph-mon[215646]: pgmap v710: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T15:18:50.320 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:50 smithi017 ceph-mon[215646]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:50.459 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:50.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:50.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:50.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:50.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:50 smithi027 ceph-mon[195233]: pgmap v710: 109 pgs: 109 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 692 op/s 2023-09-23T15:18:50.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:50 smithi027 ceph-mon[195233]: from='client.35688 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:18:50.598 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:50.650 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:50.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:50.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:50.785 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:50.840 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:50.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:50.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:50.945 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:18:50.974 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:51.029 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:18:51.029 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:18:51.029 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:18:51.029 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:18:51.029 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:18:51.036 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:18:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:51.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:18:51.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:51.252 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:52.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:52 smithi017 ceph-mon[222878]: pgmap v711: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:18:52.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:52 smithi017 ceph-mon[215646]: pgmap v711: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:18:52.454 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:52.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:52.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:52.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:52.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:52 smithi027 ceph-mon[195233]: pgmap v711: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:18:52.595 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:52.655 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:52.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:52.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:52.685 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:52.707 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:52.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:18:52.761 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:52.876 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:53.162 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:53.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:53.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:53.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:53.388 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:53.544 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:53.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:53.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:53.794 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:53.917 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:53.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:53.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:54.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:54.062 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:54.112 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:54.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:54.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:54.219 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:54.219 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:18:54.249 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:54.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:54 smithi027 ceph-mon[195233]: pgmap v712: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T15:18:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:54 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:18:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3244404994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:54 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.350 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:54.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:54.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:54.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:54.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[222878]: pgmap v712: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T15:18:54.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:18:54.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3244404994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.530 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[215646]: pgmap v712: 109 pgs: 109 active+clean; 4.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 681 op/s 2023-09-23T15:18:54.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:18:54.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3244404994' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:54 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:18:54.543 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:55.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:55.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[222878]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:55.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[222878]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T15:18:55.492 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[222878]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:55.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[215646]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:55.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[215646]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T15:18:55.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:55 smithi017 ceph-mon[215646]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.562 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:55.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:55.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:55 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:18:55.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:55 smithi027 ceph-mon[195233]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:18:55.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:55 smithi027 ceph-mon[195233]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T15:18:55.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:55 smithi027 ceph-mon[195233]: failed to encode map e249 with expected crc 2023-09-23T15:18:55.670 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:55.718 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:56.164 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:56.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:56.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:56.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:18:56.353 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:56.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:56 smithi017 ceph-mon[215646]: pgmap v714: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T15:18:56.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:56 smithi017 ceph-mon[222878]: pgmap v714: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T15:18:56.548 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:18:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:56.562 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:18:56.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:56 smithi027 ceph-mon[195233]: pgmap v714: 109 pgs: 109 active+clean; 4.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 685 op/s 2023-09-23T15:18:56.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:56.706 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:56.786 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:56.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:56.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:56.849 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:18:56.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:18:56.993 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:18:57.033 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:57.042 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:18:57.093 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:18:57.168 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:18:57.168 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:18:57.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:57.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:18:57.223 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:18:57.227 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 327 (0%) 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:avg 46.7143 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.28378 -> 5.28378 (expected baseline 6.32778) 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 55 pgs (1.17737 -> 1.17737 * mean) 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0246 2023-09-23T15:18:57.228 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.2462 2023-09-23T15:18:57.254 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:18:57.264 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:18:57.270 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1303708747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:57.270 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:57 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:57.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:18:57.307 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:18:57.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:18:57.349 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.380 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.381 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:18:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1303708747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:57.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:57 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:57.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1303708747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:18:57.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:57 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:57.870 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:18:57.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:57.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:18:57.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:58.017 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:58.073 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:58.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:58.179 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:18:58.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:18:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:18:58] "GET /metrics HTTP/1.1" 200 36289 "" "Prometheus/2.43.0" 2023-09-23T15:18:58.260 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:58.276 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:58 smithi027 ceph-mon[195233]: pgmap v715: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T15:18:58.277 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:58 smithi027 ceph-mon[195233]: from='client.55531 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:18:58.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:18:58.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[215646]: pgmap v715: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T15:18:58.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[215646]: from='client.55531 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:18:58.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:58.521 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:58.521 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[222878]: pgmap v715: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 674 op/s 2023-09-23T15:18:58.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[222878]: from='client.55531 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:18:58.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:58.522 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:18:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:18:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:18:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:18:59.275 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:18:59.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T15:18:59.469 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:18:59.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:59.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:18:59.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:18:59.603 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:59.662 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:18:59.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:59.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:18:59.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:18:59.798 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:18:59.862 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:18:59.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:18:59.868 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:18:59.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:00.000 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:00.061 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:00.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:00.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:00.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:00.264 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:00.287 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:00 smithi027 ceph-mon[195233]: pgmap v716: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T15:19:00.407 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:00.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:00.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:00.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:00 smithi017 ceph-mon[215646]: pgmap v716: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T15:19:00.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:00 smithi017 ceph-mon[222878]: pgmap v716: 109 pgs: 109 active+clean; 4.4 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2023-09-23T15:19:00.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:00.544 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:00.654 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:00.688 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:00.689 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:00.955 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:00.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:01.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:01.163 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:01.706 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:01.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:01.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:01.844 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:01.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:19:01.899 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:01.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:01.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:02.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:02.043 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:02.100 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:02.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:02.107 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:02.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:02.229 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:19:02.238 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:02.298 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:02 smithi017 ceph-mon[215646]: pgmap v717: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T15:19:02.309 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:02.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:02.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:02.342 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:02.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:02.491 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:02.557 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:02 smithi027 ceph-mon[195233]: pgmap v717: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T15:19:02.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:02 smithi017 ceph-mon[222878]: pgmap v717: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 657 op/s 2023-09-23T15:19:02.953 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:02.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:02.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:03.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:03.096 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:03.146 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:03.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:03.148 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:03.249 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:03.389 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:03.576 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:03.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:03 smithi017 ceph-mon[222878]: pgmap v718: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:03.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:03 smithi017 ceph-mon[215646]: pgmap v718: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:03.738 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:03.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:03.748 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:03.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:03 smithi027 ceph-mon[195233]: pgmap v718: 109 pgs: 109 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:03.850 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:03.949 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:04.048 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:04.048 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:04.496 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:19:04.669 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:04.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:04.674 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:04.775 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:04.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/789087957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.814 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:04.902 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:04.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:04.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:04.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/789087957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.905 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/789087957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:04.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:19:05.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:05.104 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:05.412 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:05.773 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:05.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:05.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T15:19:05.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:05 smithi027 ceph-mon[195233]: pgmap v720: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[222878]: pgmap v720: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: failed to encode map e250 with expected crc 2023-09-23T15:19:05.932 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:05 smithi017 ceph-mon[215646]: pgmap v720: 109 pgs: 109 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 645 op/s 2023-09-23T15:19:05.946 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:05.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:05.964 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:06.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:06.127 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:06.630 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:06.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:06.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:06.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:06.761 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:06.999 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:06.999 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:07.350 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:07.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:07.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:07.454 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:07.580 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:07.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:19:07.760 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:07.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:07.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:07.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:07.924 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:07.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:19:07.992 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:07.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:08.048 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:08 smithi017 ceph-mon[215646]: pgmap v721: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T15:19:08.048 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:08] "GET /metrics HTTP/1.1" 200 36285 "" "Prometheus/2.43.0" 2023-09-23T15:19:08.096 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:08.128 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:08.147 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:08.190 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:08.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:08.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:08.197 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:08 smithi027 ceph-mon[195233]: pgmap v721: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T15:19:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 327 (0%) 2023-09-23T15:19:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:avg 46.7143 2023-09-23T15:19:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.28378 -> 5.28378 (expected baseline 6.32778) 2023-09-23T15:19:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-09-23T15:19:08.221 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 55 pgs (1.17737 -> 1.17737 * mean) 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0227 2023-09-23T15:19:08.222 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.2275 2023-09-23T15:19:08.242 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:19:08.297 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:08.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:08 smithi017 ceph-mon[222878]: pgmap v721: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 644 op/s 2023-09-23T15:19:08.408 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:08.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:08.533 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:08.588 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:08.653 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:08.688 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:08.720 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:08.760 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.919 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.920 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.920 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:19:08.920 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:19:09.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:09 smithi017 ceph-mon[222878]: from='client.55546 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:09.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2840091366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:09 smithi017 ceph-mon[215646]: from='client.55546 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2840091366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:09.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:09 smithi027 ceph-mon[195233]: from='client.55546 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:09.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2840091366' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:09.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:09.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:09.886 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:19:09.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T15:19:09.887 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:09.887 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166941 5.1 5:8b30db7e:::benchmark_data_smithi017_288048_object55172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.79841731800002, 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40214831299999998, 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:19:09.888 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166941 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086352+0000", 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086351+0000", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086356+0000", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1039999999999998e-06 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086368+0000", 2023-09-23T15:19:09.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1618e-05 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086374+0000", 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5490000000000003e-06 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086536+0000", 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000161743 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086652+0000", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011594 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086770+0000", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011798600000000001 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091315+0000", 2023-09-23T15:19:09.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045446310000000004 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488382+0000", 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39706780000000003 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488406+0000", 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3218e-05 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488454+0000", 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8301999999999999e-05 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488501+0000", 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6573000000000003e-05 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.894 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166942 5.e 5:73e5ada8:::benchmark_data_smithi017_288048_object55173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.798382984, 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40238844899999998, 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166942 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086387+0000", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086386+0000", 2023-09-23T15:19:09.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086389+0000", 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5830000000000001e-06 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086395+0000", 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9059999999999999e-06 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086399+0000", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2300000000000002e-06 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086603+0000", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020372899999999999 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086701+0000", 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8332000000000007e-05 2023-09-23T15:19:09.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086804+0000", 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103294 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.093768+0000", 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006964034 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488259+0000", 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39449091200000003 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488702+0000", 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00044246200000000002 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488734+0000", 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2069000000000001e-05 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488775+0000", 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1278e-05 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166947 5.e 5:7dd7ba47:::benchmark_data_smithi017_288048_object55178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.79791932199998, 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40205635099999998, 2023-09-23T15:19:09.901 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166947 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086850+0000", 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086849+0000", 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086854+0000", 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6290000000000001e-06 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086866+0000", 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2478e-05 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.086873+0000", 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1299999999999998e-06 2023-09-23T15:19:09.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090167+0000", 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032944319999999999 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090246+0000", 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9369999999999994e-05 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:19:09.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.090328+0000", 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1706999999999995e-05 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488279+0000", 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397950631 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488321+0000", 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2725999999999997e-05 2023-09-23T15:19:09.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488813+0000", 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000491991 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488869+0000", 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5350999999999999e-05 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488907+0000", 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7762999999999997e-05 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166944 5.0 e147/132 5:0733dbd2:::benchmark_data_smithi017_288048_object55175:head v 147'10299, mlcod=147'10297)", 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.797688924, 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40144166599999997, 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.908 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087081+0000", 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087080+0000", 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087086+0000", 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5890000000000002e-06 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087096+0000", 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0434999999999999e-05 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087101+0000", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4890000000000003e-06 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087141+0000", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9147999999999999e-05 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.087162+0000", 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0928e-05 2023-09-23T15:19:09.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.099400+0000", 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012238123 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488522+0000", 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38912261599999998 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.34764.0:166953 5.c 5:3faa0b3c:::benchmark_data_smithi017_288048_object55184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e147)", 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.79364254999996, 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39750094200000002, 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.34764", 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/4230424776", 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166953 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.913 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091127+0000", 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091126+0000", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091130+0000", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.014e-06 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091144+0000", 2023-09-23T15:19:09.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4717e-05 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091149+0000", 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9590000000000003e-06 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091258+0000", 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108806 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091386+0000", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012736 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.091496+0000", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109987 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488306+0000", 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39681003300000001 2023-09-23T15:19:09.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488328+0000", 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2175999999999999e-05 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488380+0000", 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1913999999999998e-05 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488628+0000", 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024825900000000002 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166945 5.0 e147/132 5:0c8902eb:::benchmark_data_smithi017_288048_object55176:head v 147'10300, mlcod=147'10297)", 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.79258612700005, 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39637069699999999, 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.919 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092183+0000", 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092188+0000", 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9760000000000003e-06 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092197+0000", 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2019999999999993e-06 2023-09-23T15:19:09.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092203+0000", 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8019999999999997e-06 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092271+0000", 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8647999999999997e-05 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.092292+0000", 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0990999999999999e-05 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488527+0000", 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39623427999999999 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488554+0000", 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7458e-05 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.923 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166939 5.f e147/114 5:f794a49b:::benchmark_data_smithi017_288048_object55170:head v 147'10605, mlcod=147'10602)", 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.78841435100003, 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40201396099999998, 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:19:09.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096355+0000", 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096354+0000", 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096363+0000", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2139999999999996e-06 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096374+0000", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1022000000000001e-05 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096379+0000", 2023-09-23T15:19:09.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8340000000000004e-06 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488409+0000", 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39202973600000002 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488494+0000", 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5271000000000004e-05 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498336+0000", 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098418459999999996 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.498369+0000", 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2783000000000001e-05 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.928 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166950 5.f e147/114 5:f6b3f84e:::benchmark_data_smithi017_288048_object55181:head v 147'10606, mlcod=147'10602)", 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.78836655700002, 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40582205900000001, 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096403+0000", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096406+0000", 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4649999999999999e-06 2023-09-23T15:19:09.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096412+0000", 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7450000000000001e-06 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096415+0000", 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5650000000000002e-06 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493677+0000", 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.397262008 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.493697+0000", 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9083e-05 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502182+0000", 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084849629999999999 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.502225+0000", 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.35e-05 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.34764.0:166955 5.5 e147/132 5:a6029d8c:::benchmark_data_smithi017_288048_object55186:head v 147'10623, mlcod=147'10621)", 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.78832934299999, 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.410760976, 2023-09-23T15:19:09.933 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096440+0000", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096443+0000", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1700000000000001e-06 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096449+0000", 2023-09-23T15:19:09.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2979999999999997e-06 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096453+0000", 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4520000000000002e-06 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488252+0000", 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39179915599999998 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488294+0000", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2344e-05 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507165+0000", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018870505999999999 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.507201+0000", 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6368999999999997e-05 2023-09-23T15:19:09.937 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166947 5.e e147/114 ondisk, result = 0)", 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.78830363999998, 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39185894700000001, 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096466+0000", 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:19:09.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096468+0000", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.357e-06 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096473+0000", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2220000000000001e-06 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096477+0000", 2023-09-23T15:19:09.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.399e-06 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488307+0000", 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39183000600000001 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488315+0000", 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488325+0000", 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0294e-05 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166953 5.c e147/132 ondisk, result = 0)", 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.788285578, 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39182831099999998, 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.942 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096484+0000", 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096485+0000", 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6139999999999999e-06 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096490+0000", 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8550000000000001e-06 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.096493+0000", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9909999999999998e-06 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488288+0000", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.391795069 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488298+0000", 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4709999999999996e-06 2023-09-23T15:19:09.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488312+0000", 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4516e-05 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.34764.0:166941 5.1 e147/114 ondisk, result = 0)", 2023-09-23T15:19:09.946 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 531.78439290599999, 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38801138200000002, 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100377+0000", 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100376+0000", 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.100382+0000", 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0780000000000001e-06 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488291+0000", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38790906000000003 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488299+0000", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8660000000000006e-06 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488367+0000", 2023-09-23T15:19:09.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8133000000000006e-05 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488376+0000", 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.935e-06 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:10:18.488388+0000", 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2139e-05 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.950 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78602 6.e 6:7eeb81e7:::benchmark_data_smithi017_350392_object78601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.33616972300001, 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6491542219999999, 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78602 2023-09-23T15:19:09.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548600+0000", 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548599+0000", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548629+0000", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9839999999999999e-05 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548639+0000", 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6940000000000007e-06 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548644+0000", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8779999999999997e-06 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548691+0000", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7481000000000002e-05 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548837+0000", 2023-09-23T15:19:09.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014598700000000001 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.548993+0000", 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015569999999999999 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565316+0000", 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016322705 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565327+0000", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0994e-05 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847765+0000", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28243886899999998 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847776+0000", 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0519999999999999e-05 2023-09-23T15:19:09.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197689+0000", 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3499132 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.197754+0000", 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78611 6.e 6:7d1942e2:::benchmark_data_smithi017_350392_object78610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 388.32721039, 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6733330400000002, 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78611 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.958 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557559+0000", 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557557+0000", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557602+0000", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4189e-05 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557615+0000", 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3869999999999999e-05 2023-09-23T15:19:09.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.557627+0000", 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1229999999999999e-05 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565352+0000", 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077249190000000002 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:09.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.565359+0000", 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0609999999999998e-06 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847801+0000", 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28244210600000003 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.847808+0000", 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5240000000000003e-06 2023-09-23T15:19:09.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201122+0000", 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.353314205 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201310+0000", 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018789999999999999 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:09.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201482+0000", 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017193500000000001 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.204567+0000", 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030849520000000002 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.210867+0000", 2023-09-23T15:19:09.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062997979999999997 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228500+0000", 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017632618999999999 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.228573+0000", 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3799000000000003e-05 2023-09-23T15:19:09.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.230892+0000", 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023187149999999998 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:147647 6.2 6:4814bf33:::benchmark_data_smithi017_350392_object47256:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:19:09.966 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 263.945849082, 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25844827100000001, 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 147647 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938920+0000", 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938919+0000", 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:09.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938928+0000", 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938946+0000", 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8111e-05 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938954+0000", 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1780000000000003e-06 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.938996+0000", 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2521999999999997e-05 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939088+0000", 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1113000000000007e-05 2023-09-23T15:19:09.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.939182+0000", 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4487000000000005e-05 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:45.942153+0000", 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029708239999999999 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.143408+0000", 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20125490800000001 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197272+0000", 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.053864243999999999 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197325+0000", 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2802e-05 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:14:46.197369+0000", 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3869000000000003e-05 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:124289 7.1 7:8a50cffb:::benchmark_data_smithi017_422824_object30419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:08.691553+0000", 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.1932168400000001, 2023-09-23T15:19:09.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027421299, 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124289 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691553+0000", 2023-09-23T15:19:09.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691553+0000", 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691551+0000", 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691558+0000", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8000000000000001e-06 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691570+0000", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2533e-05 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.691577+0000", 2023-09-23T15:19:09.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6150000000000002e-06 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.696991+0000", 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005413719 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.697142+0000", 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000150994 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.697275+0000", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013308800000000001 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.700399+0000", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031239750000000002 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.704218+0000", 2023-09-23T15:19:09.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038192389999999999 2023-09-23T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.718911+0000", 2023-09-23T15:19:09.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014692709999999999 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.718946+0000", 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5456000000000001e-05 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.718974+0000", 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7949999999999998e-05 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.981 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:124347 7.1 7:88253b07:::benchmark_data_smithi017_422824_object30477:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:08.777457+0000", 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.1073124560000001, 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019924984, 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:09.982 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124347 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777457+0000", 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777457+0000", 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777456+0000", 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777462+0000", 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4300000000000003e-06 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777476+0000", 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3439e-05 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777481+0000", 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2050000000000001e-06 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777528+0000", 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7617000000000002e-05 2023-09-23T15:19:09.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777668+0000", 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013988600000000001 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.777806+0000", 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013809900000000001 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.780365+0000", 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025587299999999999 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.780533+0000", 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000168095 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.797278+0000", 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016744565999999999 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.797335+0000", 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6922000000000002e-05 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.797382+0000", 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7264e-05 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:124375 7.1 7:81563d2f:::benchmark_data_smithi017_422824_object30505:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:08.812643+0000", 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.072126366, 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.121595943, 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124375 2023-09-23T15:19:09.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812643+0000", 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812643+0000", 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812642+0000", 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812647+0000", 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2490000000000001e-06 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812662+0000", 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.526e-05 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812676+0000", 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.378e-05 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812730+0000", 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3656999999999998e-05 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812855+0000", 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125587 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.812994+0000", 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138763 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:09.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.816318+0000", 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033237649999999998 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.816478+0000", 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015993499999999999 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.934173+0000", 2023-09-23T15:19:09.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11769535 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.934210+0000", 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7209999999999998e-05 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:08.934239+0000", 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8707000000000001e-05 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.995 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:124688 7.1 7:82ee30e3:::benchmark_data_smithi017_422824_object30818:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:09.352508+0000", 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.53226146500000004, 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020228416999999999, 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:09.996 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124688 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352508+0000", 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352508+0000", 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:09.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352507+0000", 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352512+0000", 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.13e-06 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352522+0000", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9809999999999998e-06 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352527+0000", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2290000000000002e-06 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352563+0000", 2023-09-23T15:19:09.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6208999999999998e-05 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352656+0000", 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2305000000000005e-05 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.352720+0000", 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4115999999999994e-05 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:10.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.355226+0000", 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025065909999999999 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.356183+0000", 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00095638899999999998 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.372650+0000", 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016467345000000001 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.372691+0000", 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1362999999999997e-05 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.372736+0000", 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4996000000000001e-05 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:10.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:124733 7.1 7:84289ea2:::benchmark_data_smithi017_422824_object30863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:09.435524+0000", 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.44924517800000002, 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020233069999999999, 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 124733 2023-09-23T15:19:10.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435524+0000", 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435524+0000", 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435522+0000", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435530+0000", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6240000000000002e-06 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435546+0000", 2023-09-23T15:19:10.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5982999999999999e-05 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435554+0000", 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9810000000000003e-06 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435605+0000", 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.075e-05 2023-09-23T15:19:10.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435737+0000", 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000132223 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.435851+0000", 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011411100000000001 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:10.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.439180+0000", 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033290889999999999 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.439382+0000", 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020191100000000001 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.455693+0000", 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016311398000000001 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.455728+0000", 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4334e-05 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:09.455757+0000", 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9606000000000001e-05 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:10.009 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:10.010 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:10.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:10.050 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:10.111 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:10.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:10.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:10.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:10.274 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:10.284 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:10 smithi027 ceph-mon[195233]: pgmap v722: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T15:19:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.398 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:10.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:10.415 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[215646]: pgmap v722: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[222878]: pgmap v722: 109 pgs: 109 active+clean; 4.0 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 640 op/s 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:10.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:10.607 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:10.636 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:19:10.725 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:10.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:10.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:10.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:10.957 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:11.074 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:11.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:11.101 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:11.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:11.282 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:11.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:19:11.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:19:11.327 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:11.379 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:11.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:11.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:19:11.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:19:11.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:11.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:19:11.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:19:11.395 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:11.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:11.537 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:11.705 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:11.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:11.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:11.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:11.838 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:11.855 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:11.881 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:11.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:11.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:11.984 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:12.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:12.054 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:12.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:12.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:12.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:19:12.158 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:12.192 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:12.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:12 smithi027 ceph-mon[195233]: pgmap v723: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T15:19:12.375 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:12.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:12.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:12.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:12 smithi017 ceph-mon[215646]: pgmap v723: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T15:19:12.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:12 smithi017 ceph-mon[222878]: pgmap v723: 109 pgs: 109 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 612 op/s 2023-09-23T15:19:12.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:12.518 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:12.673 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:12.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:12.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:12.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:12.820 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:12.889 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:12.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:12.892 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:12.921 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:12.921 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:19:12.921 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:19:12.921 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:12.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:13.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:13.269 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:19:13.300 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1153058683' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.300 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.300 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:13.300 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1153058683' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.300 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.301 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1153058683' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:13.744 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:19:13.786 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:13.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:13.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:13.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:14.006 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:14.132 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:14.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:14.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:14.214 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:14.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:14.353 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[215646]: pgmap v724: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[215646]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[222878]: pgmap v724: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:14.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:14.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:14 smithi017 ceph-mon[222878]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T15:19:14.478 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:14.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:14.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:14.528 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:14.586 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:14 smithi027 ceph-mon[195233]: pgmap v724: 109 pgs: 109 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:14 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:14 smithi027 ceph-mon[195233]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T15:19:14.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:14.890 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:14.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:14.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:15.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:15.038 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 327 (0%) 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:avg 46.7143 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.28378 -> 5.28378 (expected baseline 6.32778) 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 55 pgs (1.17737 -> 1.17737 * mean) 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:19:15.056 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0215 2023-09-23T15:19:15.057 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.2150 2023-09-23T15:19:15.068 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:19:15.085 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:15.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:15.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:15.296 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:15.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:15 smithi027 ceph-mon[195233]: failed to encode map e251 with expected crc 2023-09-23T15:19:15.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:15 smithi017 ceph-mon[215646]: failed to encode map e251 with expected crc 2023-09-23T15:19:15.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:15 smithi017 ceph-mon[222878]: failed to encode map e251 with expected crc 2023-09-23T15:19:15.814 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:15.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:15.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:15.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:15.974 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:19:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:19:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:19:15.974 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:15.993 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:16.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:19:16.460 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:16.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:16.519 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:19:16.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T15:19:16.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:16 smithi017 ceph-mon[222878]: pgmap v726: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T15:19:16.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:16 smithi017 ceph-mon[222878]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:16.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:16 smithi017 ceph-mon[215646]: pgmap v726: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T15:19:16.548 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:16 smithi017 ceph-mon[215646]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:16.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:16.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:16 smithi027 ceph-mon[195233]: pgmap v726: 109 pgs: 109 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 593 op/s 2023-09-23T15:19:16.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:16 smithi027 ceph-mon[195233]: from='client.45641 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:16.599 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:16.799 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:16.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:16.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:16.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:19:16.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:16.936 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:16.987 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:17.109 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:17.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:17.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:17.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:17.293 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:17.359 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:17.523 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:17.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:17.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:17.541 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:17.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:17.615 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:17.642 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:17.677 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:17.691 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:17.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:17.697 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:17.762 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:17.801 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:17.859 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:17.866 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.866 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.867 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:19:17.887 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:17.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:17.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:17.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:18.045 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:18.291 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:18] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2023-09-23T15:19:18.293 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:18 smithi017 ceph-mon[222878]: pgmap v727: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T15:19:18.293 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2090807467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:18.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:18 smithi027 ceph-mon[195233]: pgmap v727: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T15:19:18.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2090807467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:18.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:18 smithi017 ceph-mon[215646]: pgmap v727: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 585 op/s 2023-09-23T15:19:18.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2090807467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:18.861 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:18.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:18.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:18.958 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:18.959 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:18.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:19.000 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:19.201 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:19.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:19.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:19.305 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:19.333 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:19.377 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:19.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:19.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:19.558 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:19.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:19:19.789 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:19.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:19.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:19.989 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:20.141 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:20.153 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:20.215 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:20.253 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:20.309 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:20 smithi017 ceph-mon[215646]: pgmap v728: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T15:19:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:20 smithi017 ceph-mon[222878]: pgmap v728: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T15:19:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:20 smithi027 ceph-mon[195233]: pgmap v728: 109 pgs: 109 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 581 op/s 2023-09-23T15:19:21.035 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:21.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:21.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:21.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:21.173 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:21.294 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:21.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:21.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:21.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:21.430 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:21.519 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:21.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:21.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:21.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:21.658 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:21.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:19:21.817 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:21.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:21.886 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:21.886 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:21.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:21.989 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 327 (0%) 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:avg 46.7143 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.28378 -> 5.28378 (expected baseline 6.32778) 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 55 pgs (1.17737 -> 1.17737 * mean) 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0203 2023-09-23T15:19:21.990 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.2027 2023-09-23T15:19:22.034 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:19:22.056 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:22.282 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:22.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:22.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:22.326 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:19:22.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:22.425 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:22 smithi027 ceph-mon[195233]: pgmap v729: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T15:19:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:22.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:19:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:22 smithi017 ceph-mon[215646]: pgmap v729: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T15:19:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:22 smithi017 ceph-mon[222878]: pgmap v729: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 572 op/s 2023-09-23T15:19:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:23.138 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:19:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:19:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[222878]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:19:23.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:19:23.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:23.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T15:19:23.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:23 smithi017 ceph-mon[215646]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:23.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:19:23.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:19:23.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3087172737' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:23.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T15:19:23.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:23 smithi027 ceph-mon[195233]: failed to encode map e252 with expected crc 2023-09-23T15:19:23.635 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:23.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:23.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:23.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:23.773 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:19:23.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T15:19:23.778 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:23.843 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:23.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:23.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:23.947 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:23.996 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:24.056 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:24.169 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:24.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:24.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:24.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:24.328 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:24 smithi017 ceph-mon[215646]: pgmap v731: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T15:19:24.380 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:24.595 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:24 smithi027 ceph-mon[195233]: pgmap v731: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T15:19:24.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:24.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:24.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:24 smithi017 ceph-mon[222878]: pgmap v731: 109 pgs: 109 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2023-09-23T15:19:24.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:24.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:24.948 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:24.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:24.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:24.965 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:24.965 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:25.053 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:25.153 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:25.286 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:25.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:25.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:25.480 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:25.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:25 smithi027 ceph-mon[195233]: pgmap v732: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s 2023-09-23T15:19:25.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:25 smithi017 ceph-mon[215646]: pgmap v732: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s 2023-09-23T15:19:25.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:25 smithi017 ceph-mon[222878]: pgmap v732: 109 pgs: 109 active+clean; 3.4 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 598 op/s 2023-09-23T15:19:25.756 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:25.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:19:25.780 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:25.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:25.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:25.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:19:25.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:26.038 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:26.208 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:26.313 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:26.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:26.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:26.425 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:26.462 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:26.525 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:26.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:26.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:26.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:26.669 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:26.671 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:26.684 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3182583765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:26.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3182583765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:26.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:26.803 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3182583765' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:26.812 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:26.833 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:26.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:26.836 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:26.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:26.968 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:27.013 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:27.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:27.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:27.038 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.039 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.040 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.040 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:19:27.117 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:27.185 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:27.897 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:27.897 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:19:27.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:19:27.897 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:19:27.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51892 6.a e178/171 6:5225cf16:::benchmark_data_smithi017_350392_object51891:head v 178'3239, mlcod=178'3237)", 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.63784193399999, 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31651939499999998, 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256882+0000", 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256881+0000", 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256908+0000", 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6789999999999999e-05 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256919+0000", 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0925e-05 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256925+0000", 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9009999999999999e-06 2023-09-23T15:19:27.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256977+0000", 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1675999999999998e-05 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.256998+0000", 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0988999999999998e-05 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573352+0000", 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31635466800000001 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573401+0000", 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9128999999999997e-05 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.902 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51896 6.5 6:ae721bbc:::benchmark_data_smithi017_350392_object51895:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.637015873, 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32171442, 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51896 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257708+0000", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257707+0000", 2023-09-23T15:19:27.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257767+0000", 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0673e-05 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257781+0000", 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.343e-05 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257786+0000", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.395e-06 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257836+0000", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9305000000000003e-05 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257953+0000", 2023-09-23T15:19:27.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117194 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258083+0000", 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013044599999999999 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575282+0000", 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31719867800000001 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575314+0000", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2376000000000003e-05 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579321+0000", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0040069040000000004 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579377+0000", 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6243999999999999e-05 2023-09-23T15:19:27.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579423+0000", 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5207999999999998e-05 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51897 6.3 6:cc65482f:::benchmark_data_smithi017_350392_object51896:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:19:27.909 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.636923964, 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32175430300000002, 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51897 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257800+0000", 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257799+0000", 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257840+0000", 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0457000000000002e-05 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257850+0000", 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0448000000000001e-05 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257856+0000", 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2780000000000001e-06 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.257902+0000", 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6032999999999998e-05 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258014+0000", 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112636 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258155+0000", 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141037 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575305+0000", 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31715001799999998 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579465+0000", 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041599280000000002 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579507+0000", 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1842999999999999e-05 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579554+0000", 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7205e-05 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.915 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51891 6.8 e178/171 6:1e19a070:::benchmark_data_smithi017_350392_object51890:head v 178'3166, mlcod=178'3163)", 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.63639064099999, 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32032912000000002, 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.916 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258333+0000", 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.258417+0000", 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4161999999999998e-05 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573347+0000", 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.314930302 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573358+0000", 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0491e-05 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573497+0000", 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013895499999999999 2023-09-23T15:19:27.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573523+0000", 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6495999999999998e-05 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.578611+0000", 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050881320000000004 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.578663+0000", 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1143000000000001e-05 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:51902 6.3 6:c005208d:::benchmark_data_smithi017_350392_object51901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e178)", 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.63115333299999, 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31611320199999998, 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.922 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 51902 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263571+0000", 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263570+0000", 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.263619+0000", 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9474000000000001e-05 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573336+0000", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30971668099999999 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573355+0000", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9483999999999998e-05 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575330+0000", 2023-09-23T15:19:27.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001974422 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575458+0000", 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128398 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.575588+0000", 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012964199999999999 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579564+0000", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039762349999999998 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579593+0000", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.923e-05 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579625+0000", 2023-09-23T15:19:27.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2081e-05 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.579684+0000", 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8517000000000002e-05 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45074.0:51912 6.a e178/171 6:5916b166:::benchmark_data_smithi017_350392_object51911:head v 178'3240, mlcod=178'3237)", 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 431.62615782099999, 2023-09-23T15:19:27.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30900894899999998, 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268566+0000", 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268565+0000", 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.268593+0000", 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7339999999999999e-05 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573405+0000", 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30481260799999998 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573412+0000", 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6950000000000003e-06 2023-09-23T15:19:27.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573697+0000", 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028508500000000003 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.573716+0000", 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9199999999999999e-05 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.577538+0000", 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038215749999999998 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:16.577575+0000", 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7370999999999998e-05 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.934 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78614 6.3 6:c41e9216:::benchmark_data_smithi017_350392_object78613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 406.32131201200002, 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.723727365, 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78614 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573412+0000", 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573410+0000", 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:27.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573445+0000", 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5234000000000002e-05 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573463+0000", 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7955e-05 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.573472+0000", 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2269999999999993e-06 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574191+0000", 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071970900000000002 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574206+0000", 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4419999999999999e-05 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.858632+0000", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28442586600000003 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.858645+0000", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3179e-05 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174166+0000", 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.315521355 2023-09-23T15:19:27.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174350+0000", 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018351899999999999 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.174561+0000", 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021169400000000001 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182906+0000", 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0083446349999999996 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182925+0000", 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9171999999999999e-05 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.294629+0000", 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.11170437599999999 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.294669+0000", 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9439e-05 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.945 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297139+0000", 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024704890000000002 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78628 6.3 6:cf9578ef:::benchmark_data_smithi017_350392_object78627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 406.29185265799998, 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6971857319999999, 2023-09-23T15:19:27.946 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78628 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602871+0000", 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602870+0000", 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602947+0000", 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7049e-05 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602962+0000", 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.507e-05 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602968+0000", 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3640000000000004e-06 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603027+0000", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9113000000000001e-05 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603035+0000", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3730000000000004e-06 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863310+0000", 2023-09-23T15:19:27.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26027560100000002 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863321+0000", 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0592e-05 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.179778+0000", 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.316457467 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.179963+0000", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000184599 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.180154+0000", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019075400000000001 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186399+0000", 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062453960000000003 2023-09-23T15:19:27.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189483+0000", 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030839719999999999 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297211+0000", 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.10772744400000001 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.297266+0000", 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5265e-05 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300057+0000", 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027911820000000001 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78631 6.3 6:cd655684:::benchmark_data_smithi017_350392_object78630:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:19:27.954 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 406.29015367699998, 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.697940681, 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78631 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604570+0000", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604569+0000", 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:27.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604681+0000", 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111625 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604696+0000", 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5099999999999999e-05 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604703+0000", 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2189999999999998e-06 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604758+0000", 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4718e-05 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.604770+0000", 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2207e-05 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863347+0000", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.258576906 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863355+0000", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0190000000000006e-06 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.182958+0000", 2023-09-23T15:19:27.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3196027080000001 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.183100+0000", 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014283899999999999 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.183276+0000", 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017582399999999999 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.192597+0000", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0093209339999999995 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193414+0000", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00081709899999999995 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300122+0000", 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.106707381 2023-09-23T15:19:27.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.300186+0000", 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4689000000000004e-05 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.302511+0000", 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023246389999999999 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78639 6.3 6:ce866323:::benchmark_data_smithi017_350392_object78638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 406.28165126900001, 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6955158290000001, 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78639 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.963 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613073+0000", 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613071+0000", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613125+0000", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3279999999999998e-05 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613143+0000", 2023-09-23T15:19:27.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8697000000000001e-05 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613149+0000", 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1699999999999996e-06 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613197+0000", 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8810999999999997e-05 2023-09-23T15:19:27.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.613206+0000", 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4700000000000002e-06 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863411+0000", 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25020520600000001 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863419+0000", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186425+0000", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3230066009999999 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186588+0000", 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016276600000000001 2023-09-23T15:19:27.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.186770+0000", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000182265 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193425+0000", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066544129999999996 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.195385+0000", 2023-09-23T15:19:27.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001960074 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.305864+0000", 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.110479492 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.305937+0000", 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2986999999999998e-05 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308589+0000", 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026511519999999999 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78642 6.3 6:cf556c25:::benchmark_data_smithi017_350392_object78641:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 406.27717338500003, 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.693634818, 2023-09-23T15:19:27.971 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78642 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617551+0000", 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617550+0000", 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617601+0000", 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1498000000000001e-05 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617612+0000", 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1119e-05 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617619+0000", 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.613e-06 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617661+0000", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1548000000000003e-05 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.617667+0000", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1550000000000004e-06 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863446+0000", 2023-09-23T15:19:27.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24577917099999999 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.863456+0000", 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0068000000000001e-05 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189512+0000", 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3260563809999999 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189670+0000", 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015796199999999999 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.189860+0000", 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190189 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.193432+0000", 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035713189999999999 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.195427+0000", 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019955120000000001 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308655+0000", 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.113228176 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.308719+0000", 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3638000000000003e-05 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.311185+0000", 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024663950000000001 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:126246 7.7 7:e8edcd6f:::benchmark_data_smithi017_422824_object32376:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:27.979 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.765266+0000", 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.129458424999999, 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30125656499999998, 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126246 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765266+0000", 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765266+0000", 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765265+0000", 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765269+0000", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8060000000000002e-06 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765278+0000", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765282+0000", 2023-09-23T15:19:27.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0589999999999996e-06 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765334+0000", 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2237999999999997e-05 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765439+0000", 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010506 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.765535+0000", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5563999999999995e-05 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.768385+0000", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028499659999999998 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066384+0000", 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29799910099999999 2023-09-23T15:19:27.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066428+0000", 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4177000000000001e-05 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066475+0000", 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6492000000000001e-05 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066522+0000", 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7497999999999998e-05 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:126255 7.c 7:3e5b26c5:::benchmark_data_smithi017_422824_object32385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.773676+0000", 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.121047910000001, 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.296761053, 2023-09-23T15:19:27.986 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126255 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773676+0000", 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773676+0000", 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773675+0000", 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773684+0000", 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8929999999999994e-06 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773696+0000", 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.208e-05 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773700+0000", 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6999999999999999e-06 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773736+0000", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5679999999999997e-05 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773823+0000", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6520999999999994e-05 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.773893+0000", 2023-09-23T15:19:27.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9907999999999995e-05 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776220+0000", 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023271979999999999 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070319+0000", 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29409927800000002 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070359+0000", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0209000000000001e-05 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070399+0000", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9279999999999997e-05 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070437+0000", 2023-09-23T15:19:27.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8605000000000001e-05 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:126257 7.d 7:bc82c68f:::benchmark_data_smithi017_422824_object32387:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.776561+0000", 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.118163187, 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29373411599999999, 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:27.993 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126257 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776561+0000", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776561+0000", 2023-09-23T15:19:27.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776559+0000", 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776570+0000", 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1003e-05 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776593+0000", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2846000000000001e-05 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776605+0000", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1513e-05 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776690+0000", 2023-09-23T15:19:27.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5143999999999997e-05 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.776919+0000", 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022942899999999999 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.777026+0000", 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010720199999999999 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.779703+0000", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002676533 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066487+0000", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28678400799999998 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070203+0000", 2023-09-23T15:19:27.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037162200000000001 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070253+0000", 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0328999999999999e-05 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.070295+0000", 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1597999999999997e-05 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:27.999 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:126258 7.7 7:e8062312:::benchmark_data_smithi017_422824_object32388:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.780701+0000", 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.114023329999998, 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29462094500000002, 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126258 2023-09-23T15:19:28.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780701+0000", 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780701+0000", 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780700+0000", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780704+0000", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0080000000000004e-06 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066359+0000", 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.285654556 2023-09-23T15:19:28.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066377+0000", 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8345000000000001e-05 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066556+0000", 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017873100000000001 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066660+0000", 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010394499999999999 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.066739+0000", 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9093999999999999e-05 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.072222+0000", 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054834890000000002 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.075113+0000", 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028905390000000001 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.075222+0000", 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010927199999999999 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.075278+0000", 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5967000000000001e-05 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.075322+0000", 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3603000000000002e-05 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.006 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:128634 7.d 7:b89b10df:::benchmark_data_smithi017_422824_object34764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:16.249223+0000", 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.645500709, 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.298284467, 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 128634 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249223+0000", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249223+0000", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249222+0000", 2023-09-23T15:19:28.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249231+0000", 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0229999999999998e-06 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249240+0000", 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3530000000000001e-06 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249245+0000", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.515e-06 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249297+0000", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1761999999999997e-05 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249422+0000", 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125313 2023-09-23T15:19:28.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249542+0000", 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119733 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.255846+0000", 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063044989999999999 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265483+0000", 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096364410000000008 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547380+0000", 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28189782800000002 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547444+0000", 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3937999999999997e-05 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547508+0000", 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3404999999999993e-05 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:128637 7.d 7:b2b757d2:::benchmark_data_smithi017_422824_object34767:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:16.249256+0000", 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.645468358, 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.298445921, 2023-09-23T15:19:28.013 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 128637 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249256+0000", 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249256+0000", 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249255+0000", 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249257+0000", 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0080000000000001e-06 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249261+0000", 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8179999999999997e-06 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.249264+0000", 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.486e-06 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.252814+0000", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035499030000000001 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.252940+0000", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126105 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.253060+0000", 2023-09-23T15:19:28.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120234 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.256002+0000", 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029416479999999998 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265508+0000", 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095065879999999998 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547604+0000", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28209559099999998 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547654+0000", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0661999999999997e-05 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547702+0000", 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7209000000000002e-05 2023-09-23T15:19:28.019 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:128645 7.d 7:b37c2892:::benchmark_data_smithi017_422824_object34775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:16.265728+0000", 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.628995773, 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29815265699999999, 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.020 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 128645 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265728+0000", 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265728+0000", 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265728+0000", 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265730+0000", 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.996e-06 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265734+0000", 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7629999999999998e-06 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265736+0000", 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.243e-06 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.542559+0000", 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27682349499999997 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.542707+0000", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014771399999999999 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.542856+0000", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014845099999999999 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.556943+0000", 2023-09-23T15:19:28.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014087185 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.561154+0000", 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004211261 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563814+0000", 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026602710000000001 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.025 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563852+0000", 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7997999999999998e-05 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563881+0000", 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8561999999999999e-05 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:128646 7.d 7:bb11e543:::benchmark_data_smithi017_422824_object34776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:16.265747+0000", 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.628976685, 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29821860300000003, 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 128646 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.027 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265747+0000", 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265747+0000", 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265747+0000", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265749+0000", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7460000000000001e-06 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265752+0000", 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.664e-06 2023-09-23T15:19:28.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265755+0000", 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3130000000000001e-06 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547782+0000", 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.282026901 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.547922+0000", 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014004100000000001 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.548078+0000", 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000156778 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.556946+0000", 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088674129999999993 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.561184+0000", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004238538 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563906+0000", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002721453 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563934+0000", 2023-09-23T15:19:28.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8428999999999998e-05 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563966+0000", 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1621000000000001e-05 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:128648 7.d 7:bd8c024a:::benchmark_data_smithi017_422824_object34778:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e251)", 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:16.265765+0000", 2023-09-23T15:19:28.033 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 11.628958738, 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.298356281, 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 128648 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265765+0000", 2023-09-23T15:19:28.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265765+0000", 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265765+0000", 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265767+0000", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.663e-06 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265770+0000", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.399e-06 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.265772+0000", 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.249e-06 2023-09-23T15:19:28.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.553245+0000", 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28747299599999998 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.553360+0000", 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114721 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.553471+0000", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011054599999999999 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.557028+0000", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0035578239999999998 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.563230+0000", 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062011849999999997 2023-09-23T15:19:28.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.564023+0000", 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00079352199999999996 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.564076+0000", 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2741000000000002e-05 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:19:28.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:16.564122+0000", 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5773999999999999e-05 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:19:28.040 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:28.082 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:28.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:28.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:28.136 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:28 smithi017 ceph-mon[215646]: pgmap v733: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:28.141 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:28] "GET /metrics HTTP/1.1" 200 36284 "" "Prometheus/2.43.0" 2023-09-23T15:19:28.141 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:28 smithi017 ceph-mon[222878]: pgmap v733: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:28.156 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:28.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:28.261 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:28.345 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:28.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:28 smithi027 ceph-mon[195233]: pgmap v733: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 608 op/s 2023-09-23T15:19:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:28.352 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:28.453 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:28.487 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:28.622 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:19:28.625 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:28.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:28.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:28.768 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:28.826 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:28.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:28.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:28.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:28.968 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:29.035 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:29.036 INFO:teuthology.orchestra.run.smithi017.stdout:moved 2 / 327 (0.611621%) 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:avg 46.7143 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.28378 -> 5.49582 (expected baseline 6.32778) 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 40 -> 40 pgs (0.856269 -> 0.856269 * mean) 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.3 with 55 -> 55 pgs (1.17737 -> 1.17737 * mean) 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0192 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0211 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T15:19:29.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T15:19:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:29.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:29.043 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:19:29.083 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:29.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:29.188 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:29 smithi017 conmon[202315]: 2023-09-23T15:19:29.035+0000 7fdcb8de7700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T15:19:29.218 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:29.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:29.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:29.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:29.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:30.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:19:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:30 smithi017 ceph-mon[215646]: pgmap v734: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T15:19:30.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:30 smithi017 ceph-mon[215646]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:30.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:30 smithi017 ceph-mon[222878]: pgmap v734: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T15:19:30.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:30 smithi017 ceph-mon[222878]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:30 smithi027 ceph-mon[195233]: pgmap v734: 109 pgs: 109 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 614 op/s 2023-09-23T15:19:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:30 smithi027 ceph-mon[195233]: from='client.55576 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:19:30.613 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:30.616 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:30.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:30.759 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:30.874 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:30.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:30.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:30.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:30.991 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:30.991 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:19:30.991 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:19:30.991 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:31.020 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:31.079 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:31.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:31.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:31.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:31.322 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:31.371 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:31.473 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:31.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:31.590 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:31.622 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:31.807 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:19:31.832 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:31.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:31.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:31.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:31.973 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:32.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:32.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:32.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:32.156 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:32.201 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:32.253 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:19:32.264 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:32.268 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:32 smithi027 ceph-mon[195233]: pgmap v735: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T15:19:32.268 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:19:32.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:32.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:32.309 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:32.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:32.450 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:32 smithi017 ceph-mon[215646]: pgmap v735: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T15:19:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:19:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:32 smithi017 ceph-mon[222878]: pgmap v735: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 652 op/s 2023-09-23T15:19:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:19:33.492 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:33.492 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:19:33.492 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T15:19:33.492 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.492 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.493 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:33 smithi027 ceph-mon[195233]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:33.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:19:33.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T15:19:33.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[215646]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2302265529' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:33 smithi017 ceph-mon[222878]: failed to encode map e253 with expected crc 2023-09-23T15:19:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:33.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:33.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:33.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:33.840 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:33.961 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:33.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:33.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:34.041 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:19:34.041 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:19:34.041 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:19:34.041 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:19:34.041 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:19:34.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:34.193 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:34 smithi027 ceph-mon[195233]: pgmap v737: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:34.378 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:34.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:34 smithi017 ceph-mon[215646]: pgmap v737: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:34.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:34.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:34 smithi017 ceph-mon[222878]: pgmap v737: 109 pgs: 109 active+clean; 3.1 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:34.430 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:34.449 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:34.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:34.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:34.490 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:34.522 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:34.533 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:34.575 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:34.579 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:34.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:34.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:34.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:34.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:34.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.672 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:19:34.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:34.762 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:34.813 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:19:34 smithi027 conmon[174777]: logger=cleanup t=2023-09-23T15:19:34.757697737Z level=info msg="Completed cleanup jobs" duration=340.936986ms 2023-09-23T15:19:34.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:19:34.873 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:34.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:34.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:34.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:35.061 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:35.170 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:35.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:35.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:35.274 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:19:35.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:35.310 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:35.403 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:35.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:35.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:35.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/333944055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:35.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/333944055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:35.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/333944055' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:35.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:35.553 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:35.751 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6] out_osds: [7] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T15:19:35.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:19:35.751 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6] 2023-09-23T15:19:35.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T15:19:35.820 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:35.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:35.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:19:35.923 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:35.971 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:36.019 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:36.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:36.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:36.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:36.240 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:36.403 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:36.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:36.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:36 smithi017 ceph-mon[215646]: pgmap v738: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:36.415 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:36 smithi017 ceph-mon[222878]: pgmap v738: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:36.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:36.516 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:36.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:36 smithi027 ceph-mon[195233]: pgmap v738: 109 pgs: 109 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 655 op/s 2023-09-23T15:19:36.547 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:36.616 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:36.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:36.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:36.628 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:36.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:36.780 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:36.942 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:36.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:36.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:37.007 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:37.007 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:37.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:37.083 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:37.256 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:37.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:37.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:37.387 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:37.433 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:37.436 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:37.536 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:37.638 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:37.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:19:37.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:19:38.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:38 smithi017 ceph-mon[215646]: pgmap v739: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:38.362 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:38] "GET /metrics HTTP/1.1" 200 36287 "" "Prometheus/2.43.0" 2023-09-23T15:19:38.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:38 smithi017 ceph-mon[222878]: pgmap v739: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:38.369 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:38.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:38.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:38.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:38.506 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:38.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:38 smithi027 ceph-mon[195233]: pgmap v739: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 653 op/s 2023-09-23T15:19:38.926 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:38.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:39.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:39.126 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:39.417 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:39.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:39.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:39.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:39.551 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:39.562 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:39.618 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:39.971 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:39.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:40.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:40.217 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:40.293 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:40 smithi017 conmon[215622]: 2023-09-23T15:19:40.063+0000 7f618a1a9700 -1 mon.a@0(leader).osd e253 definitely_dead 0 2023-09-23T15:19:40.344 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.4. 2023-09-23T15:19:40.428 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:40.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:40.433 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:40.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:40.562 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:40 smithi027 ceph-mon[195233]: pgmap v740: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 651 op/s 2023-09-23T15:19:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1044963525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:40 smithi027 ceph-mon[195233]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:19:40.606 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:40.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:40.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[215646]: pgmap v740: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 651 op/s 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1044963525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[215646]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[222878]: pgmap v740: 109 pgs: 109 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 651 op/s 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1044963525' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:19:40.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:40 smithi017 ceph-mon[222878]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:19:40.712 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:40.740 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:41.351 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:19:41.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:41 smithi027 ceph-mon[195233]: failed to encode map e254 with expected crc 2023-09-23T15:19:41.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:19:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:41 smithi027 ceph-mon[195233]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T15:19:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1122542742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:41.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[215646]: failed to encode map e254 with expected crc 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[215646]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1122542742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[222878]: failed to encode map e254 with expected crc 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:19:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[222878]: osdmap e254: 8 total, 8 up, 6 in 2023-09-23T15:19:41.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1122542742' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:41.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:19:42.293 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:42.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:42.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:42.397 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:42.450 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:42.514 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:42.519 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: pgmap v742: 109 pgs: 10 peering, 99 active+clean; 2.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 730 op/s; 9.4 KiB/s, 0 objects/s recovering 2023-09-23T15:19:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T15:19:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:42 smithi027 ceph-mon[195233]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.620 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:42.659 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: pgmap v742: 109 pgs: 10 peering, 99 active+clean; 2.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 730 op/s; 9.4 KiB/s, 0 objects/s recovering 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[222878]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: pgmap v742: 109 pgs: 10 peering, 99 active+clean; 2.7 GiB data, 9.1 GiB used, 527 GiB / 536 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 730 op/s; 9.4 KiB/s, 0 objects/s recovering 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: osdmap e255: 8 total, 8 up, 6 in 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:42 smithi017 ceph-mon[215646]: failed to encode map e255 with expected crc 2023-09-23T15:19:42.720 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:42.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:42.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:42.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:42.920 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:43.034 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:43.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:43.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:43.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:43.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:43.241 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:43.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:43.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:43.347 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:43.384 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:43.439 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.443 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8837 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 253'9090 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 253'9109 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 253'8859 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8735 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 253'8959 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:43.444 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:43.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:43.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:43.546 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:43.558 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:19:43.558 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:19:43.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T15:19:43.619 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8837 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 253'9090 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 253'9109 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 253'8859 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8735 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 253'8959 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:43.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: osdmap e256: 8 total, 8 up, 6 in 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: failed to encode map e256 with expected crc 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.9 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8837 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.1 starting backfill to osd.1 from (0'0,0'0] MAX to 253'9090 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.3 starting backfill to osd.6 from (0'0,0'0] MAX to 253'9109 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.8 starting backfill to osd.5 from (0'0,0'0] MAX to 253'8859 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.4 starting backfill to osd.6 from (0'0,0'0] MAX to 253'8735 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: 7.f starting backfill to osd.5 from (0'0,0'0] MAX to 253'8959 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:43.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:43.646 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:43.896 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:43.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:43.903 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:43.944 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:43.994 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:44.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:44.038 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:44.083 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:44.097 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:44.128 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:44.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:44.208 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:44.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:44.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:44.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:44.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:44.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:19:44.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:44.420 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:44.516 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:44.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:44.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: pgmap v745: 109 pgs: 4 remapped+peering, 10 peering, 95 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 530 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-23T15:19:44.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T15:19:44.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1466932644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: pgmap v745: 109 pgs: 4 remapped+peering, 10 peering, 95 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 530 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.526 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.527 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1466932644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:44.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:44.710 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: pgmap v745: 109 pgs: 4 remapped+peering, 10 peering, 95 active+clean; 2.7 GiB data, 9.0 GiB used, 527 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 530 op/s; 26 KiB/s, 0 objects/s recovering 2023-09-23T15:19:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: osdmap e257: 8 total, 8 up, 6 in 2023-09-23T15:19:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: failed to encode map e257 with expected crc 2023-09-23T15:19:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1466932644' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:44.974 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:19:45.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[215646]: failed to encode map e258 with expected crc 2023-09-23T15:19:45.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[215646]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T15:19:45.423 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[215646]: pgmap v748: 109 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 remapped+peering, 14 peering, 87 active+clean; 2.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 325 op/s; 10342/129810 objects misplaced (7.967%); 5.8 MiB/s, 96 objects/s recovering 2023-09-23T15:19:45.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[222878]: failed to encode map e258 with expected crc 2023-09-23T15:19:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[222878]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T15:19:45.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:45 smithi017 ceph-mon[222878]: pgmap v748: 109 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 remapped+peering, 14 peering, 87 active+clean; 2.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 325 op/s; 10342/129810 objects misplaced (7.967%); 5.8 MiB/s, 96 objects/s recovering 2023-09-23T15:19:45.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:45 smithi027 ceph-mon[195233]: failed to encode map e258 with expected crc 2023-09-23T15:19:45.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:45 smithi027 ceph-mon[195233]: osdmap e258: 8 total, 8 up, 6 in 2023-09-23T15:19:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:45 smithi027 ceph-mon[195233]: pgmap v748: 109 pgs: 2 active+remapped+backfilling, 2 active+remapped+backfill_wait, 4 remapped+peering, 14 peering, 87 active+clean; 2.6 GiB data, 8.9 GiB used, 527 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 325 op/s; 10342/129810 objects misplaced (7.967%); 5.8 MiB/s, 96 objects/s recovering 2023-09-23T15:19:46.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:19:46.470 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:46.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:46.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:46.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:46.611 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:46.663 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:46.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:46.769 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:46.779 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:46.805 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:46.866 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:46.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:46.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:46.873 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:46 smithi027 ceph-mon[195233]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:19:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:46 smithi017 ceph-mon[215646]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:19:46.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:46 smithi017 ceph-mon[222878]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:19:46.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:47.006 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:47.066 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:47.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:47.073 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:47.173 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:47.263 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:47.330 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6] out_osds: [7, 4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T15:19:47.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:19:47.424 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:47.428 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:47.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:47.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:47.541 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:47.652 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:48.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:48 smithi027 ceph-mon[195233]: pgmap v749: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 remapped+peering, 4 peering, 95 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 353 op/s; 15385/128127 objects misplaced (12.008%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T15:19:48.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:48 smithi017 ceph-mon[215646]: pgmap v749: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 remapped+peering, 4 peering, 95 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 353 op/s; 15385/128127 objects misplaced (12.008%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T15:19:48.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:48 smithi017 ceph-mon[222878]: pgmap v749: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 remapped+peering, 4 peering, 95 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 16 KiB/s rd, 42 KiB/s wr, 353 op/s; 15385/128127 objects misplaced (12.008%); 8.4 MiB/s, 136 objects/s recovering 2023-09-23T15:19:48.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:48] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2023-09-23T15:19:48.640 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:19:48.787 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:48.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:48.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:48.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:48.922 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:48.972 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:48.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:49.080 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:49.110 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:49.175 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:49.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:49.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:49.182 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:49 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:19:49.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:49.319 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:49.365 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:49 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:19:49.366 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:49 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:19:49.376 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:49.589 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:49.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:49.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:49.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:49.734 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:49.796 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:49.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:49.803 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:49.903 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:49.933 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:49.996 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:50.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:50.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:50.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:50.171 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:50.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:50 smithi027 ceph-mon[195233]: pgmap v750: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 peering, 99 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 6.5 KiB/s rd, 18 KiB/s wr, 298 op/s; 15385/127608 objects misplaced (12.056%); 7.0 MiB/s, 116 objects/s recovering 2023-09-23T15:19:50.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:50 smithi017 ceph-mon[215646]: pgmap v750: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 peering, 99 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 6.5 KiB/s rd, 18 KiB/s wr, 298 op/s; 15385/127608 objects misplaced (12.056%); 7.0 MiB/s, 116 objects/s recovering 2023-09-23T15:19:50.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:50 smithi017 ceph-mon[222878]: pgmap v750: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 4 peering, 99 active+clean; 2.6 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 6.5 KiB/s rd, 18 KiB/s wr, 298 op/s; 15385/127608 objects misplaced (12.056%); 7.0 MiB/s, 116 objects/s recovering 2023-09-23T15:19:50.843 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:50.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:50.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:50.988 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:51.048 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:51.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:51.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:51.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:51.182 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:51.241 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:51.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:51.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:51.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:51.381 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:51.475 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:51.669 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:51.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:51.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:51.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:51.875 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:52.103 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:19:52.103 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:19:52.103 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:19:52.103 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:19:52.104 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:19:52.148 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[215646]: pgmap v751: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 583 op/s; 12219/119193 objects misplaced (10.251%); 22 MiB/s, 355 objects/s recovering 2023-09-23T15:19:52.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/405129032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:52.149 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:52.206 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:19:52.207 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:52.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:52.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:52.326 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:52.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[222878]: pgmap v751: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 583 op/s; 12219/119193 objects misplaced (10.251%); 22 MiB/s, 355 objects/s recovering 2023-09-23T15:19:52.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/405129032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:52.409 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:52.417 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:52 smithi027 ceph-mon[195233]: pgmap v751: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 583 op/s; 12219/119193 objects misplaced (10.251%); 22 MiB/s, 355 objects/s recovering 2023-09-23T15:19:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/405129032' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:52.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:19:53.434 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[215646]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[215646]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[215646]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[222878]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[222878]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T15:19:53.435 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:53 smithi017 ceph-mon[222878]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:53 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:19:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:53 smithi027 ceph-mon[195233]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:53 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:19:53.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:53 smithi027 ceph-mon[195233]: osdmap e259: 8 total, 8 up, 6 in 2023-09-23T15:19:53.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:53 smithi027 ceph-mon[195233]: failed to encode map e259 with expected crc 2023-09-23T15:19:53.659 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:53.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:53.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:53.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:53.804 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:53.864 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:53.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:53.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:53.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:54.057 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:54.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:54 smithi017 ceph-mon[215646]: pgmap v753: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 12219/118743 objects misplaced (10.290%); 19 MiB/s, 299 objects/s recovering 2023-09-23T15:19:54.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:54 smithi017 ceph-mon[222878]: pgmap v753: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 12219/118743 objects misplaced (10.290%); 19 MiB/s, 299 objects/s recovering 2023-09-23T15:19:54.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:54 smithi027 ceph-mon[195233]: pgmap v753: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.4 GiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 488 op/s; 12219/118743 objects misplaced (10.290%); 19 MiB/s, 299 objects/s recovering 2023-09-23T15:19:55.010 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:55.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:55.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:55.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:55.158 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:55.220 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:55.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:55.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:19:55.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:19:55.358 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:55.414 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:19:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:55.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:19:55.520 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:19:55.636 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:55.863 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:19:55.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:55.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:19:55.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:56.087 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:56.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:56 smithi017 ceph-mon[222878]: pgmap v754: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 9781/112641 objects misplaced (8.683%); 27 MiB/s, 433 objects/s recovering 2023-09-23T15:19:56.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:56 smithi017 ceph-mon[215646]: pgmap v754: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 9781/112641 objects misplaced (8.683%); 27 MiB/s, 433 objects/s recovering 2023-09-23T15:19:56.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:56 smithi027 ceph-mon[195233]: pgmap v754: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.3 GiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 571 op/s; 9781/112641 objects misplaced (8.683%); 27 MiB/s, 433 objects/s recovering 2023-09-23T15:19:56.674 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:19:56.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:56.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:19:56.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:56.811 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:56.886 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:56.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:56.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:56.964 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:19:56.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:19:57.029 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:57.075 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:19:57.161 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:19:57.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:57.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:19:57.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:57.305 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:57.359 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:57.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:57.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:57.426 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.426 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3724081381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[215646]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3724081381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:19:57.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:57 smithi017 ceph-mon[222878]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T15:19:57.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: failed to encode map e260 with expected crc 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: osdmap e260: 8 total, 8 up, 6 in 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3724081381' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:19:57.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:57 smithi027 ceph-mon[195233]: osdmap e261: 8 total, 8 up, 6 in 2023-09-23T15:19:57.497 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:57.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:19:57.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:19:57.546 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:57.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:57.626 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:19:57.651 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:19:57.663 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:19:57.679 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:57.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:19:57.728 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:19:57.731 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:19:57.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:57.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:19:57.760 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:19:57.788 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:19:57.816 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:19:57.819 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.819 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.820 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:19:57.834 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:19:57.861 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:57.906 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:19:57.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:57.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:19:58.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:19:58.047 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:58.099 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:19:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:19:58] "GET /metrics HTTP/1.1" 200 36370 "" "Prometheus/2.43.0" 2023-09-23T15:19:58.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e260 with expected crc 2023-09-23T15:19:58.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: pgmap v756: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 9017/109617 objects misplaced (8.226%); 34 MiB/s, 548 objects/s recovering 2023-09-23T15:19:58.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3873535934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e260 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: pgmap v756: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 9017/109617 objects misplaced (8.226%); 34 MiB/s, 548 objects/s recovering 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3873535934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T15:19:58.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:58.448 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e260 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: pgmap v756: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.2 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 749 op/s; 9017/109617 objects misplaced (8.226%); 34 MiB/s, 548 objects/s recovering 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e261 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3873535934' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: osdmap e262: 8 total, 8 up, 6 in 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: failed to encode map e262 with expected crc 2023-09-23T15:19:58.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:19:58.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:19:59.449 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:19:59.472 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:19:59.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:59.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:19:59.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:59 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:19:59.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:19:59 smithi017 ceph-mon[215646]: failed to encode map e262 with expected crc 2023-09-23T15:19:59.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:59 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:19:59.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:19:59 smithi017 ceph-mon[222878]: failed to encode map e262 with expected crc 2023-09-23T15:19:59.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:19:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:59 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:19:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:19:59 smithi027 ceph-mon[195233]: failed to encode map e262 with expected crc 2023-09-23T15:19:59.615 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:19:59.670 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:19:59.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:19:59.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:19:59.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:19:59.896 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:00.402 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:00.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:00.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:00.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:00.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:00 smithi027 ceph-mon[195233]: pgmap v759: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 9017/109032 objects misplaced (8.270%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:20:00.537 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:00 smithi027 ceph-mon[195233]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T15:20:00.538 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:00 smithi027 ceph-mon[195233]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T15:20:00.543 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:00.601 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:00.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:00.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[222878]: pgmap v759: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 9017/109032 objects misplaced (8.270%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:20:00.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[222878]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T15:20:00.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[222878]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T15:20:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[215646]: pgmap v759: 109 pgs: 3 active+remapped+backfilling, 3 active+remapped+backfill_wait, 103 active+clean; 2.2 GiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 539 op/s; 9017/109032 objects misplaced (8.270%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:20:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[215646]: Health detail: HEALTH_WARN nodeep-scrub flag(s) set 2023-09-23T15:20:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:00 smithi017 ceph-mon[215646]: [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set 2023-09-23T15:20:00.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:00.750 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:00.813 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:00.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:00.818 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:00.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:00.950 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:01.201 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:01.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:01.256 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:01.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:01.331 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:01.336 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:01.389 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:01.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:01.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:01.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:20:01.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: failed to encode map e262 with expected crc 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:20:01.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: failed to encode map e262 with expected crc 2023-09-23T15:20:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:01 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:20:01.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:01.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:20:01.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: osdmap e263: 8 total, 8 up, 6 in 2023-09-23T15:20:01.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e263 with expected crc 2023-09-23T15:20:01.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: failed to encode map e262 with expected crc 2023-09-23T15:20:01.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:01 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:20:01.531 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:01.588 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:01.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:01.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:01.773 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:01.992 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:02.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:02.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:02.016 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:20:02.016 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:20:02.108 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:02.137 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:02.197 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:02.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:02.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:02.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:02.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[215646]: pgmap v761: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 5462/102579 objects misplaced (5.325%); 44 MiB/s, 704 objects/s recovering 2023-09-23T15:20:02.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[215646]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[215646]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T15:20:02.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[215646]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.378 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[215646]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[222878]: pgmap v761: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 5462/102579 objects misplaced (5.325%); 44 MiB/s, 704 objects/s recovering 2023-09-23T15:20:02.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[222878]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[222878]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T15:20:02.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[222878]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.379 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:02 smithi017 ceph-mon[222878]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.389 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:02.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:20:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:02 smithi027 ceph-mon[195233]: pgmap v761: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 530 op/s; 5462/102579 objects misplaced (5.325%); 44 MiB/s, 704 objects/s recovering 2023-09-23T15:20:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:02 smithi027 ceph-mon[195233]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:02 smithi027 ceph-mon[195233]: osdmap e264: 8 total, 8 up, 6 in 2023-09-23T15:20:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:02 smithi027 ceph-mon[195233]: failed to encode map e264 with expected crc 2023-09-23T15:20:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:02 smithi027 ceph-mon[195233]: failed to encode map e264 with expected crc 2023-09-23T15:20:03.586 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:03 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:20:03.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:03 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:20:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:03 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:20:04.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:04 smithi017 ceph-mon[222878]: pgmap v763: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 5462/102102 objects misplaced (5.350%); 35 MiB/s, 563 objects/s recovering 2023-09-23T15:20:04.592 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:04 smithi017 ceph-mon[215646]: pgmap v763: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 5462/102102 objects misplaced (5.350%); 35 MiB/s, 563 objects/s recovering 2023-09-23T15:20:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:04 smithi027 ceph-mon[195233]: pgmap v763: 109 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+remapped+backfill_wait, 105 active+clean; 2.1 GiB data, 7.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 417 op/s; 5462/102102 objects misplaced (5.350%); 35 MiB/s, 563 objects/s recovering 2023-09-23T15:20:04.725 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:04.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:04.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:04.831 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:04.858 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:04.901 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:04.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:04.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:05.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:05.039 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:05.270 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[215646]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.403 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:05.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:05.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:05.507 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:05.534 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:05.583 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.588 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:05 smithi027 ceph-mon[195233]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:05.588 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: osdmap e265: 8 total, 8 up, 6 in 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.602 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.603 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:05 smithi017 ceph-mon[222878]: failed to encode map e265 with expected crc 2023-09-23T15:20:05.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:05.721 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:05.781 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:05.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:05.785 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:05.798 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:05.886 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:05.914 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:05.968 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:05.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:05.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:06.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:06.170 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:06.299 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:06.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:06.367 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: pgmap v765: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 105 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 4057/96087 objects misplaced (4.222%); 45 MiB/s, 712 objects/s recovering 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.389 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[215646]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: pgmap v765: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 105 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 4057/96087 objects misplaced (4.222%); 45 MiB/s, 712 objects/s recovering 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:06 smithi017 ceph-mon[222878]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.407 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:06.525 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: pgmap v765: 109 pgs: 1 active+clean+remapped, 1 active+remapped+backfill_wait, 1 peering, 1 active+remapped+backfilling, 105 active+clean; 2.0 GiB data, 7.7 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 718 op/s; 4057/96087 objects misplaced (4.222%); 45 MiB/s, 712 objects/s recovering 2023-09-23T15:20:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: osdmap e266: 8 total, 8 up, 6 in 2023-09-23T15:20:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:06 smithi027 ceph-mon[195233]: failed to encode map e266 with expected crc 2023-09-23T15:20:06.818 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:06.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:06.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:07.008 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:07.101 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:20:07.144 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:20:07.168 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:07.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:07.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:07.274 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:07.330 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:07.394 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2895676644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:20:07.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[215646]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2895676644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:07.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:07 smithi017 ceph-mon[222878]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T15:20:07.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:07.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:07.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:07.527 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2895676644' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2644623164' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:07.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:07 smithi027 ceph-mon[195233]: osdmap e267: 8 total, 8 up, 6 in 2023-09-23T15:20:08.148 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:08] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T15:20:08.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: pgmap v767: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 2280/93666 objects misplaced (2.434%); 29 MiB/s, 469 objects/s recovering 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: 1.d deep-scrub starts 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[222878]: failed to encode map e268 with expected crc 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: pgmap v767: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 2280/93666 objects misplaced (2.434%); 29 MiB/s, 469 objects/s recovering 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: 1.d deep-scrub starts 2023-09-23T15:20:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T15:20:08.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:08 smithi017 ceph-mon[215646]: failed to encode map e268 with expected crc 2023-09-23T15:20:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: pgmap v767: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 495 op/s; 2280/93666 objects misplaced (2.434%); 29 MiB/s, 469 objects/s recovering 2023-09-23T15:20:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: failed to encode map e267 with expected crc 2023-09-23T15:20:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: 1.d deep-scrub starts 2023-09-23T15:20:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: osdmap e268: 8 total, 8 up, 6 in 2023-09-23T15:20:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:08 smithi027 ceph-mon[195233]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.307 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:09.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:09.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:09.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:09.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[215646]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[215646]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:20:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[222878]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[222878]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:09 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:20:09.449 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:09.501 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:09.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:09.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:09.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:09.701 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:09 smithi027 ceph-mon[195233]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:09 smithi027 ceph-mon[195233]: failed to encode map e268 with expected crc 2023-09-23T15:20:09.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:09 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:20:09.882 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:09.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:09.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:09.992 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:10.020 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:10.105 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:10.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:10.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:10.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:10.244 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:10.302 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:10.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:10.306 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:10.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:10.440 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:10.498 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:10.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:10.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:10.507 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:10 smithi027 ceph-mon[195233]: pgmap v770: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 2280/93186 objects misplaced (2.447%); 24 MiB/s, 387 objects/s recovering 2023-09-23T15:20:10.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:10.634 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:20:10.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:10 smithi017 ceph-mon[215646]: pgmap v770: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 2280/93186 objects misplaced (2.447%); 24 MiB/s, 387 objects/s recovering 2023-09-23T15:20:10.639 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:10 smithi017 ceph-mon[222878]: pgmap v770: 109 pgs: 1 active+remapped+backfilling, 1 active+clean+remapped, 1 active+remapped+backfill_wait, 106 active+clean; 1.9 GiB data, 7.5 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 361 op/s; 2280/93186 objects misplaced (2.447%); 24 MiB/s, 387 objects/s recovering 2023-09-23T15:20:10.639 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:10.706 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:10.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:10.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:10.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:10.844 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:10.903 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:10.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:10.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:10.929 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:20:10.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:20:11.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:11.043 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:20:11.082 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:20:11.102 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:11.128 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:20:11.179 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:20:11.217 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:20:11.261 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:20:11.296 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.297 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:20:11.333 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:11.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:11.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:11.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:160062 7.d e268/223 7:b9a8d0f4:::benchmark_data_smithi017_422824_object66192:head v 268'10175, mlcod=268'10172)", 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:11.427999+0000", 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.034224407999999998, 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.034257168999999997, 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:11.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.427999+0000", 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.427999+0000", 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:11.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.427998+0000", 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.428007+0000", 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.456e-06 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.428013+0000", 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8590000000000001e-06 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.428017+0000", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1989999999999999e-06 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.432563+0000", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045457609999999997 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.432581+0000", 2023-09-23T15:20:11.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8128999999999999e-05 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:160081 7.d e268/223 7:b6573496:::benchmark_data_smithi017_422824_object66211:head v 268'10176, mlcod=268'10173)", 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:11.451690+0000", 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.010533483, 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010642453, 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:11.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451690+0000", 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451690+0000", 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451689+0000", 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451699+0000", 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5200000000000003e-06 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:11.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451706+0000", 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7060000000000004e-06 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.451712+0000", 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.412e-06 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.452517+0000", 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00080469999999999999 2023-09-23T15:20:11.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.452538+0000", 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1160000000000001e-05 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:160080 7.f 7:fd8e8a7d:::benchmark_data_smithi017_422824_object66210:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:11.455134+0000", 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0070889220000000001, 2023-09-23T15:20:11.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0073010139999999998, 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 160080 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455134+0000", 2023-09-23T15:20:11.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455134+0000", 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455133+0000", 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:11.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455137+0000", 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4360000000000004e-06 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455147+0000", 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3589999999999997e-06 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.455154+0000", 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6009999999999997e-06 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458981+0000", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038268299999999998 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.459103+0000", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121766 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.459220+0000", 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116958 2023-09-23T15:20:11.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.462348+0000", 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031283280000000001 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:160084 7.d e268/223)", 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:11.457530+0000", 2023-09-23T15:20:11.482 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0046931969999999996, 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049853889999999998, 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457530+0000", 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:11.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457530+0000", 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457529+0000", 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457547+0000", 2023-09-23T15:20:11.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7700999999999999e-05 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457570+0000", 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2370000000000001e-05 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.457578+0000", 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8820000000000007e-06 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:11.485 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:160067 7.e e268/223 7:7f2b7174:::benchmark_data_smithi017_422824_object66197:head v 268'10095, mlcod=268'10093)", 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:11.458892+0000", 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0033309020000000002, 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036764110000000001, 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:11.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458892+0000", 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458892+0000", 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458892+0000", 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458898+0000", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.874e-06 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458908+0000", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0701e-05 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458915+0000", 2023-09-23T15:20:11.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8000000000000001e-06 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458962+0000", 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7113000000000001e-05 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:11.458985+0000", 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2303999999999999e-05 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.489 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 5 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:20:11.490 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:11.543 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:11.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:11.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:11.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: 1.e scrub starts 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: 1.e scrub ok 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: pgmap v771: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 972/87015 objects misplaced (1.117%); 30 MiB/s, 475 objects/s recovering 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/673379416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:11.683 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:11 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:20:11.686 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:20:11.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: 1.e scrub starts 2023-09-23T15:20:11.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: 1.e scrub ok 2023-09-23T15:20:11.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: pgmap v771: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 972/87015 objects misplaced (1.117%); 30 MiB/s, 475 objects/s recovering 2023-09-23T15:20:11.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/673379416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:11.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: 1.e scrub starts 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: 1.e scrub ok 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: pgmap v771: 109 pgs: 1 active+remapped+backfilling, 108 active+clean; 1.8 GiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 503 op/s; 972/87015 objects misplaced (1.117%); 30 MiB/s, 475 objects/s recovering 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/673379416' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:11.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:11 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:20:11.792 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:11.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:11.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:11.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:11.989 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:12 smithi017 ceph-mon[215646]: 1.5 deep-scrub starts 2023-09-23T15:20:12.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:12 smithi017 ceph-mon[222878]: 1.5 deep-scrub starts 2023-09-23T15:20:12.746 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:12.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:12.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:12.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:12 smithi027 ceph-mon[195233]: 1.5 deep-scrub starts 2023-09-23T15:20:12.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:12.929 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:13.381 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:13.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:13.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:13.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:20:13.490 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:13.553 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:13.590 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[222878]: pgmap v772: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 107 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 972/86508 objects misplaced (1.124%); 24 MiB/s, 391 objects/s recovering 2023-09-23T15:20:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:20:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[222878]: 1.d scrub ok 2023-09-23T15:20:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[215646]: pgmap v772: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 107 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 972/86508 objects misplaced (1.124%); 24 MiB/s, 391 objects/s recovering 2023-09-23T15:20:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:20:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[215646]: 1.d scrub ok 2023-09-23T15:20:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:13.756 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:13.770 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:13.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:13.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:13 smithi027 ceph-mon[195233]: pgmap v772: 109 pgs: 1 active+clean+scrubbing, 1 active+remapped+backfilling, 107 active+clean; 1.8 GiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 413 op/s; 972/86508 objects misplaced (1.124%); 24 MiB/s, 391 objects/s recovering 2023-09-23T15:20:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:13 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:20:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:13 smithi027 ceph-mon[195233]: 1.d scrub ok 2023-09-23T15:20:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:13.878 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:13.906 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:13.954 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:14.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:14.093 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:14.292 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:14.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:14.295 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:14.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:14.429 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:14.636 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.638 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:14.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:14.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:14.768 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.768 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: osdmap e269: 8 total, 8 up, 6 in 2023-09-23T15:20:14.768 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.769 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.769 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: failed to encode map e269 with expected crc 2023-09-23T15:20:14.769 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.769 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:14.771 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:14.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:20:14.815 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:14.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:14.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:14.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:14.969 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: pgmap v775: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 20 MiB/s, 324 objects/s recovering 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[222878]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T15:20:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: pgmap v775: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 20 MiB/s, 324 objects/s recovering 2023-09-23T15:20:15.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:15 smithi017 ceph-mon[215646]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: osdmap e270: 8 total, 8 up, 6 in 2023-09-23T15:20:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: pgmap v775: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 636 op/s; 20 MiB/s, 324 objects/s recovering 2023-09-23T15:20:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:15 smithi027 ceph-mon[195233]: failed to encode map e270 with expected crc 2023-09-23T15:20:15.866 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:15.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:15.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:15.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:16.079 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:16.317 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:16.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:16.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:16.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:16.455 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:16.507 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:16.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:16 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:20:16.612 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:16 smithi017 ceph-mon[215646]: 1.5 scrub ok 2023-09-23T15:20:16.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:16.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:16 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:20:16.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:16 smithi017 ceph-mon[222878]: 1.5 scrub ok 2023-09-23T15:20:16.619 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:16.642 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:16.689 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:16.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:16.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:16.726 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:16.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:16 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:20:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:16 smithi027 ceph-mon[195233]: 1.5 scrub ok 2023-09-23T15:20:16.878 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:17.116 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:17.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:17.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:17.264 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:17.317 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:17.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:17.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:17.424 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:17.504 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:18.094 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:20:18.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[215646]: pgmap v776: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 16 MiB/s, 263 objects/s recovering 2023-09-23T15:20:18.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1993015656' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.095 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[222878]: pgmap v776: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 16 MiB/s, 263 objects/s recovering 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1993015656' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:18.096 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:18] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T15:20:18.097 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:20:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:18 smithi027 ceph-mon[195233]: pgmap v776: 109 pgs: 1 peering, 1 active+clean+scrubbing, 107 active+clean; 1.6 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 617 op/s; 16 MiB/s, 263 objects/s recovering 2023-09-23T15:20:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1993015656' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:20:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:18.513 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:18.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:18.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:18.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:18.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:19.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[215646]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:19 smithi017 ceph-mon[222878]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:20:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2605092242' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: osdmap e271: 8 total, 8 up, 6 in 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:19 smithi027 ceph-mon[195233]: failed to encode map e271 with expected crc 2023-09-23T15:20:19.947 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:19.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:19.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:20.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:20.094 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:20.110 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:20 smithi017 ceph-mon[215646]: pgmap v778: 109 pgs: 1 peering, 108 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 9.5 MiB/s, 152 objects/s recovering 2023-09-23T15:20:20.110 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:20 smithi017 ceph-mon[222878]: pgmap v778: 109 pgs: 1 peering, 108 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 9.5 MiB/s, 152 objects/s recovering 2023-09-23T15:20:20.145 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:20.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:20.153 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:20.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:20.330 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:20.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:20 smithi027 ceph-mon[195233]: pgmap v778: 109 pgs: 1 peering, 108 active+clean; 1.6 GiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 475 op/s; 9.5 MiB/s, 152 objects/s recovering 2023-09-23T15:20:20.467 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:20:20.562 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:20.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:20.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:20.839 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:21.016 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:20:21.016 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:20:21.016 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:20:21.016 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:20:21.017 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:20:21.366 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:21.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:21.377 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:21.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:21.507 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:21.851 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:20:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:22 smithi017 ceph-mon[222878]: pgmap v779: 109 pgs: 109 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:20:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:22 smithi017 ceph-mon[215646]: pgmap v779: 109 pgs: 109 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:20:22.367 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:22.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:22.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:22.408 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:20:22.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:20:22.457 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:20:22.492 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:22.523 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:20:22.566 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:20:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:22 smithi027 ceph-mon[195233]: pgmap v779: 109 pgs: 109 active+clean; 1.4 GiB data, 6.4 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 759 op/s; 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:20:22.599 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:22.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:22.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:20:22.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:20:22.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:20:22.736 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:20:22.783 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:22.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.786 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:20:22.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:22.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:22.832 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:22.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:22.996 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:23.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3262888777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:23.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3262888777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3262888777' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:23.835 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:23.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:23.856 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:23.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:24.028 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:24.403 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:24 smithi017 ceph-mon[222878]: pgmap v780: 109 pgs: 109 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T15:20:24.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:24 smithi017 ceph-mon[215646]: pgmap v780: 109 pgs: 109 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T15:20:24.550 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:24.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:24.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:24 smithi027 ceph-mon[195233]: pgmap v780: 109 pgs: 109 active+clean; 1.4 GiB data, 6.3 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 522 op/s 2023-09-23T15:20:24.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:24.755 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:25.551 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:25.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:25.561 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:25.661 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:25.695 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:25.754 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:25.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:25.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:25.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:25.890 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:25.934 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:25.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:26.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:26.131 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:26.273 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:20:26.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:26 smithi017 ceph-mon[222878]: pgmap v781: 109 pgs: 109 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 568 op/s 2023-09-23T15:20:26.376 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:26 smithi017 ceph-mon[215646]: pgmap v781: 109 pgs: 109 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 568 op/s 2023-09-23T15:20:26.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:26 smithi027 ceph-mon[195233]: pgmap v781: 109 pgs: 109 active+clean; 1.3 GiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 568 op/s 2023-09-23T15:20:26.700 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:26.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:26.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:26.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:26.843 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:26.892 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:26.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:26.997 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:27.141 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:27.314 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:27.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:27.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:27.435 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:27.547 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:27.666 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:27.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:27.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:27.775 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:27.851 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:27.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[222878]: pgmap v782: 109 pgs: 109 active+clean; 1.2 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2039185053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[215646]: pgmap v782: 109 pgs: 109 active+clean; 1.2 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2039185053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:27.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:27.978 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:27.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:28.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:27 smithi027 ceph-mon[195233]: pgmap v782: 109 pgs: 109 active+clean; 1.2 GiB data, 5.9 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 593 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:27 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2039185053' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:20:28.168 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:28.236 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:20:28.240 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:20:28.280 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:28] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1871365336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T15:20:28.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1871365336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:28.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]: dispatch 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]: dispatch 2023-09-23T15:20:28.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:20:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:20:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1871365336' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: osdmap e272: 8 total, 8 up, 6 in 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]: dispatch 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]: dispatch 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]: dispatch 2023-09-23T15:20:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:29.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:29.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:29.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:29.800 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:29.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:20:29.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:29.974 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:30.212 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:30.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:30.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:30.282 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.282 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: pgmap v784: 109 pgs: 109 active+clean; 1.2 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: failed to encode map e273 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]': finished 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]': finished 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[215646]: failed to encode map e273 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.283 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: pgmap v784: 109 pgs: 109 active+clean; 1.2 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: failed to encode map e273 with expected crc 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]': finished 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]': finished 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T15:20:30.284 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:30 smithi017 ceph-mon[222878]: failed to encode map e273 with expected crc 2023-09-23T15:20:30.335 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:30.455 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:20:30.455 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:20:30.468 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: failed to encode map e272 with expected crc 2023-09-23T15:20:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: pgmap v784: 109 pgs: 109 active+clean; 1.2 GiB data, 5.8 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 602 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: failed to encode map e273 with expected crc 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.7", "id": [3, 2]}]': finished 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.d", "id": [5, 2]}]': finished 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.5", "id": [3, 2]}]': finished 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: osdmap e273: 8 total, 8 up, 6 in 2023-09-23T15:20:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:30 smithi027 ceph-mon[195233]: failed to encode map e273 with expected crc 2023-09-23T15:20:31.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[215646]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T15:20:31.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[215646]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[215646]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[215646]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[222878]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T15:20:31.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[222878]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[222878]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:31 smithi017 ceph-mon[222878]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:31 smithi027 ceph-mon[195233]: osdmap e274: 8 total, 8 up, 6 in 2023-09-23T15:20:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:31 smithi027 ceph-mon[195233]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:31 smithi027 ceph-mon[195233]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:31 smithi027 ceph-mon[195233]: failed to encode map e274 with expected crc 2023-09-23T15:20:31.623 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:20:31.680 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:31.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:31.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:31.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:31.814 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:20:31.822 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:31.876 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:31.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:31.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:31.981 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:32.160 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:32.317 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:32.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:32.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:32.375 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:32.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:32.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:32 smithi017 ceph-mon[222878]: pgmap v787: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T15:20:32.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:32 smithi017 ceph-mon[222878]: failed to encode map e274 with expected crc 2023-09-23T15:20:32.524 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:32.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:32 smithi017 ceph-mon[215646]: pgmap v787: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T15:20:32.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:32 smithi017 ceph-mon[215646]: failed to encode map e274 with expected crc 2023-09-23T15:20:32.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:32 smithi027 ceph-mon[195233]: pgmap v787: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 718 op/s 2023-09-23T15:20:32.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:32 smithi027 ceph-mon[195233]: failed to encode map e274 with expected crc 2023-09-23T15:20:32.771 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:32.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:32.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:32.879 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:32.955 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:33.035 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:33.046 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:33.057 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:20:33.093 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:20:33.105 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:33.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:33.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:33.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:20:33.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:20:33.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:33.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:20:33.254 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:33.276 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:20:33.311 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2903991662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:33.312 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2903991662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:33.312 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:33.316 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:20:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:33.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:33.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:20:33.385 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.385 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.385 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.386 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:20:33.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:33.508 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2903991662' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:34.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:34 smithi017 ceph-mon[215646]: pgmap v788: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T15:20:34.554 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:34 smithi017 ceph-mon[222878]: pgmap v788: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T15:20:34.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:34 smithi027 ceph-mon[195233]: pgmap v788: 109 pgs: 1 peering, 108 active+clean; 1.0 GiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 577 op/s 2023-09-23T15:20:35.000 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:35.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:35.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:35.111 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:35.140 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:35.190 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:35.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:35.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:35.295 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:35.325 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:35.376 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:35.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:35.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:35.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:35.556 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:36.115 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:36.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:36.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:36.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:36.301 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:36.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:36 smithi017 ceph-mon[222878]: pgmap v789: 109 pgs: 109 active+clean; 906 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T15:20:36.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:36 smithi017 ceph-mon[215646]: pgmap v789: 109 pgs: 109 active+clean; 906 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T15:20:36.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:36 smithi027 ceph-mon[195233]: pgmap v789: 109 pgs: 109 active+clean; 906 MiB data, 4.9 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 852 op/s 2023-09-23T15:20:36.618 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:36.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:36.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:36.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:36.769 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:36.826 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:36.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:36.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:36.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:36.964 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:37.010 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:37.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:37.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:37.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:37.213 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:37.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:20:37.685 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:37.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:37.821 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:37.951 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:38.093 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:38.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:38.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:38.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:38] "GET /metrics HTTP/1.1" 200 36349 "" "Prometheus/2.43.0" 2023-09-23T15:20:38.200 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:38.304 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:38.355 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:20:38.445 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:38.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:38.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:38.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:38 smithi027 ceph-mon[195233]: pgmap v790: 109 pgs: 109 active+clean; 838 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T15:20:38.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3808632832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.586 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:38 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.588 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:38.640 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:38.644 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:20:38.644 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:177366 7.8 7:1d0ca04f:::benchmark_data_smithi017_422824_object83496:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:38.612617+0000", 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.022178099, 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022215817999999998, 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 177366 2023-09-23T15:20:38.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612617+0000", 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612617+0000", 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612615+0000", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999976 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612625+0000", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8840000000000006e-06 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612641+0000", 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5696e-05 2023-09-23T15:20:38.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.612650+0000", 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2159999999999995e-06 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.613160+0000", 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00050980600000000002 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.613296+0000", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013619899999999999 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.613431+0000", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013484099999999999 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.619513+0000", 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060821620000000003 2023-09-23T15:20:38.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.621879+0000", 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023655719999999998 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:177380 7.8 7:12b0e109:::benchmark_data_smithi017_422824_object83510:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e275)", 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:38.626565+0000", 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0082308850000000003, 2023-09-23T15:20:38.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083660160000000004, 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 177380 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626565+0000", 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:38.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626565+0000", 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626563+0000", 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:38.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626571+0000", 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7829999999999999e-06 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626584+0000", 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3468999999999999e-05 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626593+0000", 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8240000000000006e-06 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626653+0000", 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9803999999999999e-05 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626799+0000", 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014598399999999999 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:38.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.626931+0000", 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013179000000000001 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:38.632262+0000", 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0053314779999999997 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2023-09-23T15:20:38.655 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:20:38.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:38.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:38.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[222878]: pgmap v790: 109 pgs: 109 active+clean; 838 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T15:20:38.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3808632832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[215646]: pgmap v790: 109 pgs: 109 active+clean; 838 MiB data, 4.6 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 819 op/s 2023-09-23T15:20:38.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3808632832' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.673 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:38 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:20:38.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:38.786 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:38.841 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:38.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:38.845 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:38.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:39.020 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/592930054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:39.309 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/592930054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:39.310 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:39.327 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:20:39.491 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:20:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: osdmap e275: 8 total, 8 up, 6 in 2023-09-23T15:20:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: failed to encode map e275 with expected crc 2023-09-23T15:20:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/592930054' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:39.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:20:40.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:20:40.279 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:40.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:40.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:40.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: pgmap v792: 109 pgs: 109 active+clean; 823 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 548 op/s 2023-09-23T15:20:40.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:40.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:20:40.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[215646]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: pgmap v792: 109 pgs: 109 active+clean; 823 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 548 op/s 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:20:40.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T15:20:40.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:40 smithi017 ceph-mon[222878]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:40.419 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:40.471 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:40.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:40.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:40.579 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: pgmap v792: 109 pgs: 109 active+clean; 823 MiB data, 4.5 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 548 op/s 2023-09-23T15:20:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:20:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:20:40.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: osdmap e276: 8 total, 8 up, 6 in 2023-09-23T15:20:40.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:40 smithi027 ceph-mon[195233]: failed to encode map e276 with expected crc 2023-09-23T15:20:40.606 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:40.651 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:40.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:40.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:40.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:40.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:20:40.850 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:41.176 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:41.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:41.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:41.291 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:41.386 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:41.628 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:41.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:41.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:41.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:41.760 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:41.804 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:41.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:41.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:41.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:41.937 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:41.981 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:41.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:41.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:42.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:20:42.177 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:42.409 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:20:42.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:42.416 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:20:42.446 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:42.517 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:42.581 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:42 smithi027 ceph-mon[195233]: pgmap v794: 109 pgs: 109 active+clean; 624 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T15:20:42.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:42 smithi017 ceph-mon[215646]: pgmap v794: 109 pgs: 109 active+clean; 624 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T15:20:42.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:42 smithi017 ceph-mon[222878]: pgmap v794: 109 pgs: 109 active+clean; 624 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 841 op/s 2023-09-23T15:20:42.638 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:42.813 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:42.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:42.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:42.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:43.016 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:43.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:20:43.224 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:20:43.280 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:43.282 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:20:43.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:43.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:43.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:20:43.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:43.396 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:20:43.458 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:20:43.492 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:43.506 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:20:43.560 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:20:43.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:43.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/554659915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:43.604 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:20:43.619 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.620 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:20:43.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:43.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/554659915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:43.653 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:43.654 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/554659915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:44 smithi027 ceph-mon[195233]: pgmap v795: 109 pgs: 109 active+clean; 612 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T15:20:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:44.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:44 smithi017 ceph-mon[215646]: pgmap v795: 109 pgs: 109 active+clean; 612 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T15:20:44.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:44.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:44 smithi017 ceph-mon[222878]: pgmap v795: 109 pgs: 109 active+clean; 612 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 588 op/s 2023-09-23T15:20:44.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:44.779 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:44.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:44.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:44.890 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:44.952 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:45.586 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:45.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:45.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:45.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:45.725 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:46.176 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:46.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:46.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:46.391 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:46.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:46 smithi017 ceph-mon[222878]: pgmap v796: 109 pgs: 109 active+clean; 484 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T15:20:46.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:46 smithi017 ceph-mon[215646]: pgmap v796: 109 pgs: 109 active+clean; 484 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T15:20:46.549 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:46.625 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:46.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:46.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:46.674 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:20:46.674 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:20:46.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:46.775 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:46 smithi027 ceph-mon[195233]: pgmap v796: 109 pgs: 109 active+clean; 484 MiB data, 3.6 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 707 op/s 2023-09-23T15:20:46.777 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:46.828 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:46.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:46.835 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:46.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:47.018 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:47.783 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:47.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:47.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:47.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:47.954 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:47.955 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:20:48.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:20:48.071 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:48] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T15:20:48.281 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:48.287 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:48.385 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:20:48.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:48.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:48 smithi017 ceph-mon[215646]: pgmap v797: 109 pgs: 109 active+clean; 425 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 765 op/s 2023-09-23T15:20:48.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:20:48.421 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:48.473 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:48.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:48.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:48.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:48.662 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:48.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:48 smithi017 ceph-mon[222878]: pgmap v797: 109 pgs: 109 active+clean; 425 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 765 op/s 2023-09-23T15:20:48.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:20:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:48 smithi027 ceph-mon[195233]: pgmap v797: 109 pgs: 109 active+clean; 425 MiB data, 3.4 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 765 op/s 2023-09-23T15:20:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:20:49.120 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:49.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:49.126 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:49.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:49.256 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:49.321 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:49.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:49.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:49.426 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:49.453 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:49.496 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:49.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:49.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:49.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:20:49.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T15:20:49.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:49 smithi027 ceph-mon[195233]: pgmap v799: 109 pgs: 109 active+clean; 410 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 546 op/s 2023-09-23T15:20:49.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.665 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[215646]: pgmap v799: 109 pgs: 109 active+clean; 410 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 546 op/s 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3719928516' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: osdmap e277: 8 total, 8 up, 6 in 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:49.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:49 smithi017 ceph-mon[222878]: pgmap v799: 109 pgs: 109 active+clean; 410 MiB data, 3.3 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 546 op/s 2023-09-23T15:20:49.673 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:49.900 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:49.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:49.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:49.942 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:20:49.942 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:20:50.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:50.037 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:50.087 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:50.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:50.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:50.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:50.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:50.288 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:50.309 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:50.481 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:50.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:50.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:50.591 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:50.620 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:50.639 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:50 smithi017 ceph-mon[215646]: failed to encode map e277 with expected crc 2023-09-23T15:20:50.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:50 smithi017 ceph-mon[222878]: failed to encode map e277 with expected crc 2023-09-23T15:20:50.672 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:50.676 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:50 smithi027 ceph-mon[195233]: failed to encode map e277 with expected crc 2023-09-23T15:20:50.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:50.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:50.778 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:50.868 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:51.093 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:51.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:51.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:51.292 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:51.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[222878]: pgmap v800: 109 pgs: 109 active+clean; 207 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T15:20:51.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4006991366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:51.430 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:51.458 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:20:51.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[215646]: pgmap v800: 109 pgs: 109 active+clean; 207 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T15:20:51.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4006991366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:51.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:51 smithi027 ceph-mon[195233]: pgmap v800: 109 pgs: 109 active+clean; 207 MiB data, 2.8 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 667 op/s 2023-09-23T15:20:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4006991366' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[215646]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T15:20:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:52.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:20:52.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:52.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:52 smithi017 ceph-mon[222878]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: failed to encode map e278 with expected crc 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:20:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:52 smithi027 ceph-mon[195233]: osdmap e278: 8 total, 8 up, 6 in 2023-09-23T15:20:53.387 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:53.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:53.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:53.411 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:53.508 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:53.539 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:53.592 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:53.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:53.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:53 smithi017 ceph-mon[215646]: pgmap v802: 109 pgs: 109 active+clean; 193 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:20:53.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:53 smithi017 ceph-mon[222878]: pgmap v802: 109 pgs: 109 active+clean; 193 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:20:53.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:53.723 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 93867 objects 2023-09-23T15:20:53.723 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :150.512 2023-09-23T15:20:53.744 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:53.848 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:53.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:53 smithi027 ceph-mon[195233]: pgmap v802: 109 pgs: 109 active+clean; 193 MiB data, 2.7 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:20:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:53.851 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:53.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:53.978 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:20:53.979 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:53.998 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:20:54.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:20:54.022 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:54.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:54.026 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:54.041 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:20:54.062 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:20:54.084 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:20:54.103 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:20:54.123 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.124 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.125 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.125 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:20:54.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:54.160 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:54.224 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78609 6.1 6:80d4e179:::benchmark_data_smithi017_350392_object78608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 492.37420100600002, 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3694808949999999, 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.225 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78609 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848309+0000", 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848307+0000", 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848333+0000", 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5845000000000001e-05 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848345+0000", 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2347000000000001e-05 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848356+0000", 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0528e-05 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849744+0000", 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001388467 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849757+0000", 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2284e-05 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.198127+0000", 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3483700409999999 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.200339+0000", 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022119430000000001 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:20:54.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.200633+0000", 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029397499999999998 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214375+0000", 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013742694 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214502+0000", 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012641799999999999 2023-09-23T15:20:54.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214546+0000", 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4276000000000001e-05 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214607+0000", 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0668999999999998e-05 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.217790+0000", 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031828379999999999 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78617 6.1 6:81e02ebd:::benchmark_data_smithi017_350392_object78616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 492.37413912699998, 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3832878639999999, 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.233 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78617 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848371+0000", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848370+0000", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848402+0000", 2023-09-23T15:20:54.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2126999999999999e-05 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848411+0000", 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2410000000000001e-06 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848420+0000", 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8629999999999997e-06 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849784+0000", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013640639999999999 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849792+0000", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7810000000000005e-06 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204128+0000", 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.354335651 2023-09-23T15:20:54.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204325+0000", 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000197455 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204530+0000", 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020487500000000001 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214389+0000", 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0098588370000000005 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218413+0000", 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040238640000000003 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228773+0000", 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010360163 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228831+0000", 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8516e-05 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231658+0000", 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028269829999999999 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78637 6.1 6:82bb8763:::benchmark_data_smithi017_350392_object78636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 492.37408302599999, 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3859898829999999, 2023-09-23T15:20:54.241 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78637 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:20:54.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848427+0000", 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848426+0000", 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848450+0000", 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4318e-05 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848458+0000", 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4340000000000003e-06 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.848464+0000", 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1079999999999998e-06 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849813+0000", 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001349329 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:20:54.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.849820+0000", 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0110000000000003e-06 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208302+0000", 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3584813790000001 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208482+0000", 2023-09-23T15:20:54.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017982600000000001 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.208669+0000", 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018774899999999999 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.214462+0000", 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057931709999999997 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218460+0000", 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039972669999999997 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231725+0000", 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013265279 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.231798+0000", 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2589000000000006e-05 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.234417+0000", 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026190129999999999 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:147640 6.f 6:f5e90299:::benchmark_data_smithi017_350392_object47249:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e206)", 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 368.29595881, 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27895413200000002, 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:20:54.250 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147640 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926551+0000", 2023-09-23T15:20:54.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926549+0000", 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926557+0000", 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.559e-06 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926571+0000", 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3725e-05 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926580+0000", 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5640000000000002e-06 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926626+0000", 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5689000000000003e-05 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926756+0000", 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129863 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:20:54.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.926870+0000", 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114127 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.931667+0000", 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0047970349999999998 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:45.931726+0000", 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8600000000000001e-05 2023-09-23T15:20:54.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205421+0000", 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27369536900000002 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205468+0000", 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6739000000000001e-05 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:14:46.205505+0000", 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7401000000000003e-05 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64529 7.e 7:7d16ebd8:::benchmark_data_smithi017_422824_object64528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 189.37483438999999, 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.273188133, 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.257 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64529 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847696+0000", 2023-09-23T15:20:54.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0910000000000001e-05 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847703+0000", 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847709+0000", 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4909999999999996e-06 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847787+0000", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8174000000000001e-05 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847961+0000", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017452599999999999 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.848087+0000", 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012575800000000001 2023-09-23T15:20:54.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.850201+0000", 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021134719999999999 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120707+0000", 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.270506674 2023-09-23T15:20:54.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120747+0000", 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9394999999999998e-05 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120796+0000", 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9567000000000002e-05 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120863+0000", 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7117000000000001e-05 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64532 7.e 7:731a4c04:::benchmark_data_smithi017_422824_object64531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 189.371435338, 2023-09-23T15:20:54.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28321638999999998, 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64532 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851073+0000", 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851107+0000", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4767000000000001e-05 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851120+0000", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2638e-05 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851128+0000", 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1519999999999992e-06 2023-09-23T15:20:54.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851174+0000", 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6133e-05 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118070+0000", 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26689595700000002 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118233+0000", 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016251300000000001 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.125753+0000", 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075200090000000002 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.271 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.128398+0000", 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026455459999999999 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134191+0000", 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057925729999999996 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134247+0000", 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6369999999999997e-05 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134291+0000", 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3470999999999998e-05 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64534 7.5 7:a537e92b:::benchmark_data_smithi017_422824_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 189.371367787, 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28027905600000003, 2023-09-23T15:20:54.273 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64534 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:20:54.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851141+0000", 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851169+0000", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7999000000000001e-05 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851177+0000", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5970000000000003e-06 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851182+0000", 2023-09-23T15:20:54.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4040000000000003e-06 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117946+0000", 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266763213 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118119+0000", 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017326 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118260+0000", 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140767 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129302+0000", 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01104228 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129468+0000", 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166195 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131318+0000", 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018502740000000001 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131369+0000", 2023-09-23T15:20:54.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0327000000000001e-05 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131421+0000", 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2238999999999999e-05 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.280 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64539 7.5 7:aff3ef34:::benchmark_data_smithi017_422824_object64538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 189.37131244899999, 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28619797000000002, 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64539 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851217+0000", 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0457000000000001e-05 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117948+0000", 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26673075899999998 2023-09-23T15:20:54.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117957+0000", 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9560000000000003e-06 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120513+0000", 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002556034 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120654+0000", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014076799999999999 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120798+0000", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144394 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129322+0000", 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085240860000000002 2023-09-23T15:20:54.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129497+0000", 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000174737 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137299+0000", 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078019229999999997 2023-09-23T15:20:54.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137345+0000", 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6551000000000002e-05 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137395+0000", 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9852999999999999e-05 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.287 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126249 7.8 7:1dd9f547:::benchmark_data_smithi017_422824_object32379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 102.45101903299999, 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30324335800000002, 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.288 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126249 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771489+0000", 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771499+0000", 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0219e-05 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771514+0000", 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4487999999999999e-05 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771523+0000", 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771607+0000", 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3652999999999993e-05 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771729+0000", 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012218299999999999 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771846+0000", 2023-09-23T15:20:54.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011697900000000001 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.779901+0000", 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080551970000000001 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.782538+0000", 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002637393 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074627+0000", 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29208838199999998 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074687+0000", 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0074000000000001e-05 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074734+0000", 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7129000000000002e-05 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126250 7.8 7:12de4e94:::benchmark_data_smithi017_422824_object32380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 102.45096204399999, 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30340214700000001, 2023-09-23T15:20:54.295 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126250 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771547+0000", 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.297 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771551+0000", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.873e-06 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771562+0000", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1371e-05 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771569+0000", 2023-09-23T15:20:54.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775770+0000", 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042005970000000004 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775901+0000", 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131066 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.776027+0000", 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012558299999999999 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.781218+0000", 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051909429999999999 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.788400+0000", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007182127 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074879+0000", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28647872800000002 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074918+0000", 2023-09-23T15:20:54.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9526000000000001e-05 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074950+0000", 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1764999999999998e-05 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147540 7.5 7:a064fe8e:::benchmark_data_smithi017_422824_object53670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:20:54.302 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 66.093805805000002, 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35421415899999997, 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147540 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128703+0000", 2023-09-23T15:20:54.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128709+0000", 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2330000000000003e-06 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128731+0000", 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2753000000000001e-05 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128739+0000", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128795+0000", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5846e-05 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128906+0000", 2023-09-23T15:20:54.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111094 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.129011+0000", 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010522 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.132677+0000", 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036654330000000001 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.134043+0000", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013660079999999999 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482827+0000", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.348784551 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482880+0000", 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2193e-05 2023-09-23T15:20:54.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482918+0000", 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8566999999999999e-05 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147555 7.e 7:7b3c5bac:::benchmark_data_smithi017_422824_object53685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 66.084017275999997, 2023-09-23T15:20:54.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35786704400000002, 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147555 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138491+0000", 2023-09-23T15:20:54.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138497+0000", 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7359999999999998e-06 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138506+0000", 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4890000000000001e-06 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138512+0000", 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.344e-06 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138557+0000", 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5674999999999999e-05 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428127+0000", 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28956913299999998 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130603 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432003+0000", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037461870000000002 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432016+0000", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2327000000000001e-05 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496277+0000", 2023-09-23T15:20:54.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.064261756000000003 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496320+0000", 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2994999999999997e-05 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496359+0000", 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9004000000000001e-05 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.316 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147557 7.5 7:a3342b39:::benchmark_data_smithi017_422824_object53687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 66.081583941999995, 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35537888099999998, 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.317 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147557 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140925+0000", 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140930+0000", 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1170000000000003e-06 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428068+0000", 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28713744600000002 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428083+0000", 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5126000000000001e-05 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428149+0000", 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:20:54.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010804899999999999 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428331+0000", 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4103000000000005e-05 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.321 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432563+0000", 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042319059999999997 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432991+0000", 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042866099999999999 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496219+0000", 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063227189000000003 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496265+0000", 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6489000000000002e-05 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496305+0000", 2023-09-23T15:20:54.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9665000000000001e-05 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:148579 7.8 7:14985ae7:::benchmark_data_smithi017_422824_object54709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 64.079729732000004, 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27762552400000001, 2023-09-23T15:20:54.324 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 148579 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142779+0000", 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142785+0000", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7640000000000002e-06 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142795+0000", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.042e-05 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142816+0000", 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0897e-05 2023-09-23T15:20:54.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143029+0000", 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021310799999999999 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143147+0000", 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117836 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143269+0000", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001225 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.151514+0000", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082451160000000003 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156566+0000", 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005051303 2023-09-23T15:20:54.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156883+0000", 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000317378 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420224+0000", 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.263340776 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420275+0000", 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.058e-05 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420406+0000", 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013101000000000001 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165850 7.8 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.766990579999998, 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.360395309, 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165850 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.332 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:20:54.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455522+0000", 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.613e-06 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455530+0000", 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2649999999999996e-06 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455537+0000", 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1770000000000003e-06 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455589+0000", 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2590999999999998e-05 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455703+0000", 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011421000000000001 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455815+0000", 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112071 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460827+0000", 2023-09-23T15:20:54.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050111419999999997 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468359+0000", 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075321540000000001 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815826+0000", 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34746724800000001 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815875+0000", 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8959999999999999e-05 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815914+0000", 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9490999999999998e-05 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165867 7.8 7:1ac8c744:::benchmark_data_smithi017_422824_object71997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.748970321000002, 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35163571799999999, 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165867 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.339 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473542+0000", 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1200000000000002e-06 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473549+0000", 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2429999999999999e-06 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473556+0000", 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1999999999999997e-06 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473610+0000", 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3355e-05 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473737+0000", 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126804 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.474340+0000", 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060306200000000002 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482320+0000", 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079798620000000008 2023-09-23T15:20:54.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482361+0000", 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1823000000000003e-05 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825087+0000", 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34272576999999999 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825135+0000", 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7395000000000003e-05 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825175+0000", 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0583000000000001e-05 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165870 7.8 7:198dc808:::benchmark_data_smithi017_422824_object72000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:20:54.345 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.746087420000002, 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35318146500000003, 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165870 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476421+0000", 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476428+0000", 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3429999999999998e-06 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476446+0000", 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7997000000000001e-05 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476455+0000", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478546+0000", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002091192 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478670+0000", 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012399199999999999 2023-09-23T15:20:54.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478787+0000", 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116907 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482437+0000", 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036504319999999999 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.483321+0000", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088376899999999996 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829506+0000", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34618508100000001 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829557+0000", 2023-09-23T15:20:54.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0427999999999999e-05 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829604+0000", 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7024000000000002e-05 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165871 7.5 7:a7f75625:::benchmark_data_smithi017_422824_object72001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:20:54.352 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.745980649000003, 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27785160399999997, 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165871 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476528+0000", 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476533+0000", 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.318e-06 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476542+0000", 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7059999999999999e-06 2023-09-23T15:20:54.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476547+0000", 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6260000000000003e-06 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477473+0000", 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00092625100000000001 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477603+0000", 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129806 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477726+0000", 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012271 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.481919+0000", 2023-09-23T15:20:54.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041935080000000003 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482075+0000", 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015598599999999999 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754297+0000", 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27222164999999998 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754340+0000", 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2945000000000003e-05 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754381+0000", 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0768e-05 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179356 7.e 7:7f6e6587:::benchmark_data_smithi017_422824_object85486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.533758219999999, 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33625833300000002, 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179356 2023-09-23T15:20:54.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688750+0000", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688756+0000", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2060000000000004e-06 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2061e-05 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688776+0000", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0299999999999994e-06 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688855+0000", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8975999999999997e-05 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688972+0000", 2023-09-23T15:20:54.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116957 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689084+0000", 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111329 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692868+0000", 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003784379 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695532+0000", 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026644920000000001 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024919+0000", 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32938610699999998 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024969+0000", 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0522000000000002e-05 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025010+0000", 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0778000000000002e-05 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179365 7.e 7:78918128:::benchmark_data_smithi017_422824_object85495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.521544435999999, 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32416663400000001, 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179365 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.367 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700964+0000", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700970+0000", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6659999999999996e-06 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700981+0000", 2023-09-23T15:20:54.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1184e-05 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700987+0000", 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1789999999999996e-06 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701026+0000", 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8707000000000001e-05 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701105+0000", 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9220000000000004e-05 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701175+0000", 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9696000000000005e-05 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704027+0000", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028522439999999999 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704091+0000", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3796999999999999e-05 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025048+0000", 2023-09-23T15:20:54.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32095755500000001 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025094+0000", 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5488999999999998e-05 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025132+0000", 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8133999999999998e-05 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.373 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.374 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:20:54.374 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:20:54.374 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:20:54.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1013664965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:54.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1013664965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:54.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1013664965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:20:55.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:20:55.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:55 smithi017 ceph-mon[222878]: pgmap v803: 109 pgs: 109 active+clean; 61 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 727 op/s 2023-09-23T15:20:55.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:55 smithi017 ceph-mon[215646]: pgmap v803: 109 pgs: 109 active+clean; 61 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 727 op/s 2023-09-23T15:20:55.624 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:20:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:55.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:20:55.728 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:55.756 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:55.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:20:55.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:55 smithi027 ceph-mon[195233]: pgmap v803: 109 pgs: 109 active+clean; 61 MiB data, 2.4 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 727 op/s 2023-09-23T15:20:55.854 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:55.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:55.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:55.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:55.986 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:56.051 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:56.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:56.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:56.116 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:56.155 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:56.188 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:56.239 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:56.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:56.346 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:56.380 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:56.427 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:56.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:56.446 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:20:56.448 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:20:56.505 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:56.533 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:20:56.569 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:57.125 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T15:20:57.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T15:20:57.517 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":803,"stamp":"2023-09-23T15:20:54.623432+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":63969203,"num_objects":931,"num_object_clones":0,"num_object_copies":2793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":931,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":187740,"num_write_kb":6020214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13229,"num_bytes_recovered":865933296,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94265,"ondisk_log_size":94265,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2474668,"kb_used_data":335804,"kb_used_omap":30,"kb_used_meta":2138465,"kb_avail":559996244,"statfs":{"total":575970213888,"available":573436153856,"internally_reserved":0,"allocated":343863296,"data_stored":337535724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":2189788760},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-381222936,"num_objects":-5818,"num_object_clones":0,"num_object_copies":-17454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5818,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002216"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":35,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.401100+0000","last_change":"2023-09-23T15:19:41.337833+0000","last_active":"2023-09-23T15:20:48.401100+0000","last_peered":"2023-09-23T15:20:48.401100+0000","last_clean":"2023-09-23T15:20:48.401100+0000","last_became_active":"2023-09-23T15:19:41.337459+0000","last_became_peered":"2023-09-23T15:19:41.337459+0000","last_unstale":"2023-09-23T15:20:48.401100+0000","last_undegraded":"2023-09-23T15:20:48.401100+0000","last_fullsized":"2023-09-23T15:20:48.401100+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:15:23.895299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":216,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420653+0000","last_change":"2023-09-23T15:19:41.351439+0000","last_active":"2023-09-23T15:20:39.420653+0000","last_peered":"2023-09-23T15:20:39.420653+0000","last_clean":"2023-09-23T15:20:39.420653+0000","last_became_active":"2023-09-23T15:19:41.351029+0000","last_became_peered":"2023-09-23T15:19:41.351029+0000","last_unstale":"2023-09-23T15:20:39.420653+0000","last_undegraded":"2023-09-23T15:20:39.420653+0000","last_fullsized":"2023-09-23T15:20:39.420653+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:42.043427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.403345+0000","last_change":"2023-09-23T15:19:41.348299+0000","last_active":"2023-09-23T15:20:48.403345+0000","last_peered":"2023-09-23T15:20:48.403345+0000","last_clean":"2023-09-23T15:20:48.403345+0000","last_became_active":"2023-09-23T15:19:41.347335+0000","last_became_peered":"2023-09-23T15:19:41.347335+0000","last_unstale":"2023-09-23T15:20:48.403345+0000","last_undegraded":"2023-09-23T15:20:48.403345+0000","last_fullsized":"2023-09-23T15:20:48.403345+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:09:40.120925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":246,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.395408+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:20:48.395408+0000","last_peered":"2023-09-23T15:20:48.395408+0000","last_clean":"2023-09-23T15:20:48.395408+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:20:48.395408+0000","last_undegraded":"2023-09-23T15:20:48.395408+0000","last_fullsized":"2023-09-23T15:20:48.395408+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":277,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:45.451476+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:20:45.451476+0000","last_peered":"2023-09-23T15:20:45.451476+0000","last_clean":"2023-09-23T15:20:45.451476+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:20:45.451476+0000","last_undegraded":"2023-09-23T15:20:45.451476+0000","last_fullsized":"2023-09-23T15:20:45.451476+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":252,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.407174+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:20:48.407174+0000","last_peered":"2023-09-23T15:20:48.407174+0000","last_clean":"2023-09-23T15:20:48.407174+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:20:48.407174+0000","last_undegraded":"2023-09-23T15:20:48.407174+0000","last_fullsized":"2023-09-23T15:20:48.407174+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398738+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:20:48.398738+0000","last_peered":"2023-09-23T15:20:48.398738+0000","last_clean":"2023-09-23T15:20:48.398738+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:20:48.398738+0000","last_undegraded":"2023-09-23T15:20:48.398738+0000","last_fullsized":"2023-09-23T15:20:48.398738+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":259,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421392+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:20:39.421392+0000","last_peered":"2023-09-23T15:20:39.421392+0000","last_clean":"2023-09-23T15:20:39.421392+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:20:39.421392+0000","last_undegraded":"2023-09-23T15:20:39.421392+0000","last_fullsized":"2023-09-23T15:20:39.421392+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":594,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382431+0000","last_change":"2023-09-23T15:19:44.390803+0000","last_active":"2023-09-23T15:20:48.382431+0000","last_peered":"2023-09-23T15:20:48.382431+0000","last_clean":"2023-09-23T15:20:48.382431+0000","last_became_active":"2023-09-23T15:19:44.390458+0000","last_became_peered":"2023-09-23T15:19:44.390458+0000","last_unstale":"2023-09-23T15:20:48.382431+0000","last_undegraded":"2023-09-23T15:20:48.382431+0000","last_fullsized":"2023-09-23T15:20:48.382431+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:01.314328+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":267,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427810+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:20:39.427810+0000","last_peered":"2023-09-23T15:20:39.427810+0000","last_clean":"2023-09-23T15:20:39.427810+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:20:39.427810+0000","last_undegraded":"2023-09-23T15:20:39.427810+0000","last_fullsized":"2023-09-23T15:20:39.427810+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":202,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421329+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:20:39.421329+0000","last_peered":"2023-09-23T15:20:39.421329+0000","last_clean":"2023-09-23T15:20:39.421329+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:20:39.421329+0000","last_undegraded":"2023-09-23T15:20:39.421329+0000","last_fullsized":"2023-09-23T15:20:39.421329+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":156,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420563+0000","last_change":"2023-09-23T15:19:41.400054+0000","last_active":"2023-09-23T15:20:39.420563+0000","last_peered":"2023-09-23T15:20:39.420563+0000","last_clean":"2023-09-23T15:20:39.420563+0000","last_became_active":"2023-09-23T15:19:41.358484+0000","last_became_peered":"2023-09-23T15:19:41.358484+0000","last_unstale":"2023-09-23T15:20:39.420563+0000","last_undegraded":"2023-09-23T15:20:39.420563+0000","last_fullsized":"2023-09-23T15:20:39.420563+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:12:25.075054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":152,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.404939+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:20:48.404939+0000","last_peered":"2023-09-23T15:20:48.404939+0000","last_clean":"2023-09-23T15:20:48.404939+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:20:48.404939+0000","last_undegraded":"2023-09-23T15:20:48.404939+0000","last_fullsized":"2023-09-23T15:20:48.404939+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":99,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406968+0000","last_change":"2023-09-23T15:19:41.396887+0000","last_active":"2023-09-23T15:20:48.406968+0000","last_peered":"2023-09-23T15:20:48.406968+0000","last_clean":"2023-09-23T15:20:48.406968+0000","last_became_active":"2023-09-23T15:19:41.358777+0000","last_became_peered":"2023-09-23T15:19:41.358777+0000","last_unstale":"2023-09-23T15:20:48.406968+0000","last_undegraded":"2023-09-23T15:20:48.406968+0000","last_fullsized":"2023-09-23T15:20:48.406968+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:30.237212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":246,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406663+0000","last_change":"2023-09-23T15:19:41.347627+0000","last_active":"2023-09-23T15:20:48.406663+0000","last_peered":"2023-09-23T15:20:48.406663+0000","last_clean":"2023-09-23T15:20:48.406663+0000","last_became_active":"2023-09-23T15:19:41.347230+0000","last_became_peered":"2023-09-23T15:19:41.347230+0000","last_unstale":"2023-09-23T15:20:48.406663+0000","last_undegraded":"2023-09-23T15:20:48.406663+0000","last_fullsized":"2023-09-23T15:20:48.406663+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:02.874471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":586,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.449012+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:20:51.449012+0000","last_peered":"2023-09-23T15:20:51.449012+0000","last_clean":"2023-09-23T15:20:51.449012+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:20:51.449012+0000","last_undegraded":"2023-09-23T15:20:51.449012+0000","last_fullsized":"2023-09-23T15:20:51.449012+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":51,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382167+0000","last_change":"2023-09-23T15:19:41.375930+0000","last_active":"2023-09-23T15:20:48.382167+0000","last_peered":"2023-09-23T15:20:48.382167+0000","last_clean":"2023-09-23T15:20:48.382167+0000","last_became_active":"2023-09-23T15:19:41.345893+0000","last_became_peered":"2023-09-23T15:19:41.345893+0000","last_unstale":"2023-09-23T15:20:48.382167+0000","last_undegraded":"2023-09-23T15:20:48.382167+0000","last_fullsized":"2023-09-23T15:20:48.382167+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:56.369302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":270,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420232+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:20:39.420232+0000","last_peered":"2023-09-23T15:20:39.420232+0000","last_clean":"2023-09-23T15:20:39.420232+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:20:39.420232+0000","last_undegraded":"2023-09-23T15:20:39.420232+0000","last_fullsized":"2023-09-23T15:20:39.420232+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":596,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426958+0000","last_change":"2023-09-23T15:19:44.374862+0000","last_active":"2023-09-23T15:20:39.426958+0000","last_peered":"2023-09-23T15:20:39.426958+0000","last_clean":"2023-09-23T15:20:39.426958+0000","last_became_active":"2023-09-23T15:19:44.374464+0000","last_became_peered":"2023-09-23T15:19:44.374464+0000","last_unstale":"2023-09-23T15:20:39.426958+0000","last_undegraded":"2023-09-23T15:20:39.426958+0000","last_fullsized":"2023-09-23T15:20:39.426958+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:35.374020+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":244,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390631+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:20:48.390631+0000","last_peered":"2023-09-23T15:20:48.390631+0000","last_clean":"2023-09-23T15:20:48.390631+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:20:48.390631+0000","last_undegraded":"2023-09-23T15:20:48.390631+0000","last_fullsized":"2023-09-23T15:20:48.390631+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":245,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427649+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:20:39.427649+0000","last_peered":"2023-09-23T15:20:39.427649+0000","last_clean":"2023-09-23T15:20:39.427649+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:20:39.427649+0000","last_undegraded":"2023-09-23T15:20:39.427649+0000","last_fullsized":"2023-09-23T15:20:39.427649+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":549,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.412053+0000","last_change":"2023-09-23T15:19:44.401028+0000","last_active":"2023-09-23T15:20:48.412053+0000","last_peered":"2023-09-23T15:20:48.412053+0000","last_clean":"2023-09-23T15:20:48.412053+0000","last_became_active":"2023-09-23T15:19:44.400692+0000","last_became_peered":"2023-09-23T15:19:44.400692+0000","last_unstale":"2023-09-23T15:20:48.412053+0000","last_undegraded":"2023-09-23T15:20:48.412053+0000","last_fullsized":"2023-09-23T15:20:48.412053+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:35.692134+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":40,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406306+0000","last_change":"2023-09-23T15:19:41.357368+0000","last_active":"2023-09-23T15:20:48.406306+0000","last_peered":"2023-09-23T15:20:48.406306+0000","last_clean":"2023-09-23T15:20:48.406306+0000","last_became_active":"2023-09-23T15:19:41.356967+0000","last_became_peered":"2023-09-23T15:19:41.356967+0000","last_unstale":"2023-09-23T15:20:48.406306+0000","last_undegraded":"2023-09-23T15:20:48.406306+0000","last_fullsized":"2023-09-23T15:20:48.406306+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:01.852687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":233,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.395603+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:20:48.395603+0000","last_peered":"2023-09-23T15:20:48.395603+0000","last_clean":"2023-09-23T15:20:48.395603+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:20:48.395603+0000","last_undegraded":"2023-09-23T15:20:48.395603+0000","last_fullsized":"2023-09-23T15:20:48.395603+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":576,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427367+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:20:39.427367+0000","last_peered":"2023-09-23T15:20:39.427367+0000","last_clean":"2023-09-23T15:20:39.427367+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:20:39.427367+0000","last_undegraded":"2023-09-23T15:20:39.427367+0000","last_fullsized":"2023-09-23T15:20:39.427367+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":244,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406094+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:20:48.406094+0000","last_peered":"2023-09-23T15:20:48.406094+0000","last_clean":"2023-09-23T15:20:48.406094+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:20:48.406094+0000","last_undegraded":"2023-09-23T15:20:48.406094+0000","last_fullsized":"2023-09-23T15:20:48.406094+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":270,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:45.454475+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:20:45.454475+0000","last_peered":"2023-09-23T15:20:45.454475+0000","last_clean":"2023-09-23T15:20:45.454475+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:20:45.454475+0000","last_undegraded":"2023-09-23T15:20:45.454475+0000","last_fullsized":"2023-09-23T15:20:45.454475+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"278'11969","reported_seq":12034,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.663776+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:20:51.663776+0000","last_peered":"2023-09-23T15:20:51.663776+0000","last_clean":"2023-09-23T15:20:51.663776+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:20:51.663776+0000","last_undegraded":"2023-09-23T15:20:51.663776+0000","last_fullsized":"2023-09-23T15:20:51.663776+0000","mapping_epoch":223,"log_start":"247'6730","ondisk_log_start":"247'6730","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5239,"log_dups_size":0,"ondisk_log_size":5239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":5046272,"num_objects":77,"num_object_clones":0,"num_object_copies":231,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11969,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":504,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.401623+0000","last_change":"2023-09-23T15:20:12.948292+0000","last_active":"2023-09-23T15:20:48.401623+0000","last_peered":"2023-09-23T15:20:48.401623+0000","last_clean":"2023-09-23T15:20:48.401623+0000","last_became_active":"2023-09-23T15:19:44.391800+0000","last_became_peered":"2023-09-23T15:19:44.391800+0000","last_unstale":"2023-09-23T15:20:48.401623+0000","last_undegraded":"2023-09-23T15:20:48.401623+0000","last_fullsized":"2023-09-23T15:20:48.401623+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:48.241223+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":49,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398546+0000","last_change":"2023-09-23T15:19:41.373544+0000","last_active":"2023-09-23T15:20:48.398546+0000","last_peered":"2023-09-23T15:20:48.398546+0000","last_clean":"2023-09-23T15:20:48.398546+0000","last_became_active":"2023-09-23T15:19:41.348943+0000","last_became_peered":"2023-09-23T15:19:41.348943+0000","last_unstale":"2023-09-23T15:20:48.398546+0000","last_undegraded":"2023-09-23T15:20:48.398546+0000","last_fullsized":"2023-09-23T15:20:48.398546+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:26.751596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":238,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426791+0000","last_change":"2023-09-23T15:19:41.362493+0000","last_active":"2023-09-23T15:20:39.426791+0000","last_peered":"2023-09-23T15:20:39.426791+0000","last_clean":"2023-09-23T15:20:39.426791+0000","last_became_active":"2023-09-23T15:19:41.362177+0000","last_became_peered":"2023-09-23T15:19:41.362177+0000","last_unstale":"2023-09-23T15:20:39.426791+0000","last_undegraded":"2023-09-23T15:20:39.426791+0000","last_fullsized":"2023-09-23T15:20:39.426791+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:12:13.726981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"277'11514","reported_seq":11578,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:50.457329+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:20:50.457329+0000","last_peered":"2023-09-23T15:20:50.457329+0000","last_clean":"2023-09-23T15:20:50.457329+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:20:50.457329+0000","last_undegraded":"2023-09-23T15:20:50.457329+0000","last_fullsized":"2023-09-23T15:20:50.457329+0000","mapping_epoch":223,"log_start":"247'6712","ondisk_log_start":"247'6712","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4802,"log_dups_size":0,"ondisk_log_size":4802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10092544,"num_objects":154,"num_object_clones":0,"num_object_copies":462,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":154,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11514,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":580,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420154+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:20:39.420154+0000","last_peered":"2023-09-23T15:20:39.420154+0000","last_clean":"2023-09-23T15:20:39.420154+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:20:39.420154+0000","last_undegraded":"2023-09-23T15:20:39.420154+0000","last_fullsized":"2023-09-23T15:20:39.420154+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":49,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.400751+0000","last_change":"2023-09-23T15:19:41.390268+0000","last_active":"2023-09-23T15:20:48.400751+0000","last_peered":"2023-09-23T15:20:48.400751+0000","last_clean":"2023-09-23T15:20:48.400751+0000","last_became_active":"2023-09-23T15:19:41.350379+0000","last_became_peered":"2023-09-23T15:19:41.350379+0000","last_unstale":"2023-09-23T15:20:48.400751+0000","last_undegraded":"2023-09-23T15:20:48.400751+0000","last_fullsized":"2023-09-23T15:20:48.400751+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:43.286774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":202,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.403270+0000","last_change":"2023-09-23T15:19:41.348519+0000","last_active":"2023-09-23T15:20:48.403270+0000","last_peered":"2023-09-23T15:20:48.403270+0000","last_clean":"2023-09-23T15:20:48.403270+0000","last_became_active":"2023-09-23T15:19:41.347811+0000","last_became_peered":"2023-09-23T15:19:41.347811+0000","last_unstale":"2023-09-23T15:20:48.403270+0000","last_undegraded":"2023-09-23T15:20:48.403270+0000","last_fullsized":"2023-09-23T15:20:48.403270+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:33.450715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"278'11532","reported_seq":16460,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.701357+0000","last_change":"2023-09-23T15:19:57.116571+0000","last_active":"2023-09-23T15:20:53.701357+0000","last_peered":"2023-09-23T15:20:53.701357+0000","last_clean":"2023-09-23T15:20:53.701357+0000","last_became_active":"2023-09-23T15:19:57.082962+0000","last_became_peered":"2023-09-23T15:19:57.082962+0000","last_unstale":"2023-09-23T15:20:53.701357+0000","last_undegraded":"2023-09-23T15:20:53.701357+0000","last_fullsized":"2023-09-23T15:20:53.701357+0000","mapping_epoch":260,"log_start":"244'5668","ondisk_log_start":"244'5668","created":223,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:37.326045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11532,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2420,"num_bytes_recovered":158597120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":520,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406937+0000","last_change":"2023-09-23T15:19:44.393819+0000","last_active":"2023-09-23T15:20:48.406937+0000","last_peered":"2023-09-23T15:20:48.406937+0000","last_clean":"2023-09-23T15:20:48.406937+0000","last_became_active":"2023-09-23T15:19:44.393471+0000","last_became_peered":"2023-09-23T15:19:44.393471+0000","last_unstale":"2023-09-23T15:20:48.406937+0000","last_undegraded":"2023-09-23T15:20:48.406937+0000","last_fullsized":"2023-09-23T15:20:48.406937+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:56.059420+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":19,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.383074+0000","last_change":"2023-09-23T15:20:30.328653+0000","last_active":"2023-09-23T15:20:48.383074+0000","last_peered":"2023-09-23T15:20:48.383074+0000","last_clean":"2023-09-23T15:20:48.383074+0000","last_became_active":"2023-09-23T15:20:30.289411+0000","last_became_peered":"2023-09-23T15:20:30.289411+0000","last_unstale":"2023-09-23T15:20:48.383074+0000","last_undegraded":"2023-09-23T15:20:48.383074+0000","last_fullsized":"2023-09-23T15:20:48.383074+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.407018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":266,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406466+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:20:48.406466+0000","last_peered":"2023-09-23T15:20:48.406466+0000","last_clean":"2023-09-23T15:20:48.406466+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:20:48.406466+0000","last_undegraded":"2023-09-23T15:20:48.406466+0000","last_fullsized":"2023-09-23T15:20:48.406466+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"278'11538","reported_seq":16403,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.698055+0000","last_change":"2023-09-23T15:19:58.104306+0000","last_active":"2023-09-23T15:20:53.698055+0000","last_peered":"2023-09-23T15:20:53.698055+0000","last_clean":"2023-09-23T15:20:53.698055+0000","last_became_active":"2023-09-23T15:19:58.073145+0000","last_became_peered":"2023-09-23T15:19:58.073145+0000","last_unstale":"2023-09-23T15:20:53.698055+0000","last_undegraded":"2023-09-23T15:20:53.698055+0000","last_fullsized":"2023-09-23T15:20:53.698055+0000","mapping_epoch":261,"log_start":"241'4833","ondisk_log_start":"241'4833","created":223,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:21:12.135152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11538,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2385,"num_bytes_recovered":156237824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":553,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426686+0000","last_change":"2023-09-23T15:20:10.484333+0000","last_active":"2023-09-23T15:20:39.426686+0000","last_peered":"2023-09-23T15:20:39.426686+0000","last_clean":"2023-09-23T15:20:39.426686+0000","last_became_active":"2023-09-23T15:19:44.404058+0000","last_became_peered":"2023-09-23T15:19:44.404058+0000","last_unstale":"2023-09-23T15:20:39.426686+0000","last_undegraded":"2023-09-23T15:20:39.426686+0000","last_fullsized":"2023-09-23T15:20:39.426686+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:10.484180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:10.484180+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:18:30.529028+0000","scrub_duration":0.018611978000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":236,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427260+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:20:39.427260+0000","last_peered":"2023-09-23T15:20:39.427260+0000","last_clean":"2023-09-23T15:20:39.427260+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:20:39.427260+0000","last_undegraded":"2023-09-23T15:20:39.427260+0000","last_fullsized":"2023-09-23T15:20:39.427260+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390395+0000","last_change":"2023-09-23T15:19:41.353927+0000","last_active":"2023-09-23T15:20:48.390395+0000","last_peered":"2023-09-23T15:20:48.390395+0000","last_clean":"2023-09-23T15:20:48.390395+0000","last_became_active":"2023-09-23T15:19:41.353066+0000","last_became_peered":"2023-09-23T15:19:41.353066+0000","last_unstale":"2023-09-23T15:20:48.390395+0000","last_undegraded":"2023-09-23T15:20:48.390395+0000","last_fullsized":"2023-09-23T15:20:48.390395+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:28.877030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"278'11752","reported_seq":16186,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.686780+0000","last_change":"2023-09-23T15:20:05.659905+0000","last_active":"2023-09-23T15:20:53.686780+0000","last_peered":"2023-09-23T15:20:53.686780+0000","last_clean":"2023-09-23T15:20:53.686780+0000","last_became_active":"2023-09-23T15:20:05.566095+0000","last_became_peered":"2023-09-23T15:20:05.566095+0000","last_unstale":"2023-09-23T15:20:53.686780+0000","last_undegraded":"2023-09-23T15:20:53.686780+0000","last_fullsized":"2023-09-23T15:20:53.686780+0000","mapping_epoch":265,"log_start":"245'5890","ondisk_log_start":"245'5890","created":223,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:23:35.636458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11752,"num_write_kb":376064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2172,"num_bytes_recovered":142213120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":576,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421126+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:20:39.421126+0000","last_peered":"2023-09-23T15:20:39.421126+0000","last_clean":"2023-09-23T15:20:39.421126+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:20:39.421126+0000","last_undegraded":"2023-09-23T15:20:39.421126+0000","last_fullsized":"2023-09-23T15:20:39.421126+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":255,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421755+0000","last_change":"2023-09-23T15:19:41.387781+0000","last_active":"2023-09-23T15:20:39.421755+0000","last_peered":"2023-09-23T15:20:39.421755+0000","last_clean":"2023-09-23T15:20:39.421755+0000","last_became_active":"2023-09-23T15:19:41.346358+0000","last_became_peered":"2023-09-23T15:19:41.346358+0000","last_unstale":"2023-09-23T15:20:39.421755+0000","last_undegraded":"2023-09-23T15:20:39.421755+0000","last_fullsized":"2023-09-23T15:20:39.421755+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:52.625203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":94,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420318+0000","last_change":"2023-09-23T15:19:41.357313+0000","last_active":"2023-09-23T15:20:39.420318+0000","last_peered":"2023-09-23T15:20:39.420318+0000","last_clean":"2023-09-23T15:20:39.420318+0000","last_became_active":"2023-09-23T15:19:41.356953+0000","last_became_peered":"2023-09-23T15:19:41.356953+0000","last_unstale":"2023-09-23T15:20:39.420318+0000","last_undegraded":"2023-09-23T15:20:39.420318+0000","last_fullsized":"2023-09-23T15:20:39.420318+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"278'11790","reported_seq":11855,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.698676+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:20:53.698676+0000","last_peered":"2023-09-23T15:20:53.698676+0000","last_clean":"2023-09-23T15:20:53.698676+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:20:53.698676+0000","last_undegraded":"2023-09-23T15:20:53.698676+0000","last_fullsized":"2023-09-23T15:20:53.698676+0000","mapping_epoch":223,"log_start":"241'5091","ondisk_log_start":"241'5091","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11790,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":581,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427059+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:20:39.427059+0000","last_peered":"2023-09-23T15:20:39.427059+0000","last_clean":"2023-09-23T15:20:39.427059+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:20:39.427059+0000","last_undegraded":"2023-09-23T15:20:39.427059+0000","last_fullsized":"2023-09-23T15:20:39.427059+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":263,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406323+0000","last_change":"2023-09-23T15:19:41.358477+0000","last_active":"2023-09-23T15:20:48.406323+0000","last_peered":"2023-09-23T15:20:48.406323+0000","last_clean":"2023-09-23T15:20:48.406323+0000","last_became_active":"2023-09-23T15:19:41.356725+0000","last_became_peered":"2023-09-23T15:19:41.356725+0000","last_unstale":"2023-09-23T15:20:48.406323+0000","last_undegraded":"2023-09-23T15:20:48.406323+0000","last_fullsized":"2023-09-23T15:20:48.406323+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:41:18.157901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382740+0000","last_change":"2023-09-23T15:19:41.347475+0000","last_active":"2023-09-23T15:20:48.382740+0000","last_peered":"2023-09-23T15:20:48.382740+0000","last_clean":"2023-09-23T15:20:48.382740+0000","last_became_active":"2023-09-23T15:19:41.346823+0000","last_became_peered":"2023-09-23T15:19:41.346823+0000","last_unstale":"2023-09-23T15:20:48.382740+0000","last_undegraded":"2023-09-23T15:20:48.382740+0000","last_fullsized":"2023-09-23T15:20:48.382740+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:04.246472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"278'12002","reported_seq":12066,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.694887+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:20:53.694887+0000","last_peered":"2023-09-23T15:20:53.694887+0000","last_clean":"2023-09-23T15:20:53.694887+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:20:53.694887+0000","last_undegraded":"2023-09-23T15:20:53.694887+0000","last_fullsized":"2023-09-23T15:20:53.694887+0000","mapping_epoch":223,"log_start":"247'6601","ondisk_log_start":"247'6601","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12002,"num_write_kb":384001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":532,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.412051+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:20:48.412051+0000","last_peered":"2023-09-23T15:20:48.412051+0000","last_clean":"2023-09-23T15:20:48.412051+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:20:48.412051+0000","last_undegraded":"2023-09-23T15:20:48.412051+0000","last_fullsized":"2023-09-23T15:20:48.412051+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381994+0000","last_change":"2023-09-23T15:19:41.347670+0000","last_active":"2023-09-23T15:20:48.381994+0000","last_peered":"2023-09-23T15:20:48.381994+0000","last_clean":"2023-09-23T15:20:48.381994+0000","last_became_active":"2023-09-23T15:19:41.347355+0000","last_became_peered":"2023-09-23T15:19:41.347355+0000","last_unstale":"2023-09-23T15:20:48.381994+0000","last_undegraded":"2023-09-23T15:20:48.381994+0000","last_fullsized":"2023-09-23T15:20:48.381994+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:52.941354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":246,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406765+0000","last_change":"2023-09-23T15:19:41.362667+0000","last_active":"2023-09-23T15:20:48.406765+0000","last_peered":"2023-09-23T15:20:48.406765+0000","last_clean":"2023-09-23T15:20:48.406765+0000","last_became_active":"2023-09-23T15:19:41.362383+0000","last_became_peered":"2023-09-23T15:19:41.362383+0000","last_unstale":"2023-09-23T15:20:48.406765+0000","last_undegraded":"2023-09-23T15:20:48.406765+0000","last_fullsized":"2023-09-23T15:20:48.406765+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:05.135891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"278'11842","reported_seq":11905,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.705344+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:20:53.705344+0000","last_peered":"2023-09-23T15:20:53.705344+0000","last_clean":"2023-09-23T15:20:53.705344+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:20:53.705344+0000","last_undegraded":"2023-09-23T15:20:53.705344+0000","last_fullsized":"2023-09-23T15:20:53.705344+0000","mapping_epoch":223,"log_start":"247'6455","ondisk_log_start":"247'6455","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11842,"num_write_kb":378944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":576,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427284+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:20:39.427284+0000","last_peered":"2023-09-23T15:20:39.427284+0000","last_clean":"2023-09-23T15:20:39.427284+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:20:39.427284+0000","last_undegraded":"2023-09-23T15:20:39.427284+0000","last_fullsized":"2023-09-23T15:20:39.427284+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":280,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421211+0000","last_change":"2023-09-23T15:19:41.352619+0000","last_active":"2023-09-23T15:20:39.421211+0000","last_peered":"2023-09-23T15:20:39.421211+0000","last_clean":"2023-09-23T15:20:39.421211+0000","last_became_active":"2023-09-23T15:19:41.352315+0000","last_became_peered":"2023-09-23T15:19:41.352315+0000","last_unstale":"2023-09-23T15:20:39.421211+0000","last_undegraded":"2023-09-23T15:20:39.421211+0000","last_fullsized":"2023-09-23T15:20:39.421211+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:07.918430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406174+0000","last_change":"2023-09-23T15:19:41.347761+0000","last_active":"2023-09-23T15:20:48.406174+0000","last_peered":"2023-09-23T15:20:48.406174+0000","last_clean":"2023-09-23T15:20:48.406174+0000","last_became_active":"2023-09-23T15:19:41.347412+0000","last_became_peered":"2023-09-23T15:19:41.347412+0000","last_unstale":"2023-09-23T15:20:48.406174+0000","last_undegraded":"2023-09-23T15:20:48.406174+0000","last_fullsized":"2023-09-23T15:20:48.406174+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:23.311511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"277'11665","reported_seq":16433,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:50.372856+0000","last_change":"2023-09-23T15:20:01.319545+0000","last_active":"2023-09-23T15:20:50.372856+0000","last_peered":"2023-09-23T15:20:50.372856+0000","last_clean":"2023-09-23T15:20:50.372856+0000","last_became_active":"2023-09-23T15:20:01.290760+0000","last_became_peered":"2023-09-23T15:20:01.290760+0000","last_unstale":"2023-09-23T15:20:50.372856+0000","last_undegraded":"2023-09-23T15:20:50.372856+0000","last_fullsized":"2023-09-23T15:20:50.372856+0000","mapping_epoch":263,"log_start":"248'6875","ondisk_log_start":"248'6875","created":223,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:18.856179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10813440,"num_objects":165,"num_object_clones":0,"num_object_copies":495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11665,"num_write_kb":378560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153419776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":500,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.407115+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:20:48.407115+0000","last_peered":"2023-09-23T15:20:48.407115+0000","last_clean":"2023-09-23T15:20:48.407115+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:20:48.407115+0000","last_undegraded":"2023-09-23T15:20:48.407115+0000","last_fullsized":"2023-09-23T15:20:48.407115+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":456,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.402933+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:20:48.402933+0000","last_peered":"2023-09-23T15:20:48.402933+0000","last_clean":"2023-09-23T15:20:48.402933+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:20:48.402933+0000","last_undegraded":"2023-09-23T15:20:48.402933+0000","last_fullsized":"2023-09-23T15:20:48.402933+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":253,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427545+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:20:39.427545+0000","last_peered":"2023-09-23T15:20:39.427545+0000","last_clean":"2023-09-23T15:20:39.427545+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:20:39.427545+0000","last_undegraded":"2023-09-23T15:20:39.427545+0000","last_fullsized":"2023-09-23T15:20:39.427545+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":211,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382085+0000","last_change":"2023-09-23T15:19:41.350708+0000","last_active":"2023-09-23T15:20:48.382085+0000","last_peered":"2023-09-23T15:20:48.382085+0000","last_clean":"2023-09-23T15:20:48.382085+0000","last_became_active":"2023-09-23T15:19:41.350435+0000","last_became_peered":"2023-09-23T15:19:41.350435+0000","last_unstale":"2023-09-23T15:20:48.382085+0000","last_undegraded":"2023-09-23T15:20:48.382085+0000","last_fullsized":"2023-09-23T15:20:48.382085+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:15.309375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"278'11564","reported_seq":11627,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.705179+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:20:53.705179+0000","last_peered":"2023-09-23T15:20:53.705179+0000","last_clean":"2023-09-23T15:20:53.705179+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:20:53.705179+0000","last_undegraded":"2023-09-23T15:20:53.705179+0000","last_fullsized":"2023-09-23T15:20:53.705179+0000","mapping_epoch":223,"log_start":"246'6130","ondisk_log_start":"246'6130","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":574,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426852+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:20:39.426852+0000","last_peered":"2023-09-23T15:20:39.426852+0000","last_clean":"2023-09-23T15:20:39.426852+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:20:39.426852+0000","last_undegraded":"2023-09-23T15:20:39.426852+0000","last_fullsized":"2023-09-23T15:20:39.426852+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":795,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427581+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:20:39.427581+0000","last_peered":"2023-09-23T15:20:39.427581+0000","last_clean":"2023-09-23T15:20:39.427581+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:20:39.427581+0000","last_undegraded":"2023-09-23T15:20:39.427581+0000","last_fullsized":"2023-09-23T15:20:39.427581+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390662+0000","last_change":"2023-09-23T15:19:41.345625+0000","last_active":"2023-09-23T15:20:48.390662+0000","last_peered":"2023-09-23T15:20:48.390662+0000","last_clean":"2023-09-23T15:20:48.390662+0000","last_became_active":"2023-09-23T15:19:41.345256+0000","last_became_peered":"2023-09-23T15:19:41.345256+0000","last_unstale":"2023-09-23T15:20:48.390662+0000","last_undegraded":"2023-09-23T15:20:48.390662+0000","last_fullsized":"2023-09-23T15:20:48.390662+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:39.495384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.401971+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:20:48.401971+0000","last_peered":"2023-09-23T15:20:48.401971+0000","last_clean":"2023-09-23T15:20:48.401971+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:20:48.401971+0000","last_undegraded":"2023-09-23T15:20:48.401971+0000","last_fullsized":"2023-09-23T15:20:48.401971+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"278'11472","reported_seq":15100,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722269+0000","last_change":"2023-09-23T15:20:14.742573+0000","last_active":"2023-09-23T15:20:53.722269+0000","last_peered":"2023-09-23T15:20:53.722269+0000","last_clean":"2023-09-23T15:20:53.722269+0000","last_became_active":"2023-09-23T15:20:14.670681+0000","last_became_peered":"2023-09-23T15:20:14.670681+0000","last_unstale":"2023-09-23T15:20:53.722269+0000","last_undegraded":"2023-09-23T15:20:53.722269+0000","last_fullsized":"2023-09-23T15:20:53.722269+0000","mapping_epoch":269,"log_start":"244'5666","ondisk_log_start":"244'5666","created":223,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:51.982520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11472,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1767,"num_bytes_recovered":115605504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":603,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390712+0000","last_change":"2023-09-23T15:19:44.378313+0000","last_active":"2023-09-23T15:20:48.390712+0000","last_peered":"2023-09-23T15:20:48.390712+0000","last_clean":"2023-09-23T15:20:48.390712+0000","last_became_active":"2023-09-23T15:19:44.377464+0000","last_became_peered":"2023-09-23T15:19:44.377464+0000","last_unstale":"2023-09-23T15:20:48.390712+0000","last_undegraded":"2023-09-23T15:20:48.390712+0000","last_fullsized":"2023-09-23T15:20:48.390712+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:26.547533+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":893,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381924+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:20:48.381924+0000","last_peered":"2023-09-23T15:20:48.381924+0000","last_clean":"2023-09-23T15:20:48.381924+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:20:48.381924+0000","last_undegraded":"2023-09-23T15:20:48.381924+0000","last_fullsized":"2023-09-23T15:20:48.381924+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":153,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.401058+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:20:48.401058+0000","last_peered":"2023-09-23T15:20:48.401058+0000","last_clean":"2023-09-23T15:20:48.401058+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:20:48.401058+0000","last_undegraded":"2023-09-23T15:20:48.401058+0000","last_fullsized":"2023-09-23T15:20:48.401058+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":176,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406086+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:20:48.406086+0000","last_peered":"2023-09-23T15:20:48.406086+0000","last_clean":"2023-09-23T15:20:48.406086+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:20:48.406086+0000","last_undegraded":"2023-09-23T15:20:48.406086+0000","last_fullsized":"2023-09-23T15:20:48.406086+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"278'11808","reported_seq":11873,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.718882+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:20:53.718882+0000","last_peered":"2023-09-23T15:20:53.718882+0000","last_clean":"2023-09-23T15:20:53.718882+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:20:53.718882+0000","last_undegraded":"2023-09-23T15:20:53.718882+0000","last_fullsized":"2023-09-23T15:20:53.718882+0000","mapping_epoch":223,"log_start":"245'5984","ondisk_log_start":"245'5984","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11808,"num_write_kb":377856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":564,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427243+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:20:39.427243+0000","last_peered":"2023-09-23T15:20:39.427243+0000","last_clean":"2023-09-23T15:20:39.427243+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:20:39.427243+0000","last_undegraded":"2023-09-23T15:20:39.427243+0000","last_fullsized":"2023-09-23T15:20:39.427243+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":569,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.404959+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:20:48.404959+0000","last_peered":"2023-09-23T15:20:48.404959+0000","last_clean":"2023-09-23T15:20:48.404959+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:20:48.404959+0000","last_undegraded":"2023-09-23T15:20:48.404959+0000","last_fullsized":"2023-09-23T15:20:48.404959+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":194,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.412318+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:20:48.412318+0000","last_peered":"2023-09-23T15:20:48.412318+0000","last_clean":"2023-09-23T15:20:48.412318+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:20:48.412318+0000","last_undegraded":"2023-09-23T15:20:48.412318+0000","last_fullsized":"2023-09-23T15:20:48.412318+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":248,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406714+0000","last_change":"2023-09-23T15:20:30.274293+0000","last_active":"2023-09-23T15:20:48.406714+0000","last_peered":"2023-09-23T15:20:48.406714+0000","last_clean":"2023-09-23T15:20:48.406714+0000","last_became_active":"2023-09-23T15:20:30.273898+0000","last_became_peered":"2023-09-23T15:20:30.273898+0000","last_unstale":"2023-09-23T15:20:48.406714+0000","last_undegraded":"2023-09-23T15:20:48.406714+0000","last_fullsized":"2023-09-23T15:20:48.406714+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:13.354556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"278'11612","reported_seq":11678,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722589+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:20:53.722589+0000","last_peered":"2023-09-23T15:20:53.722589+0000","last_clean":"2023-09-23T15:20:53.722589+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:20:53.722589+0000","last_undegraded":"2023-09-23T15:20:53.722589+0000","last_fullsized":"2023-09-23T15:20:53.722589+0000","mapping_epoch":223,"log_start":"240'4870","ondisk_log_start":"240'4870","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11612,"num_write_kb":371584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":549,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.411526+0000","last_change":"2023-09-23T15:19:44.386115+0000","last_active":"2023-09-23T15:20:48.411526+0000","last_peered":"2023-09-23T15:20:48.411526+0000","last_clean":"2023-09-23T15:20:48.411526+0000","last_became_active":"2023-09-23T15:19:44.385763+0000","last_became_peered":"2023-09-23T15:19:44.385763+0000","last_unstale":"2023-09-23T15:20:48.411526+0000","last_undegraded":"2023-09-23T15:20:48.411526+0000","last_fullsized":"2023-09-23T15:20:48.411526+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:11.988180+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":364,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.400824+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:20:48.400824+0000","last_peered":"2023-09-23T15:20:48.400824+0000","last_clean":"2023-09-23T15:20:48.400824+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:20:48.400824+0000","last_undegraded":"2023-09-23T15:20:48.400824+0000","last_fullsized":"2023-09-23T15:20:48.400824+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":234,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.405695+0000","last_change":"2023-09-23T15:19:41.349771+0000","last_active":"2023-09-23T15:20:48.405695+0000","last_peered":"2023-09-23T15:20:48.405695+0000","last_clean":"2023-09-23T15:20:48.405695+0000","last_became_active":"2023-09-23T15:19:41.349349+0000","last_became_peered":"2023-09-23T15:19:41.349349+0000","last_unstale":"2023-09-23T15:20:48.405695+0000","last_undegraded":"2023-09-23T15:20:48.405695+0000","last_fullsized":"2023-09-23T15:20:48.405695+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:33:41.777395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.391748+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:20:48.391748+0000","last_peered":"2023-09-23T15:20:48.391748+0000","last_clean":"2023-09-23T15:20:48.391748+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:20:48.391748+0000","last_undegraded":"2023-09-23T15:20:48.391748+0000","last_fullsized":"2023-09-23T15:20:48.391748+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"277'11443","reported_seq":11505,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:49.770371+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:20:49.770371+0000","last_peered":"2023-09-23T15:20:49.770371+0000","last_clean":"2023-09-23T15:20:49.770371+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:20:49.770371+0000","last_undegraded":"2023-09-23T15:20:49.770371+0000","last_fullsized":"2023-09-23T15:20:49.770371+0000","mapping_epoch":223,"log_start":"244'5731","ondisk_log_start":"244'5731","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5712,"log_dups_size":0,"ondisk_log_size":5712,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12255232,"num_objects":187,"num_object_clones":0,"num_object_copies":561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11443,"num_write_kb":372160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":500,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406618+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:20:48.406618+0000","last_peered":"2023-09-23T15:20:48.406618+0000","last_clean":"2023-09-23T15:20:48.406618+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:20:48.406618+0000","last_undegraded":"2023-09-23T15:20:48.406618+0000","last_fullsized":"2023-09-23T15:20:48.406618+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":936,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.394769+0000","last_change":"2023-09-23T15:19:44.387907+0000","last_active":"2023-09-23T15:20:48.394769+0000","last_peered":"2023-09-23T15:20:48.394769+0000","last_clean":"2023-09-23T15:20:48.394769+0000","last_became_active":"2023-09-23T15:19:44.387402+0000","last_became_peered":"2023-09-23T15:19:44.387402+0000","last_unstale":"2023-09-23T15:20:48.394769+0000","last_undegraded":"2023-09-23T15:20:48.394769+0000","last_fullsized":"2023-09-23T15:20:48.394769+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:28.617856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":152,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.402751+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:20:48.402751+0000","last_peered":"2023-09-23T15:20:48.402751+0000","last_clean":"2023-09-23T15:20:48.402751+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:20:48.402751+0000","last_undegraded":"2023-09-23T15:20:48.402751+0000","last_fullsized":"2023-09-23T15:20:48.402751+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390576+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:20:48.390576+0000","last_peered":"2023-09-23T15:20:48.390576+0000","last_clean":"2023-09-23T15:20:48.390576+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:20:48.390576+0000","last_undegraded":"2023-09-23T15:20:48.390576+0000","last_fullsized":"2023-09-23T15:20:48.390576+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"277'11789","reported_seq":16003,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:50.373316+0000","last_change":"2023-09-23T15:20:08.129441+0000","last_active":"2023-09-23T15:20:50.373316+0000","last_peered":"2023-09-23T15:20:50.373316+0000","last_clean":"2023-09-23T15:20:50.373316+0000","last_became_active":"2023-09-23T15:20:08.105976+0000","last_became_peered":"2023-09-23T15:20:08.105976+0000","last_unstale":"2023-09-23T15:20:50.373316+0000","last_undegraded":"2023-09-23T15:20:50.373316+0000","last_fullsized":"2023-09-23T15:20:50.373316+0000","mapping_epoch":267,"log_start":"248'6994","ondisk_log_start":"248'6994","created":223,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4795,"log_dups_size":0,"ondisk_log_size":4795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:42.365664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10289152,"num_objects":157,"num_object_clones":0,"num_object_copies":471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":157,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11789,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2062,"num_bytes_recovered":135135232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":557,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382917+0000","last_change":"2023-09-23T15:20:15.108514+0000","last_active":"2023-09-23T15:20:48.382917+0000","last_peered":"2023-09-23T15:20:48.382917+0000","last_clean":"2023-09-23T15:20:48.382917+0000","last_became_active":"2023-09-23T15:19:44.370417+0000","last_became_peered":"2023-09-23T15:19:44.370417+0000","last_unstale":"2023-09-23T15:20:48.382917+0000","last_undegraded":"2023-09-23T15:20:48.382917+0000","last_fullsized":"2023-09-23T15:20:48.382917+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:23.480089+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":19,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381879+0000","last_change":"2023-09-23T15:20:30.309625+0000","last_active":"2023-09-23T15:20:48.381879+0000","last_peered":"2023-09-23T15:20:48.381879+0000","last_clean":"2023-09-23T15:20:48.381879+0000","last_became_active":"2023-09-23T15:20:30.261983+0000","last_became_peered":"2023-09-23T15:20:30.261983+0000","last_unstale":"2023-09-23T15:20:48.381879+0000","last_undegraded":"2023-09-23T15:20:48.381879+0000","last_fullsized":"2023-09-23T15:20:48.381879+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:52.486099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":779,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.400957+0000","last_change":"2023-09-23T15:19:44.398589+0000","last_active":"2023-09-23T15:20:48.400957+0000","last_peered":"2023-09-23T15:20:48.400957+0000","last_clean":"2023-09-23T15:20:48.400957+0000","last_became_active":"2023-09-23T15:19:44.398077+0000","last_became_peered":"2023-09-23T15:19:44.398077+0000","last_unstale":"2023-09-23T15:20:48.400957+0000","last_undegraded":"2023-09-23T15:20:48.400957+0000","last_fullsized":"2023-09-23T15:20:48.400957+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:34.985355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":260,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421477+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:20:39.421477+0000","last_peered":"2023-09-23T15:20:39.421477+0000","last_clean":"2023-09-23T15:20:39.421477+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:20:39.421477+0000","last_undegraded":"2023-09-23T15:20:39.421477+0000","last_fullsized":"2023-09-23T15:20:39.421477+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"277'11541","reported_seq":11605,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:50.445405+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:20:50.445405+0000","last_peered":"2023-09-23T15:20:50.445405+0000","last_clean":"2023-09-23T15:20:50.445405+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:20:50.445405+0000","last_undegraded":"2023-09-23T15:20:50.445405+0000","last_fullsized":"2023-09-23T15:20:50.445405+0000","mapping_epoch":223,"log_start":"248'6751","ondisk_log_start":"248'6751","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4790,"log_dups_size":0,"ondisk_log_size":4790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10682368,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11541,"num_write_kb":374528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":576,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.420067+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:20:39.420067+0000","last_peered":"2023-09-23T15:20:39.420067+0000","last_clean":"2023-09-23T15:20:39.420067+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:20:39.420067+0000","last_undegraded":"2023-09-23T15:20:39.420067+0000","last_fullsized":"2023-09-23T15:20:39.420067+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":754,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.407026+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:20:48.407026+0000","last_peered":"2023-09-23T15:20:48.407026+0000","last_clean":"2023-09-23T15:20:48.407026+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:20:48.407026+0000","last_undegraded":"2023-09-23T15:20:48.407026+0000","last_fullsized":"2023-09-23T15:20:48.407026+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.391837+0000","last_change":"2023-09-23T15:19:41.392698+0000","last_active":"2023-09-23T15:20:48.391837+0000","last_peered":"2023-09-23T15:20:48.391837+0000","last_clean":"2023-09-23T15:20:48.391837+0000","last_became_active":"2023-09-23T15:19:41.350174+0000","last_became_peered":"2023-09-23T15:19:41.350174+0000","last_unstale":"2023-09-23T15:20:48.391837+0000","last_undegraded":"2023-09-23T15:20:48.391837+0000","last_fullsized":"2023-09-23T15:20:48.391837+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:42:24.036011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":35,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426882+0000","last_change":"2023-09-23T15:19:41.355318+0000","last_active":"2023-09-23T15:20:39.426882+0000","last_peered":"2023-09-23T15:20:39.426882+0000","last_clean":"2023-09-23T15:20:39.426882+0000","last_became_active":"2023-09-23T15:19:41.354940+0000","last_became_peered":"2023-09-23T15:19:41.354940+0000","last_unstale":"2023-09-23T15:20:39.426882+0000","last_undegraded":"2023-09-23T15:20:39.426882+0000","last_fullsized":"2023-09-23T15:20:39.426882+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:57:13.355798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":241,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390788+0000","last_change":"2023-09-23T15:19:41.354183+0000","last_active":"2023-09-23T15:20:48.390788+0000","last_peered":"2023-09-23T15:20:48.390788+0000","last_clean":"2023-09-23T15:20:48.390788+0000","last_became_active":"2023-09-23T15:19:41.353520+0000","last_became_peered":"2023-09-23T15:19:41.353520+0000","last_unstale":"2023-09-23T15:20:48.390788+0000","last_undegraded":"2023-09-23T15:20:48.390788+0000","last_fullsized":"2023-09-23T15:20:48.390788+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:11.470291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382743+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:20:48.382743+0000","last_peered":"2023-09-23T15:20:48.382743+0000","last_clean":"2023-09-23T15:20:48.382743+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:20:48.382743+0000","last_undegraded":"2023-09-23T15:20:48.382743+0000","last_fullsized":"2023-09-23T15:20:48.382743+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":269,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427299+0000","last_change":"2023-09-23T15:19:41.385820+0000","last_active":"2023-09-23T15:20:39.427299+0000","last_peered":"2023-09-23T15:20:39.427299+0000","last_clean":"2023-09-23T15:20:39.427299+0000","last_became_active":"2023-09-23T15:19:41.356653+0000","last_became_peered":"2023-09-23T15:19:41.356653+0000","last_unstale":"2023-09-23T15:20:39.427299+0000","last_undegraded":"2023-09-23T15:20:39.427299+0000","last_fullsized":"2023-09-23T15:20:39.427299+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:58.426360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":201,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.421844+0000","last_change":"2023-09-23T15:19:41.346796+0000","last_active":"2023-09-23T15:20:39.421844+0000","last_peered":"2023-09-23T15:20:39.421844+0000","last_clean":"2023-09-23T15:20:39.421844+0000","last_became_active":"2023-09-23T15:19:41.346289+0000","last_became_peered":"2023-09-23T15:19:41.346289+0000","last_unstale":"2023-09-23T15:20:39.421844+0000","last_undegraded":"2023-09-23T15:20:39.421844+0000","last_fullsized":"2023-09-23T15:20:39.421844+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:27.690863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":235,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.427452+0000","last_change":"2023-09-23T15:19:41.396637+0000","last_active":"2023-09-23T15:20:39.427452+0000","last_peered":"2023-09-23T15:20:39.427452+0000","last_clean":"2023-09-23T15:20:39.427452+0000","last_became_active":"2023-09-23T15:19:41.361792+0000","last_became_peered":"2023-09-23T15:19:41.361792+0000","last_unstale":"2023-09-23T15:20:39.427452+0000","last_undegraded":"2023-09-23T15:20:39.427452+0000","last_fullsized":"2023-09-23T15:20:39.427452+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:05:36.021130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.422664+0000","last_change":"2023-09-23T15:19:41.352434+0000","last_active":"2023-09-23T15:20:39.422664+0000","last_peered":"2023-09-23T15:20:39.422664+0000","last_clean":"2023-09-23T15:20:39.422664+0000","last_became_active":"2023-09-23T15:19:41.352220+0000","last_became_peered":"2023-09-23T15:19:41.352220+0000","last_unstale":"2023-09-23T15:20:39.422664+0000","last_undegraded":"2023-09-23T15:20:39.422664+0000","last_fullsized":"2023-09-23T15:20:39.422664+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:55.334410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":251,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:39.426959+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:20:39.426959+0000","last_peered":"2023-09-23T15:20:39.426959+0000","last_clean":"2023-09-23T15:20:39.426959+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:20:39.426959+0000","last_undegraded":"2023-09-23T15:20:39.426959+0000","last_fullsized":"2023-09-23T15:20:39.426959+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":265,"reported_epoch":276,"state":"active+clean","last_fresh":"2023-09-23T15:20:45.451546+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:20:45.451546+0000","last_peered":"2023-09-23T15:20:45.451546+0000","last_clean":"2023-09-23T15:20:45.451546+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:20:45.451546+0000","last_undegraded":"2023-09-23T15:20:45.451546+0000","last_fullsized":"2023-09-23T15:20:45.451546+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":228,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398802+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:20:48.398802+0000","last_peered":"2023-09-23T15:20:48.398802+0000","last_clean":"2023-09-23T15:20:48.398802+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:20:48.398802+0000","last_undegraded":"2023-09-23T15:20:48.398802+0000","last_fullsized":"2023-09-23T15:20:48.398802+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":101,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.406245+0000","last_change":"2023-09-23T15:19:41.355242+0000","last_active":"2023-09-23T15:20:48.406245+0000","last_peered":"2023-09-23T15:20:48.406245+0000","last_clean":"2023-09-23T15:20:48.406245+0000","last_became_active":"2023-09-23T15:19:41.354874+0000","last_became_peered":"2023-09-23T15:19:41.354874+0000","last_unstale":"2023-09-23T15:20:48.406245+0000","last_undegraded":"2023-09-23T15:20:48.406245+0000","last_fullsized":"2023-09-23T15:20:48.406245+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:50:25.245475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":59179008,"num_objects":903,"num_object_clones":0,"num_object_copies":2709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":903,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186833,"num_write_kb":6007489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13147,"num_bytes_recovered":861208576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":419762176,"data_stored":419758104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89852,"ondisk_log_size":89852,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5040,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201594,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463230,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252632,"kb_used_data":3088,"kb_used_omap":0,"kb_used_meta":249536,"kb_avail":93492520,"statfs":{"total":95995035648,"available":95736340480,"internally_reserved":0,"allocated":3162112,"data_stored":2334792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":255524864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":2.2400000000000002,"15min":3.2440000000000002},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.19800000000000001},"max":{"1min":78.909000000000006,"5min":78.909000000000006,"15min":108.39400000000001},"last":2.6280000000000001},{"interface":"front","average":{"1min":6.923,"5min":2.8450000000000002,"15min":3.573},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":79.430000000000007,"5min":79.430000000000007,"15min":108.452},"last":4.101}]},{"osd":1,"last update":"Sat Sep 23 15:19:52 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":2.0870000000000002,"15min":3.02},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.4660000000000002,"5min":86.236999999999995,"15min":86.236999999999995},"last":1.331},{"interface":"front","average":{"1min":1.484,"5min":2.758,"15min":3.569},"min":{"1min":0.30199999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.7510000000000003,"5min":86.158000000000001,"15min":86.158000000000001},"last":2.4260000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4560000000000004,"5min":4.7350000000000003,"15min":7.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.27500000000000002,"15min":0.255},"max":{"1min":80.875,"5min":86.569000000000003,"15min":180.374},"last":3.5800000000000001},{"interface":"front","average":{"1min":7.7530000000000001,"5min":4.4909999999999997,"15min":7.0229999999999997},"min":{"1min":0.39500000000000002,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":84.096000000000004,"5min":88.269000000000005,"15min":181.685},"last":2.8500000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.726,"15min":4.0529999999999999},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.827,"5min":84.004999999999995,"15min":102.254},"last":0.39500000000000002},{"interface":"front","average":{"1min":2.8199999999999998,"5min":3.7240000000000002,"15min":4.1219999999999999},"min":{"1min":0.48299999999999998,"5min":0.36399999999999999,"15min":0.29799999999999999},"max":{"1min":6.9219999999999997,"5min":83.936000000000007,"15min":98.941999999999993},"last":1.49}]},{"osd":4,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52800000000000002,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":1.5569999999999999,"5min":1.7150000000000001,"15min":36.290999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.59199999999999997,"15min":1.129},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.337,"5min":2.6259999999999999,"15min":36.758000000000003},"last":0.314}]},{"osd":5,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":1.413},"min":{"1min":0.36199999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":1.4770000000000001,"5min":3.1789999999999998,"15min":131.90600000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.64300000000000002,"15min":1.4690000000000001},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.92800000000000005,"5min":3.431,"15min":131.61000000000001},"last":0.35799999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.63300000000000001,"15min":1.032},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28999999999999998},"max":{"1min":2.4399999999999999,"5min":2.9449999999999998,"15min":17},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61799999999999999,"15min":0.97299999999999998},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.25800000000000001},"max":{"1min":1.581,"5min":1.8180000000000001,"15min":7.7809999999999997},"last":0.38}]}]},{"osd":1,"up_from":183,"seq":785979015267,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362924,"kb_used_data":69156,"kb_used_omap":6,"kb_used_meta":293753,"kb_avail":93382228,"statfs":{"total":95995035648,"available":95623401472,"internally_reserved":0,"allocated":70815744,"data_stored":69561334,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":300803147},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":3.7189999999999999,"5min":3.1680000000000001,"15min":3.5259999999999998},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":13.339,"5min":19.344000000000001,"15min":19.344000000000001},"last":4.899},{"interface":"front","average":{"1min":3.3540000000000001,"5min":3.6880000000000002,"15min":4.2320000000000002},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":10.786,"5min":19.472000000000001,"15min":19.472000000000001},"last":1.1859999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":3.5550000000000002,"5min":5.4800000000000004,"15min":4.6379999999999999},"min":{"1min":0.28499999999999998,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.3200000000000003,"5min":109.375,"15min":109.375},"last":10.329000000000001},{"interface":"front","average":{"1min":3.4649999999999999,"5min":4.0170000000000003,"15min":4.1619999999999999},"min":{"1min":0.34300000000000003,"5min":0.27700000000000002,"15min":0.253},"max":{"1min":13.314,"5min":105.16,"15min":105.16},"last":9.6219999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":5.8170000000000002,"5min":8.0340000000000007,"15min":9.4860000000000007},"min":{"1min":0.27100000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":18.646999999999998,"5min":211.405,"15min":211.405},"last":4.9550000000000001},{"interface":"front","average":{"1min":4.7220000000000004,"5min":8.1280000000000001,"15min":10.484999999999999},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.27200000000000002},"max":{"1min":15.661,"5min":210.988,"15min":210.988},"last":0.69599999999999995}]},{"osd":4,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":9.1359999999999992,"5min":5.1539999999999999,"15min":3.5190000000000001},"min":{"1min":0.42699999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":80.570999999999998,"5min":80.570999999999998,"15min":80.570999999999998},"last":12.885},{"interface":"front","average":{"1min":10.07,"5min":4.5860000000000003,"15min":3.8719999999999999},"min":{"1min":0.32500000000000001,"5min":0.308,"15min":0.308},"max":{"1min":80.75,"5min":80.75,"15min":80.75},"last":4.9969999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":4.1559999999999997,"5min":2.9550000000000001,"15min":3.3839999999999999},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":34.512999999999998,"5min":34.512999999999998,"15min":34.512999999999998},"last":0.76500000000000001},{"interface":"front","average":{"1min":4.3789999999999996,"5min":3.4660000000000002,"15min":3.0939999999999999},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":37.911000000000001,"5min":37.911000000000001,"15min":37.911000000000001},"last":12.795}]},{"osd":6,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":8.8879999999999999,"5min":4.9880000000000004,"15min":4.0949999999999998},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":81.352000000000004,"5min":81.352000000000004,"15min":81.352000000000004},"last":0.73499999999999999},{"interface":"front","average":{"1min":9.298,"5min":5.6200000000000001,"15min":3.7250000000000001},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":84.716999999999999,"5min":84.716999999999999,"15min":84.716999999999999},"last":12.834}]},{"osd":7,"last update":"Sat Sep 23 15:19:54 2023","interfaces":[{"interface":"back","average":{"1min":2.3599999999999999,"5min":2.8559999999999999,"15min":3.0630000000000002},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.5500000000000007,"5min":11.561,"15min":11.561},"last":12.861000000000001},{"interface":"front","average":{"1min":3.2330000000000001,"5min":2.2410000000000001,"15min":2.9620000000000002},"min":{"1min":0.40799999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.282999999999999,"5min":16.280999999999999,"15min":16.841000000000001},"last":5.0279999999999996}]}]},{"osd":0,"up_from":95,"seq":408021893317,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":424472,"kb_used_data":16848,"kb_used_omap":8,"kb_used_meta":407607,"kb_avail":93320680,"statfs":{"total":95995035648,"available":95560376320,"internally_reserved":0,"allocated":17252352,"data_stored":16083204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":417390164},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:19:53 2023","interfaces":[{"interface":"back","average":{"1min":4.8840000000000003,"5min":3.7959999999999998,"15min":3.9870000000000001},"min":{"1min":0.222,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":48.442999999999998,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.314},{"interface":"front","average":{"1min":4.4580000000000002,"5min":3.1099999999999999,"15min":4.0309999999999997},"min":{"1min":0.219,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":51.304000000000002,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.8039999999999998,"15min":4.1529999999999996},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.17000000000000001},"max":{"1min":6.9820000000000002,"5min":15.209,"15min":122.20099999999999},"last":4.117},{"interface":"front","average":{"1min":3.1539999999999999,"5min":3.407,"15min":3.8900000000000001},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.18099999999999999},"max":{"1min":8.6679999999999993,"5min":64.466999999999999,"15min":122.071},"last":0.54900000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:20:43 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":6.3630000000000004,"15min":4.8250000000000002},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":14.785,"5min":146.315,"15min":146.315},"last":0.28999999999999998},{"interface":"front","average":{"1min":5.1890000000000001,"5min":7.5529999999999999,"15min":5.3849999999999998},"min":{"1min":0.39600000000000002,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":24.629000000000001,"5min":145.517,"15min":145.517},"last":0.59199999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.5820000000000001,"15min":1.9059999999999999},"min":{"1min":0.33900000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.609,"5min":7.0199999999999996,"15min":31.341999999999999},"last":0.438},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.008,"15min":2.0550000000000002},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.27000000000000002},"max":{"1min":7.0460000000000003,"5min":13.566000000000001,"15min":99.168999999999997},"last":0.41399999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.2770000000000001,"15min":2.1739999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.5449999999999999,"5min":64.349999999999994,"15min":109.369},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.589,"5min":1.6279999999999999,"15min":2.1040000000000001},"min":{"1min":0.38200000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":4.6390000000000002,"5min":6.9779999999999998,"15min":109.387},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":2.1840000000000002,"15min":2.1280000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.309},"max":{"1min":7.0739999999999998,"5min":13.529999999999999,"15min":99.114999999999995},"last":0.36099999999999999},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.4020000000000001,"15min":2.1309999999999998},"min":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":8.625,"5min":64.378,"15min":98.921999999999997},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":2.3450000000000002,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.39300000000000002,"15min":0.36799999999999999},"max":{"1min":8.593,"5min":64.304000000000002,"15min":99.549999999999997},"last":0.66400000000000003},{"interface":"front","average":{"1min":1.806,"5min":1.768,"15min":1.841},"min":{"1min":0.40400000000000003,"5min":0.39300000000000002,"15min":0.36399999999999999},"max":{"1min":8.6809999999999992,"5min":8.6809999999999992,"15min":31.225999999999999},"last":0.47099999999999997}]}]},{"osd":2,"up_from":105,"seq":450971566264,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":454520,"kb_used_data":15984,"kb_used_omap":11,"kb_used_meta":438452,"kb_avail":93290632,"statfs":{"total":95995035648,"available":95529607168,"internally_reserved":0,"allocated":16367616,"data_stored":15227963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":448975632},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.6309999999999998,"15min":3.0070000000000001},"min":{"1min":0.45200000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":25.236000000000001,"5min":25.236000000000001,"15min":87.132999999999996},"last":7.0919999999999996},{"interface":"front","average":{"1min":3.6549999999999998,"5min":3.516,"15min":2.6389999999999998},"min":{"1min":0.28899999999999998,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":25.198,"5min":25.198,"15min":84.820999999999998},"last":7.2380000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:19:55 2023","interfaces":[{"interface":"back","average":{"1min":5.734,"5min":3.9380000000000002,"15min":4.9500000000000002},"min":{"1min":0.25800000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":19.393000000000001,"5min":31.513999999999999,"15min":84.641000000000005},"last":4.1529999999999996},{"interface":"front","average":{"1min":4.319,"5min":3.109,"15min":3.1789999999999998},"min":{"1min":0.28799999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":19.983000000000001,"5min":19.983000000000001,"15min":84.751999999999995},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:20:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6479999999999997,"5min":5.532,"15min":4.3760000000000003},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":58.987000000000002,"5min":58.987000000000002,"15min":87.108000000000004},"last":2.867},{"interface":"front","average":{"1min":8.4789999999999992,"5min":6.0510000000000002,"15min":4.4199999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":59.457999999999998,"5min":59.457999999999998,"15min":86.158000000000001},"last":4.1130000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.9630000000000001,"15min":2.202},"min":{"1min":0.375,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":6.665,"5min":16.831,"15min":84.608000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":1.351,"5min":2.29,"15min":2.7959999999999998},"min":{"1min":0.29699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6710000000000003,"5min":18.847999999999999,"15min":87.019999999999996},"last":2.25}]},{"osd":5,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.3860000000000001,"15min":2.4620000000000002},"min":{"1min":0.33400000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.9909999999999997,"5min":18.875,"15min":84.498000000000005},"last":4.1799999999999997},{"interface":"front","average":{"1min":1.534,"5min":2.1749999999999998,"15min":2.3220000000000001},"min":{"1min":0.41399999999999998,"5min":0.34799999999999998,"15min":0.30199999999999999},"max":{"1min":6.7939999999999996,"5min":16.898,"15min":84.664000000000001},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.2149999999999999,"15min":2.4430000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32100000000000001},"max":{"1min":6.7220000000000004,"5min":16.872,"15min":84.715999999999994},"last":0.501},{"interface":"front","average":{"1min":1.53,"5min":2.472,"15min":2.5299999999999998},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":7.641,"5min":18.774000000000001,"15min":87.055000000000007},"last":2.21}]},{"osd":7,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":2.4079999999999999,"15min":2.3010000000000002},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":9.0289999999999999,"5min":18.954999999999998,"15min":84.596000000000004},"last":4.2149999999999999},{"interface":"front","average":{"1min":1.905,"5min":2.2770000000000001,"15min":2.2469999999999999},"min":{"1min":0.42899999999999999,"5min":0.42099999999999999,"15min":0.34200000000000003},"max":{"1min":6.7590000000000003,"5min":16.920999999999999,"15min":84.783000000000001},"last":0.52600000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402737,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":605080,"kb_used_data":134864,"kb_used_omap":3,"kb_used_meta":469948,"kb_avail":93140072,"statfs":{"total":95995035648,"available":95375433728,"internally_reserved":0,"allocated":138100736,"data_stored":136978494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":481226921},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.3929999999999998,"15min":3.016},"min":{"1min":0.35199999999999998,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":9.077,"5min":33.220999999999997,"15min":79.611000000000004},"last":5.0010000000000003},{"interface":"front","average":{"1min":3.4489999999999998,"5min":3.8929999999999998,"15min":3.1930000000000001},"min":{"1min":0.318,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":12.186,"5min":33.084000000000003,"15min":79.656000000000006},"last":5.875}]},{"osd":1,"last update":"Sat Sep 23 15:19:56 2023","interfaces":[{"interface":"back","average":{"1min":4.532,"5min":4.75,"15min":4.7530000000000001},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":38.497,"5min":119.343,"15min":119.343},"last":0.29099999999999998},{"interface":"front","average":{"1min":5.1900000000000004,"5min":4.8040000000000003,"15min":4.8239999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":45.761000000000003,"5min":119.505,"15min":119.505},"last":4.2169999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.0030000000000001,"15min":4.4850000000000003},"min":{"1min":0.34899999999999998,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":8.7699999999999996,"5min":18.893999999999998,"15min":109.077},"last":5.6790000000000003},{"interface":"front","average":{"1min":3.7360000000000002,"5min":4.6699999999999999,"15min":5.3620000000000001},"min":{"1min":0.48699999999999999,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":12.35,"5min":119.449,"15min":119.72799999999999},"last":5.7229999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":2.343,"15min":2.0139999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.256,"5min":12.958,"15min":54.051000000000002},"last":4.7300000000000004},{"interface":"front","average":{"1min":1.859,"5min":1.944,"15min":2.0369999999999999},"min":{"1min":0.36399999999999999,"5min":0.34999999999999998,"15min":0.26400000000000001},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":52.228999999999999},"last":0.36799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":2.3919999999999999,"15min":2.1579999999999999},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":12.228999999999999,"5min":12.999000000000001,"15min":56.960999999999999},"last":4.7789999999999999},{"interface":"front","average":{"1min":1.839,"5min":2.0750000000000002,"15min":2.23},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":12.478,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":2.52,"15min":2.3140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":12.294,"5min":13.029999999999999,"15min":78.102000000000004},"last":4.8170000000000002},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.105,"15min":2.3849999999999998},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.32300000000000001},"max":{"1min":12.446999999999999,"5min":12.446999999999999,"15min":78.028000000000006},"last":0.32400000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.1920000000000002,"15min":2.177},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.28299999999999997},"max":{"1min":12.522,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.762,"5min":2.1890000000000001,"15min":1.875},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":8.8230000000000004,"5min":18.696000000000002,"15min":18.696000000000002},"last":4.3369999999999997}]}]},{"osd":4,"up_from":0,"seq":133143986503,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724865,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":375040,"kb_used_data":95864,"kb_used_omap":0,"kb_used_meta":279168,"kb_avail":93370112,"statfs":{"total":95995035648,"available":95610994688,"internally_reserved":0,"allocated":98164736,"data_stored":97349937,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":285868032},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:09 2023","interfaces":[{"interface":"back","average":{"1min":7.7759999999999998,"5min":4.875,"15min":2.7349999999999999},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.20599999999999999},"max":{"1min":94.372,"5min":124.31,"15min":124.31},"last":4.048},{"interface":"front","average":{"1min":6.6890000000000001,"5min":5.117,"15min":2.7530000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":94.396000000000001,"5min":124.374,"15min":124.374},"last":0.24099999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.7280000000000002,"15min":2.2130000000000001},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":6.3360000000000003,"5min":124.38800000000001,"15min":124.38800000000001},"last":1.623},{"interface":"front","average":{"1min":1.363,"5min":3.0760000000000001,"15min":2.5830000000000002},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.8099999999999996,"5min":124.29000000000001,"15min":124.29000000000001},"last":3.3519999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:20:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":3.6419999999999999,"15min":3.3260000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23599999999999999},"max":{"1min":4.8780000000000001,"5min":193.00999999999999,"15min":193.00999999999999},"last":3.9409999999999998},{"interface":"front","average":{"1min":1.4850000000000001,"5min":3.9169999999999998,"15min":3.4020000000000001},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":5.1029999999999998,"5min":193.15700000000001,"15min":193.15700000000001},"last":1.0389999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":7.8499999999999996,"15min":4.9020000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":4.9320000000000004,"5min":186.56999999999999,"15min":186.56999999999999},"last":4.569},{"interface":"front","average":{"1min":1.4630000000000001,"5min":7.6669999999999998,"15min":4.694},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.46,"5min":186.74600000000001,"15min":186.74600000000001},"last":3.9220000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":2.27,"15min":1.7649999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":0.97899999999999998,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":2.3180000000000001,"15min":1.8340000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":1.1930000000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.32400000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.54900000000000004,"15min":1.119},"min":{"1min":0.32700000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0129999999999999,"5min":3.5550000000000002,"15min":15.196},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58199999999999996,"15min":0.999},"min":{"1min":0.35999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.167,"5min":2.9990000000000001,"15min":14.965999999999999},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56999999999999995,"15min":1.276},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.599,"5min":2.1869999999999998,"15min":15.301},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63900000000000001,"15min":1.2589999999999999},"min":{"1min":0.42399999999999999,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":1.9550000000000001,"5min":3.5379999999999998,"15min":15.762},"last":0.39700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":48369664,"data_stored":48365592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":104267776,"data_stored":104267776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":185204736,"data_stored":185204736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":81920000,"data_stored":81920000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:20:57.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:20:57.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:20:57.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:57.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:20:57.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:20:57.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:57.814 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:57.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:57.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:57.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:20:58.001 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.047 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:20:58.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:20:58.093 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:20:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:20:58] "GET /metrics HTTP/1.1" 200 36348 "" "Prometheus/2.43.0" 2023-09-23T15:20:58.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:20:58.185 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.237 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:20:58.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:20:58.312 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:58.341 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:20:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:58 smithi027 ceph-mon[195233]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:20:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:58 smithi027 ceph-mon[195233]: pgmap v804: 109 pgs: 109 active+clean; 9.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 778 op/s 2023-09-23T15:20:58.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:58 smithi017 ceph-mon[215646]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:20:58.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:58 smithi017 ceph-mon[215646]: pgmap v804: 109 pgs: 109 active+clean; 9.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 778 op/s 2023-09-23T15:20:58.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:58 smithi017 ceph-mon[222878]: from='client.45734 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:20:58.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:58 smithi017 ceph-mon[222878]: pgmap v804: 109 pgs: 109 active+clean; 9.4 MiB data, 2.2 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 778 op/s 2023-09-23T15:20:58.369 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.415 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:20:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:20:58.522 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:58.549 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.593 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:58.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:58.699 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:58.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.769 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:58.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:58.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:20:58.900 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:58.944 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:20:58.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:58.950 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:20:59.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:20:59.073 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:59.078 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:59.121 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:20:59.123 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:20:59.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:20:59.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:20:59.172 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:20:59.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:20:59.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:59.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:59.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3578536041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:59.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:59.267 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:59.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:59.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3578536041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:59.268 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:20:59 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:59.271 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:20:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:20:59.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:20:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3578536041' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:20:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:20:59 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:00.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[215646]: pgmap v805: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T15:21:00.307 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:00.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:00.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:21:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[222878]: pgmap v805: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T15:21:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[222878]: failed to encode map e279 with expected crc 2023-09-23T15:21:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:00.422 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[222878]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T15:21:00.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:00.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[215646]: failed to encode map e279 with expected crc 2023-09-23T15:21:00.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:00.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:00 smithi017 ceph-mon[215646]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T15:21:00.440 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:00 smithi027 ceph-mon[195233]: pgmap v805: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 649 op/s 2023-09-23T15:21:00.440 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:00 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:00.440 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:00 smithi027 ceph-mon[195233]: failed to encode map e279 with expected crc 2023-09-23T15:21:00.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:00 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:00.441 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:00 smithi027 ceph-mon[195233]: osdmap e279: 8 total, 8 up, 6 in 2023-09-23T15:21:00.443 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:00.494 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:21:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:00.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:21:00.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:21:00.628 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:00.674 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:21:00.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:00.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:21:00.780 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:00.807 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:00.850 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:00.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:00.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:00.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:01.027 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:01.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:01.133 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:01.166 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:01.773 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:21:01.773 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:21:01.773 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:21:01.773 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:21:02.068 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:02.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:02.172 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:02.205 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:02.255 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:02.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:02.360 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:02.388 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:02.434 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:02.438 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:02 smithi027 ceph-mon[195233]: pgmap v807: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 351 op/s 2023-09-23T15:21:02.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:02.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:02 smithi017 ceph-mon[215646]: pgmap v807: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 351 op/s 2023-09-23T15:21:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:02 smithi017 ceph-mon[222878]: pgmap v807: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 351 op/s 2023-09-23T15:21:02.540 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:02.567 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:02.611 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:02.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:02.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:21:02.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:02.745 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:02.787 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:02.889 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:21:02.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:02.919 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:02.962 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:02.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:02.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:03.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:03.124 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:03.234 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:03.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:03.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:03.309 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:21:03.311 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:21:03.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:03.343 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:03.377 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:03.437 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:03.441 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:03.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:03.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:03.554 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:03.572 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:03.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:03.601 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:03.622 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:03.642 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:03.660 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:21:03.680 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:03.681 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.681 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.682 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:21:03.725 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:03.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:03.727 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:03.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:03.861 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:03.912 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:03.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:03.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:04.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:04.047 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:04.188 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_2' removed 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[215646]: pgmap v808: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 302 op/s 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[215646]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2081154478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1604899108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[222878]: pgmap v808: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 302 op/s 2023-09-23T15:21:04.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[222878]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:04.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2081154478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:04.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1604899108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:04 smithi027 ceph-mon[195233]: pgmap v808: 109 pgs: 109 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 302 op/s 2023-09-23T15:21:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:04 smithi027 ceph-mon[195233]: from='client.35826 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2081154478' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1604899108' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:21:04.734 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":808,"stamp":"2023-09-23T15:21:02.625675+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":188643,"num_write_kb":6020214,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13229,"num_bytes_recovered":865933296,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":94468,"ondisk_log_size":94468,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":2196208,"kb_used_data":32192,"kb_used_omap":30,"kb_used_meta":2163937,"kb_avail":560274704,"statfs":{"total":575970213888,"available":573721296896,"internally_reserved":0,"allocated":32964608,"data_stored":26617314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":2215872088},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-198049816,"num_objects":-3023,"num_object_clones":0,"num_object_copies":-9069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-3023,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3023,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002687"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":36,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446934+0000","last_change":"2023-09-23T15:19:41.337833+0000","last_active":"2023-09-23T15:20:51.446934+0000","last_peered":"2023-09-23T15:20:51.446934+0000","last_clean":"2023-09-23T15:20:51.446934+0000","last_became_active":"2023-09-23T15:19:41.337459+0000","last_became_peered":"2023-09-23T15:19:41.337459+0000","last_unstale":"2023-09-23T15:20:51.446934+0000","last_undegraded":"2023-09-23T15:20:51.446934+0000","last_fullsized":"2023-09-23T15:20:51.446934+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:15:23.895299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":218,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444226+0000","last_change":"2023-09-23T15:19:41.351439+0000","last_active":"2023-09-23T15:20:51.444226+0000","last_peered":"2023-09-23T15:20:51.444226+0000","last_clean":"2023-09-23T15:20:51.444226+0000","last_became_active":"2023-09-23T15:19:41.351029+0000","last_became_peered":"2023-09-23T15:19:41.351029+0000","last_unstale":"2023-09-23T15:20:51.444226+0000","last_undegraded":"2023-09-23T15:20:51.444226+0000","last_fullsized":"2023-09-23T15:20:51.444226+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:42.043427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":167,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446201+0000","last_change":"2023-09-23T15:19:41.348299+0000","last_active":"2023-09-23T15:20:51.446201+0000","last_peered":"2023-09-23T15:20:51.446201+0000","last_clean":"2023-09-23T15:20:51.446201+0000","last_became_active":"2023-09-23T15:19:41.347335+0000","last_became_peered":"2023-09-23T15:19:41.347335+0000","last_unstale":"2023-09-23T15:20:51.446201+0000","last_undegraded":"2023-09-23T15:20:51.446201+0000","last_fullsized":"2023-09-23T15:20:51.446201+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:09:40.120925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":246,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.395408+0000","last_change":"2023-09-23T15:12:42.869522+0000","last_active":"2023-09-23T15:20:48.395408+0000","last_peered":"2023-09-23T15:20:48.395408+0000","last_clean":"2023-09-23T15:20:48.395408+0000","last_became_active":"2023-09-23T15:12:42.869214+0000","last_became_peered":"2023-09-23T15:12:42.869214+0000","last_unstale":"2023-09-23T15:20:48.395408+0000","last_undegraded":"2023-09-23T15:20:48.395408+0000","last_fullsized":"2023-09-23T15:20:48.395408+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:08.486290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":279,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.453076+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:20:51.453076+0000","last_peered":"2023-09-23T15:20:51.453076+0000","last_clean":"2023-09-23T15:20:51.453076+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:20:51.453076+0000","last_undegraded":"2023-09-23T15:20:51.453076+0000","last_fullsized":"2023-09-23T15:20:51.453076+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":253,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448737+0000","last_change":"2023-09-23T15:12:41.863359+0000","last_active":"2023-09-23T15:20:51.448737+0000","last_peered":"2023-09-23T15:20:51.448737+0000","last_clean":"2023-09-23T15:20:51.448737+0000","last_became_active":"2023-09-23T15:12:41.862878+0000","last_became_peered":"2023-09-23T15:12:41.862878+0000","last_unstale":"2023-09-23T15:20:51.448737+0000","last_undegraded":"2023-09-23T15:20:51.448737+0000","last_fullsized":"2023-09-23T15:20:51.448737+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:55.563804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":224,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398738+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:20:48.398738+0000","last_peered":"2023-09-23T15:20:48.398738+0000","last_clean":"2023-09-23T15:20:48.398738+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:20:48.398738+0000","last_undegraded":"2023-09-23T15:20:48.398738+0000","last_fullsized":"2023-09-23T15:20:48.398738+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":261,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.453149+0000","last_change":"2023-09-23T15:12:41.926783+0000","last_active":"2023-09-23T15:20:51.453149+0000","last_peered":"2023-09-23T15:20:51.453149+0000","last_clean":"2023-09-23T15:20:51.453149+0000","last_became_active":"2023-09-23T15:12:41.923023+0000","last_became_peered":"2023-09-23T15:12:41.923023+0000","last_unstale":"2023-09-23T15:20:51.453149+0000","last_undegraded":"2023-09-23T15:20:51.453149+0000","last_fullsized":"2023-09-23T15:20:51.453149+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:56.747801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":594,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382431+0000","last_change":"2023-09-23T15:19:44.390803+0000","last_active":"2023-09-23T15:20:48.382431+0000","last_peered":"2023-09-23T15:20:48.382431+0000","last_clean":"2023-09-23T15:20:48.382431+0000","last_became_active":"2023-09-23T15:19:44.390458+0000","last_became_peered":"2023-09-23T15:19:44.390458+0000","last_unstale":"2023-09-23T15:20:48.382431+0000","last_undegraded":"2023-09-23T15:20:48.382431+0000","last_fullsized":"2023-09-23T15:20:48.382431+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:01.314328+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":269,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.437220+0000","last_change":"2023-09-23T15:12:42.866498+0000","last_active":"2023-09-23T15:20:51.437220+0000","last_peered":"2023-09-23T15:20:51.437220+0000","last_clean":"2023-09-23T15:20:51.437220+0000","last_became_active":"2023-09-23T15:12:42.866283+0000","last_became_peered":"2023-09-23T15:12:42.866283+0000","last_unstale":"2023-09-23T15:20:51.437220+0000","last_undegraded":"2023-09-23T15:20:51.437220+0000","last_fullsized":"2023-09-23T15:20:51.437220+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:26.041225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":204,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.452044+0000","last_change":"2023-09-23T15:11:34.153136+0000","last_active":"2023-09-23T15:20:51.452044+0000","last_peered":"2023-09-23T15:20:51.452044+0000","last_clean":"2023-09-23T15:20:51.452044+0000","last_became_active":"2023-09-23T15:11:34.152800+0000","last_became_peered":"2023-09-23T15:11:34.152800+0000","last_unstale":"2023-09-23T15:20:51.452044+0000","last_undegraded":"2023-09-23T15:20:51.452044+0000","last_fullsized":"2023-09-23T15:20:51.452044+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:46.350556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":158,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444146+0000","last_change":"2023-09-23T15:19:41.400054+0000","last_active":"2023-09-23T15:20:51.444146+0000","last_peered":"2023-09-23T15:20:51.444146+0000","last_clean":"2023-09-23T15:20:51.444146+0000","last_became_active":"2023-09-23T15:19:41.358484+0000","last_became_peered":"2023-09-23T15:19:41.358484+0000","last_unstale":"2023-09-23T15:20:51.444146+0000","last_undegraded":"2023-09-23T15:20:51.444146+0000","last_fullsized":"2023-09-23T15:20:51.444146+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:12:25.075054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447291+0000","last_change":"2023-09-23T15:12:41.922599+0000","last_active":"2023-09-23T15:20:51.447291+0000","last_peered":"2023-09-23T15:20:51.447291+0000","last_clean":"2023-09-23T15:20:51.447291+0000","last_became_active":"2023-09-23T15:12:41.922141+0000","last_became_peered":"2023-09-23T15:12:41.922141+0000","last_unstale":"2023-09-23T15:20:51.447291+0000","last_undegraded":"2023-09-23T15:20:51.447291+0000","last_fullsized":"2023-09-23T15:20:51.447291+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:01.562519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":100,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447615+0000","last_change":"2023-09-23T15:19:41.396887+0000","last_active":"2023-09-23T15:20:51.447615+0000","last_peered":"2023-09-23T15:20:51.447615+0000","last_clean":"2023-09-23T15:20:51.447615+0000","last_became_active":"2023-09-23T15:19:41.358777+0000","last_became_peered":"2023-09-23T15:19:41.358777+0000","last_unstale":"2023-09-23T15:20:51.447615+0000","last_undegraded":"2023-09-23T15:20:51.447615+0000","last_fullsized":"2023-09-23T15:20:51.447615+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:30.237212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":247,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447414+0000","last_change":"2023-09-23T15:19:41.347627+0000","last_active":"2023-09-23T15:20:51.447414+0000","last_peered":"2023-09-23T15:20:51.447414+0000","last_clean":"2023-09-23T15:20:51.447414+0000","last_became_active":"2023-09-23T15:19:41.347230+0000","last_became_peered":"2023-09-23T15:19:41.347230+0000","last_unstale":"2023-09-23T15:20:51.447414+0000","last_undegraded":"2023-09-23T15:20:51.447414+0000","last_fullsized":"2023-09-23T15:20:51.447414+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:02.874471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":586,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.449012+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:20:51.449012+0000","last_peered":"2023-09-23T15:20:51.449012+0000","last_clean":"2023-09-23T15:20:51.449012+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:20:51.449012+0000","last_undegraded":"2023-09-23T15:20:51.449012+0000","last_fullsized":"2023-09-23T15:20:51.449012+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":51,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382167+0000","last_change":"2023-09-23T15:19:41.375930+0000","last_active":"2023-09-23T15:20:48.382167+0000","last_peered":"2023-09-23T15:20:48.382167+0000","last_clean":"2023-09-23T15:20:48.382167+0000","last_became_active":"2023-09-23T15:19:41.345893+0000","last_became_peered":"2023-09-23T15:19:41.345893+0000","last_unstale":"2023-09-23T15:20:48.382167+0000","last_undegraded":"2023-09-23T15:20:48.382167+0000","last_fullsized":"2023-09-23T15:20:48.382167+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:56.369302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":272,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448511+0000","last_change":"2023-09-23T15:12:42.893244+0000","last_active":"2023-09-23T15:20:51.448511+0000","last_peered":"2023-09-23T15:20:51.448511+0000","last_clean":"2023-09-23T15:20:51.448511+0000","last_became_active":"2023-09-23T15:12:42.892883+0000","last_became_peered":"2023-09-23T15:12:42.892883+0000","last_unstale":"2023-09-23T15:20:51.448511+0000","last_undegraded":"2023-09-23T15:20:51.448511+0000","last_fullsized":"2023-09-23T15:20:51.448511+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:20:34.475839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":598,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.438408+0000","last_change":"2023-09-23T15:19:44.374862+0000","last_active":"2023-09-23T15:20:51.438408+0000","last_peered":"2023-09-23T15:20:51.438408+0000","last_clean":"2023-09-23T15:20:51.438408+0000","last_became_active":"2023-09-23T15:19:44.374464+0000","last_became_peered":"2023-09-23T15:19:44.374464+0000","last_unstale":"2023-09-23T15:20:51.438408+0000","last_undegraded":"2023-09-23T15:20:51.438408+0000","last_fullsized":"2023-09-23T15:20:51.438408+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:35.374020+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":245,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446808+0000","last_change":"2023-09-23T15:12:42.868770+0000","last_active":"2023-09-23T15:20:51.446808+0000","last_peered":"2023-09-23T15:20:51.446808+0000","last_clean":"2023-09-23T15:20:51.446808+0000","last_became_active":"2023-09-23T15:12:42.868442+0000","last_became_peered":"2023-09-23T15:12:42.868442+0000","last_unstale":"2023-09-23T15:20:51.446808+0000","last_undegraded":"2023-09-23T15:20:51.446808+0000","last_fullsized":"2023-09-23T15:20:51.446808+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:57.725852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":247,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436975+0000","last_change":"2023-09-23T15:11:34.157109+0000","last_active":"2023-09-23T15:20:51.436975+0000","last_peered":"2023-09-23T15:20:51.436975+0000","last_clean":"2023-09-23T15:20:51.436975+0000","last_became_active":"2023-09-23T15:11:34.156837+0000","last_became_peered":"2023-09-23T15:11:34.156837+0000","last_unstale":"2023-09-23T15:20:51.436975+0000","last_undegraded":"2023-09-23T15:20:51.436975+0000","last_fullsized":"2023-09-23T15:20:51.436975+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:51.889342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":550,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457977+0000","last_change":"2023-09-23T15:19:44.401028+0000","last_active":"2023-09-23T15:20:51.457977+0000","last_peered":"2023-09-23T15:20:51.457977+0000","last_clean":"2023-09-23T15:20:51.457977+0000","last_became_active":"2023-09-23T15:19:44.400692+0000","last_became_peered":"2023-09-23T15:19:44.400692+0000","last_unstale":"2023-09-23T15:20:51.457977+0000","last_undegraded":"2023-09-23T15:20:51.457977+0000","last_fullsized":"2023-09-23T15:20:51.457977+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:35.692134+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":41,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447440+0000","last_change":"2023-09-23T15:19:41.357368+0000","last_active":"2023-09-23T15:20:51.447440+0000","last_peered":"2023-09-23T15:20:51.447440+0000","last_clean":"2023-09-23T15:20:51.447440+0000","last_became_active":"2023-09-23T15:19:41.356967+0000","last_became_peered":"2023-09-23T15:19:41.356967+0000","last_unstale":"2023-09-23T15:20:51.447440+0000","last_undegraded":"2023-09-23T15:20:51.447440+0000","last_fullsized":"2023-09-23T15:20:51.447440+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:01.852687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":233,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.395603+0000","last_change":"2023-09-23T15:11:34.144561+0000","last_active":"2023-09-23T15:20:48.395603+0000","last_peered":"2023-09-23T15:20:48.395603+0000","last_clean":"2023-09-23T15:20:48.395603+0000","last_became_active":"2023-09-23T15:11:34.140362+0000","last_became_peered":"2023-09-23T15:11:34.140362+0000","last_unstale":"2023-09-23T15:20:48.395603+0000","last_undegraded":"2023-09-23T15:20:48.395603+0000","last_fullsized":"2023-09-23T15:20:48.395603+0000","mapping_epoch":166,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":167,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:34.537477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":578,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436021+0000","last_change":"2023-09-23T15:16:46.270296+0000","last_active":"2023-09-23T15:20:51.436021+0000","last_peered":"2023-09-23T15:20:51.436021+0000","last_clean":"2023-09-23T15:20:51.436021+0000","last_became_active":"2023-09-23T15:16:46.269857+0000","last_became_peered":"2023-09-23T15:16:46.269857+0000","last_unstale":"2023-09-23T15:20:51.436021+0000","last_undegraded":"2023-09-23T15:20:51.436021+0000","last_fullsized":"2023-09-23T15:20:51.436021+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:04.733545+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":245,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446836+0000","last_change":"2023-09-23T15:04:40.703180+0000","last_active":"2023-09-23T15:20:51.446836+0000","last_peered":"2023-09-23T15:20:51.446836+0000","last_clean":"2023-09-23T15:20:51.446836+0000","last_became_active":"2023-09-23T15:04:40.702762+0000","last_became_peered":"2023-09-23T15:04:40.702762+0000","last_unstale":"2023-09-23T15:20:51.446836+0000","last_undegraded":"2023-09-23T15:20:51.446836+0000","last_fullsized":"2023-09-23T15:20:51.446836+0000","mapping_epoch":95,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:47.392201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":272,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.452808+0000","last_change":"2023-09-23T15:12:42.871236+0000","last_active":"2023-09-23T15:20:51.452808+0000","last_peered":"2023-09-23T15:20:51.452808+0000","last_clean":"2023-09-23T15:20:51.452808+0000","last_became_active":"2023-09-23T15:12:42.870937+0000","last_became_peered":"2023-09-23T15:12:42.870937+0000","last_unstale":"2023-09-23T15:20:51.452808+0000","last_undegraded":"2023-09-23T15:20:51.452808+0000","last_fullsized":"2023-09-23T15:20:51.452808+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:53.879109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"278'12046","reported_seq":12112,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.679128+0000","last_change":"2023-09-23T15:16:44.247449+0000","last_active":"2023-09-23T15:20:53.679128+0000","last_peered":"2023-09-23T15:20:53.679128+0000","last_clean":"2023-09-23T15:20:53.679128+0000","last_became_active":"2023-09-23T15:16:44.247016+0000","last_became_peered":"2023-09-23T15:16:44.247016+0000","last_unstale":"2023-09-23T15:20:53.679128+0000","last_undegraded":"2023-09-23T15:20:53.679128+0000","last_fullsized":"2023-09-23T15:20:53.679128+0000","mapping_epoch":223,"log_start":"247'6830","ondisk_log_start":"247'6830","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5216,"log_dups_size":0,"ondisk_log_size":5216,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:07.112241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12046,"num_write_kb":385472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":505,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.459030+0000","last_change":"2023-09-23T15:20:12.948292+0000","last_active":"2023-09-23T15:20:51.459030+0000","last_peered":"2023-09-23T15:20:51.459030+0000","last_clean":"2023-09-23T15:20:51.459030+0000","last_became_active":"2023-09-23T15:19:44.391800+0000","last_became_peered":"2023-09-23T15:19:44.391800+0000","last_unstale":"2023-09-23T15:20:51.459030+0000","last_undegraded":"2023-09-23T15:20:51.459030+0000","last_fullsized":"2023-09-23T15:20:51.459030+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:48.241223+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":49,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398546+0000","last_change":"2023-09-23T15:19:41.373544+0000","last_active":"2023-09-23T15:20:48.398546+0000","last_peered":"2023-09-23T15:20:48.398546+0000","last_clean":"2023-09-23T15:20:48.398546+0000","last_became_active":"2023-09-23T15:19:41.348943+0000","last_became_peered":"2023-09-23T15:19:41.348943+0000","last_unstale":"2023-09-23T15:20:48.398546+0000","last_undegraded":"2023-09-23T15:20:48.398546+0000","last_fullsized":"2023-09-23T15:20:48.398546+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:26.751596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":240,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.438305+0000","last_change":"2023-09-23T15:19:41.362493+0000","last_active":"2023-09-23T15:20:51.438305+0000","last_peered":"2023-09-23T15:20:51.438305+0000","last_clean":"2023-09-23T15:20:51.438305+0000","last_became_active":"2023-09-23T15:19:41.362177+0000","last_became_peered":"2023-09-23T15:19:41.362177+0000","last_unstale":"2023-09-23T15:20:51.438305+0000","last_undegraded":"2023-09-23T15:20:51.438305+0000","last_fullsized":"2023-09-23T15:20:51.438305+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:12:13.726981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"278'11668","reported_seq":11734,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.719548+0000","last_change":"2023-09-23T15:16:44.269132+0000","last_active":"2023-09-23T15:20:53.719548+0000","last_peered":"2023-09-23T15:20:53.719548+0000","last_clean":"2023-09-23T15:20:53.719548+0000","last_became_active":"2023-09-23T15:16:44.261329+0000","last_became_peered":"2023-09-23T15:16:44.261329+0000","last_unstale":"2023-09-23T15:20:53.719548+0000","last_undegraded":"2023-09-23T15:20:53.719548+0000","last_fullsized":"2023-09-23T15:20:53.719548+0000","mapping_epoch":223,"log_start":"248'6812","ondisk_log_start":"248'6812","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4856,"log_dups_size":0,"ondisk_log_size":4856,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:27:59.194927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11668,"num_write_kb":373376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":582,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444461+0000","last_change":"2023-09-23T15:16:46.628922+0000","last_active":"2023-09-23T15:20:51.444461+0000","last_peered":"2023-09-23T15:20:51.444461+0000","last_clean":"2023-09-23T15:20:51.444461+0000","last_became_active":"2023-09-23T15:16:46.265162+0000","last_became_peered":"2023-09-23T15:16:46.265162+0000","last_unstale":"2023-09-23T15:20:51.444461+0000","last_undegraded":"2023-09-23T15:20:51.444461+0000","last_fullsized":"2023-09-23T15:20:51.444461+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:06.414422+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":50,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.461248+0000","last_change":"2023-09-23T15:19:41.390268+0000","last_active":"2023-09-23T15:20:51.461248+0000","last_peered":"2023-09-23T15:20:51.461248+0000","last_clean":"2023-09-23T15:20:51.461248+0000","last_became_active":"2023-09-23T15:19:41.350379+0000","last_became_peered":"2023-09-23T15:19:41.350379+0000","last_unstale":"2023-09-23T15:20:51.461248+0000","last_undegraded":"2023-09-23T15:20:51.461248+0000","last_fullsized":"2023-09-23T15:20:51.461248+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:43.286774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":203,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446128+0000","last_change":"2023-09-23T15:19:41.348519+0000","last_active":"2023-09-23T15:20:51.446128+0000","last_peered":"2023-09-23T15:20:51.446128+0000","last_clean":"2023-09-23T15:20:51.446128+0000","last_became_active":"2023-09-23T15:19:41.347811+0000","last_became_peered":"2023-09-23T15:19:41.347811+0000","last_unstale":"2023-09-23T15:20:51.446128+0000","last_undegraded":"2023-09-23T15:20:51.446128+0000","last_fullsized":"2023-09-23T15:20:51.446128+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:33.450715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.9","version":"278'11532","reported_seq":16460,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.701357+0000","last_change":"2023-09-23T15:19:57.116571+0000","last_active":"2023-09-23T15:20:53.701357+0000","last_peered":"2023-09-23T15:20:53.701357+0000","last_clean":"2023-09-23T15:20:53.701357+0000","last_became_active":"2023-09-23T15:19:57.082962+0000","last_became_peered":"2023-09-23T15:19:57.082962+0000","last_unstale":"2023-09-23T15:20:53.701357+0000","last_undegraded":"2023-09-23T15:20:53.701357+0000","last_fullsized":"2023-09-23T15:20:53.701357+0000","mapping_epoch":260,"log_start":"244'5668","ondisk_log_start":"244'5668","created":223,"last_epoch_clean":261,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5864,"log_dups_size":0,"ondisk_log_size":5864,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:37.326045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11532,"num_write_kb":369024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2420,"num_bytes_recovered":158597120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":521,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448584+0000","last_change":"2023-09-23T15:19:44.393819+0000","last_active":"2023-09-23T15:20:51.448584+0000","last_peered":"2023-09-23T15:20:51.448584+0000","last_clean":"2023-09-23T15:20:51.448584+0000","last_became_active":"2023-09-23T15:19:44.393471+0000","last_became_peered":"2023-09-23T15:19:44.393471+0000","last_unstale":"2023-09-23T15:20:51.448584+0000","last_undegraded":"2023-09-23T15:20:51.448584+0000","last_fullsized":"2023-09-23T15:20:51.448584+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:56.059420+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":19,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.383074+0000","last_change":"2023-09-23T15:20:30.328653+0000","last_active":"2023-09-23T15:20:48.383074+0000","last_peered":"2023-09-23T15:20:48.383074+0000","last_clean":"2023-09-23T15:20:48.383074+0000","last_became_active":"2023-09-23T15:20:30.289411+0000","last_became_peered":"2023-09-23T15:20:30.289411+0000","last_unstale":"2023-09-23T15:20:48.383074+0000","last_undegraded":"2023-09-23T15:20:48.383074+0000","last_fullsized":"2023-09-23T15:20:48.383074+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:16.407018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,3],"acting":[2,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447080+0000","last_change":"2023-09-23T15:12:42.863800+0000","last_active":"2023-09-23T15:20:51.447080+0000","last_peered":"2023-09-23T15:20:51.447080+0000","last_clean":"2023-09-23T15:20:51.447080+0000","last_became_active":"2023-09-23T15:12:42.863458+0000","last_became_peered":"2023-09-23T15:12:42.863458+0000","last_unstale":"2023-09-23T15:20:51.447080+0000","last_undegraded":"2023-09-23T15:20:51.447080+0000","last_fullsized":"2023-09-23T15:20:51.447080+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:23.320500+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"278'11538","reported_seq":16403,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.698055+0000","last_change":"2023-09-23T15:19:58.104306+0000","last_active":"2023-09-23T15:20:53.698055+0000","last_peered":"2023-09-23T15:20:53.698055+0000","last_clean":"2023-09-23T15:20:53.698055+0000","last_became_active":"2023-09-23T15:19:58.073145+0000","last_became_peered":"2023-09-23T15:19:58.073145+0000","last_unstale":"2023-09-23T15:20:53.698055+0000","last_undegraded":"2023-09-23T15:20:53.698055+0000","last_fullsized":"2023-09-23T15:20:53.698055+0000","mapping_epoch":261,"log_start":"241'4833","ondisk_log_start":"241'4833","created":223,"last_epoch_clean":262,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6705,"log_dups_size":0,"ondisk_log_size":6705,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:21:12.135152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11538,"num_write_kb":369216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2385,"num_bytes_recovered":156237824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,5],"acting":[2,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":555,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444199+0000","last_change":"2023-09-23T15:20:10.484333+0000","last_active":"2023-09-23T15:20:51.444199+0000","last_peered":"2023-09-23T15:20:51.444199+0000","last_clean":"2023-09-23T15:20:51.444199+0000","last_became_active":"2023-09-23T15:19:44.404058+0000","last_became_peered":"2023-09-23T15:19:44.404058+0000","last_unstale":"2023-09-23T15:20:51.444199+0000","last_undegraded":"2023-09-23T15:20:51.444199+0000","last_fullsized":"2023-09-23T15:20:51.444199+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:10.484180+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:10.484180+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:18:30.529028+0000","scrub_duration":0.018611978000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436780+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:20:51.436780+0000","last_peered":"2023-09-23T15:20:51.436780+0000","last_clean":"2023-09-23T15:20:51.436780+0000","last_became_active":"2023-09-23T15:06:16.137345+0000","last_became_peered":"2023-09-23T15:06:16.137345+0000","last_unstale":"2023-09-23T15:20:51.436780+0000","last_undegraded":"2023-09-23T15:20:51.436780+0000","last_fullsized":"2023-09-23T15:20:51.436780+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:40:12.255241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":249,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390395+0000","last_change":"2023-09-23T15:19:41.353927+0000","last_active":"2023-09-23T15:20:48.390395+0000","last_peered":"2023-09-23T15:20:48.390395+0000","last_clean":"2023-09-23T15:20:48.390395+0000","last_became_active":"2023-09-23T15:19:41.353066+0000","last_became_peered":"2023-09-23T15:19:41.353066+0000","last_unstale":"2023-09-23T15:20:48.390395+0000","last_undegraded":"2023-09-23T15:20:48.390395+0000","last_fullsized":"2023-09-23T15:20:48.390395+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:28.877030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"278'11752","reported_seq":16186,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.686780+0000","last_change":"2023-09-23T15:20:05.659905+0000","last_active":"2023-09-23T15:20:53.686780+0000","last_peered":"2023-09-23T15:20:53.686780+0000","last_clean":"2023-09-23T15:20:53.686780+0000","last_became_active":"2023-09-23T15:20:05.566095+0000","last_became_peered":"2023-09-23T15:20:05.566095+0000","last_unstale":"2023-09-23T15:20:53.686780+0000","last_undegraded":"2023-09-23T15:20:53.686780+0000","last_fullsized":"2023-09-23T15:20:53.686780+0000","mapping_epoch":265,"log_start":"245'5890","ondisk_log_start":"245'5890","created":223,"last_epoch_clean":266,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5862,"log_dups_size":0,"ondisk_log_size":5862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:23:35.636458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11752,"num_write_kb":376064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2172,"num_bytes_recovered":142213120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":578,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444206+0000","last_change":"2023-09-23T15:16:46.267744+0000","last_active":"2023-09-23T15:20:51.444206+0000","last_peered":"2023-09-23T15:20:51.444206+0000","last_clean":"2023-09-23T15:20:51.444206+0000","last_became_active":"2023-09-23T15:16:46.267206+0000","last_became_peered":"2023-09-23T15:16:46.267206+0000","last_unstale":"2023-09-23T15:20:51.444206+0000","last_undegraded":"2023-09-23T15:20:51.444206+0000","last_fullsized":"2023-09-23T15:20:51.444206+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:55.124783+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":257,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.450094+0000","last_change":"2023-09-23T15:19:41.387781+0000","last_active":"2023-09-23T15:20:51.450094+0000","last_peered":"2023-09-23T15:20:51.450094+0000","last_clean":"2023-09-23T15:20:51.450094+0000","last_became_active":"2023-09-23T15:19:41.346358+0000","last_became_peered":"2023-09-23T15:19:41.346358+0000","last_unstale":"2023-09-23T15:20:51.450094+0000","last_undegraded":"2023-09-23T15:20:51.450094+0000","last_fullsized":"2023-09-23T15:20:51.450094+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:52.625203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":96,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448649+0000","last_change":"2023-09-23T15:19:41.357313+0000","last_active":"2023-09-23T15:20:51.448649+0000","last_peered":"2023-09-23T15:20:51.448649+0000","last_clean":"2023-09-23T15:20:51.448649+0000","last_became_active":"2023-09-23T15:19:41.356953+0000","last_became_peered":"2023-09-23T15:19:41.356953+0000","last_unstale":"2023-09-23T15:20:51.448649+0000","last_undegraded":"2023-09-23T15:20:51.448649+0000","last_fullsized":"2023-09-23T15:20:51.448649+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:05.944719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.e","version":"278'11790","reported_seq":11855,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.698676+0000","last_change":"2023-09-23T15:16:44.267825+0000","last_active":"2023-09-23T15:20:53.698676+0000","last_peered":"2023-09-23T15:20:53.698676+0000","last_clean":"2023-09-23T15:20:53.698676+0000","last_became_active":"2023-09-23T15:16:44.267464+0000","last_became_peered":"2023-09-23T15:16:44.267464+0000","last_unstale":"2023-09-23T15:20:53.698676+0000","last_undegraded":"2023-09-23T15:20:53.698676+0000","last_fullsized":"2023-09-23T15:20:53.698676+0000","mapping_epoch":223,"log_start":"241'5091","ondisk_log_start":"241'5091","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6699,"log_dups_size":0,"ondisk_log_size":6699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:01.614623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11790,"num_write_kb":377280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":583,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.438389+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:20:51.438389+0000","last_peered":"2023-09-23T15:20:51.438389+0000","last_clean":"2023-09-23T15:20:51.438389+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:20:51.438389+0000","last_undegraded":"2023-09-23T15:20:51.438389+0000","last_fullsized":"2023-09-23T15:20:51.438389+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":264,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446994+0000","last_change":"2023-09-23T15:19:41.358477+0000","last_active":"2023-09-23T15:20:51.446994+0000","last_peered":"2023-09-23T15:20:51.446994+0000","last_clean":"2023-09-23T15:20:51.446994+0000","last_became_active":"2023-09-23T15:19:41.356725+0000","last_became_peered":"2023-09-23T15:19:41.356725+0000","last_unstale":"2023-09-23T15:20:51.446994+0000","last_undegraded":"2023-09-23T15:20:51.446994+0000","last_fullsized":"2023-09-23T15:20:51.446994+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:41:18.157901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":202,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382740+0000","last_change":"2023-09-23T15:19:41.347475+0000","last_active":"2023-09-23T15:20:48.382740+0000","last_peered":"2023-09-23T15:20:48.382740+0000","last_clean":"2023-09-23T15:20:48.382740+0000","last_became_active":"2023-09-23T15:19:41.346823+0000","last_became_peered":"2023-09-23T15:19:41.346823+0000","last_unstale":"2023-09-23T15:20:48.382740+0000","last_undegraded":"2023-09-23T15:20:48.382740+0000","last_fullsized":"2023-09-23T15:20:48.382740+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:04.246472+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.d","version":"278'12002","reported_seq":12067,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.728216+0000","last_change":"2023-09-23T15:16:44.253277+0000","last_active":"2023-09-23T15:20:53.728216+0000","last_peered":"2023-09-23T15:20:53.728216+0000","last_clean":"2023-09-23T15:20:53.728216+0000","last_became_active":"2023-09-23T15:16:44.253017+0000","last_became_peered":"2023-09-23T15:16:44.253017+0000","last_unstale":"2023-09-23T15:20:53.728216+0000","last_undegraded":"2023-09-23T15:20:53.728216+0000","last_fullsized":"2023-09-23T15:20:53.728216+0000","mapping_epoch":223,"log_start":"247'6601","ondisk_log_start":"247'6601","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5401,"log_dups_size":0,"ondisk_log_size":5401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:02.969476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12002,"num_write_kb":384001,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":533,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457866+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:20:51.457866+0000","last_peered":"2023-09-23T15:20:51.457866+0000","last_clean":"2023-09-23T15:20:51.457866+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:20:51.457866+0000","last_undegraded":"2023-09-23T15:20:51.457866+0000","last_fullsized":"2023-09-23T15:20:51.457866+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":194,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381994+0000","last_change":"2023-09-23T15:19:41.347670+0000","last_active":"2023-09-23T15:20:48.381994+0000","last_peered":"2023-09-23T15:20:48.381994+0000","last_clean":"2023-09-23T15:20:48.381994+0000","last_became_active":"2023-09-23T15:19:41.347355+0000","last_became_peered":"2023-09-23T15:19:41.347355+0000","last_unstale":"2023-09-23T15:20:48.381994+0000","last_undegraded":"2023-09-23T15:20:48.381994+0000","last_fullsized":"2023-09-23T15:20:48.381994+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:52.941354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":247,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447354+0000","last_change":"2023-09-23T15:19:41.362667+0000","last_active":"2023-09-23T15:20:51.447354+0000","last_peered":"2023-09-23T15:20:51.447354+0000","last_clean":"2023-09-23T15:20:51.447354+0000","last_became_active":"2023-09-23T15:19:41.362383+0000","last_became_peered":"2023-09-23T15:19:41.362383+0000","last_unstale":"2023-09-23T15:20:51.447354+0000","last_undegraded":"2023-09-23T15:20:51.447354+0000","last_fullsized":"2023-09-23T15:20:51.447354+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:05.135891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.c","version":"278'11842","reported_seq":11906,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722712+0000","last_change":"2023-09-23T15:16:44.263031+0000","last_active":"2023-09-23T15:20:53.722712+0000","last_peered":"2023-09-23T15:20:53.722712+0000","last_clean":"2023-09-23T15:20:53.722712+0000","last_became_active":"2023-09-23T15:16:44.262753+0000","last_became_peered":"2023-09-23T15:16:44.262753+0000","last_unstale":"2023-09-23T15:20:53.722712+0000","last_undegraded":"2023-09-23T15:20:53.722712+0000","last_fullsized":"2023-09-23T15:20:53.722712+0000","mapping_epoch":223,"log_start":"247'6455","ondisk_log_start":"247'6455","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5387,"log_dups_size":0,"ondisk_log_size":5387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.537717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11842,"num_write_kb":378944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":578,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436015+0000","last_change":"2023-09-23T15:16:46.273192+0000","last_active":"2023-09-23T15:20:51.436015+0000","last_peered":"2023-09-23T15:20:51.436015+0000","last_clean":"2023-09-23T15:20:51.436015+0000","last_became_active":"2023-09-23T15:16:46.272781+0000","last_became_peered":"2023-09-23T15:16:46.272781+0000","last_unstale":"2023-09-23T15:20:51.436015+0000","last_undegraded":"2023-09-23T15:20:51.436015+0000","last_fullsized":"2023-09-23T15:20:51.436015+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:57.200511+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":282,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.453058+0000","last_change":"2023-09-23T15:19:41.352619+0000","last_active":"2023-09-23T15:20:51.453058+0000","last_peered":"2023-09-23T15:20:51.453058+0000","last_clean":"2023-09-23T15:20:51.453058+0000","last_became_active":"2023-09-23T15:19:41.352315+0000","last_became_peered":"2023-09-23T15:19:41.352315+0000","last_unstale":"2023-09-23T15:20:51.453058+0000","last_undegraded":"2023-09-23T15:20:51.453058+0000","last_fullsized":"2023-09-23T15:20:51.453058+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:07.918430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,5],"acting":[3,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":167,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457512+0000","last_change":"2023-09-23T15:19:41.347761+0000","last_active":"2023-09-23T15:20:51.457512+0000","last_peered":"2023-09-23T15:20:51.457512+0000","last_clean":"2023-09-23T15:20:51.457512+0000","last_became_active":"2023-09-23T15:19:41.347412+0000","last_became_peered":"2023-09-23T15:19:41.347412+0000","last_unstale":"2023-09-23T15:20:51.457512+0000","last_undegraded":"2023-09-23T15:20:51.457512+0000","last_fullsized":"2023-09-23T15:20:51.457512+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:23.311511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,5],"acting":[1,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"278'11830","reported_seq":16600,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.721882+0000","last_change":"2023-09-23T15:20:01.319545+0000","last_active":"2023-09-23T15:20:53.721882+0000","last_peered":"2023-09-23T15:20:53.721882+0000","last_clean":"2023-09-23T15:20:53.721882+0000","last_became_active":"2023-09-23T15:20:01.290760+0000","last_became_peered":"2023-09-23T15:20:01.290760+0000","last_unstale":"2023-09-23T15:20:53.721882+0000","last_undegraded":"2023-09-23T15:20:53.721882+0000","last_fullsized":"2023-09-23T15:20:53.721882+0000","mapping_epoch":263,"log_start":"248'6975","ondisk_log_start":"248'6975","created":223,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4855,"log_dups_size":0,"ondisk_log_size":4855,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:18.856179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11830,"num_write_kb":378560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2341,"num_bytes_recovered":153419776,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,1],"acting":[3,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":501,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448691+0000","last_change":"2023-09-23T15:16:46.409881+0000","last_active":"2023-09-23T15:20:51.448691+0000","last_peered":"2023-09-23T15:20:51.448691+0000","last_clean":"2023-09-23T15:20:51.448691+0000","last_became_active":"2023-09-23T15:16:46.269251+0000","last_became_peered":"2023-09-23T15:16:46.269251+0000","last_unstale":"2023-09-23T15:20:51.448691+0000","last_undegraded":"2023-09-23T15:20:51.448691+0000","last_fullsized":"2023-09-23T15:20:51.448691+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.409760+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.409760+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:40.287553+0000","scrub_duration":0.0015592480000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":457,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457807+0000","last_change":"2023-09-23T15:12:42.871506+0000","last_active":"2023-09-23T15:20:51.457807+0000","last_peered":"2023-09-23T15:20:51.457807+0000","last_clean":"2023-09-23T15:20:51.457807+0000","last_became_active":"2023-09-23T15:12:42.871062+0000","last_became_peered":"2023-09-23T15:12:42.871062+0000","last_unstale":"2023-09-23T15:20:51.457807+0000","last_undegraded":"2023-09-23T15:20:51.457807+0000","last_fullsized":"2023-09-23T15:20:51.457807+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:20.969709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":255,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436890+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:20:51.436890+0000","last_peered":"2023-09-23T15:20:51.436890+0000","last_clean":"2023-09-23T15:20:51.436890+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:20:51.436890+0000","last_undegraded":"2023-09-23T15:20:51.436890+0000","last_fullsized":"2023-09-23T15:20:51.436890+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":211,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382085+0000","last_change":"2023-09-23T15:19:41.350708+0000","last_active":"2023-09-23T15:20:48.382085+0000","last_peered":"2023-09-23T15:20:48.382085+0000","last_clean":"2023-09-23T15:20:48.382085+0000","last_became_active":"2023-09-23T15:19:41.350435+0000","last_became_peered":"2023-09-23T15:19:41.350435+0000","last_unstale":"2023-09-23T15:20:48.382085+0000","last_undegraded":"2023-09-23T15:20:48.382085+0000","last_fullsized":"2023-09-23T15:20:48.382085+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:15.309375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.7","version":"278'11564","reported_seq":11628,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722403+0000","last_change":"2023-09-23T15:16:44.265306+0000","last_active":"2023-09-23T15:20:53.722403+0000","last_peered":"2023-09-23T15:20:53.722403+0000","last_clean":"2023-09-23T15:20:53.722403+0000","last_became_active":"2023-09-23T15:16:44.264952+0000","last_became_peered":"2023-09-23T15:16:44.264952+0000","last_unstale":"2023-09-23T15:20:53.722403+0000","last_undegraded":"2023-09-23T15:20:53.722403+0000","last_fullsized":"2023-09-23T15:20:53.722403+0000","mapping_epoch":223,"log_start":"246'6130","ondisk_log_start":"246'6130","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5434,"log_dups_size":0,"ondisk_log_size":5434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:01.455591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":370048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":576,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436263+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:20:51.436263+0000","last_peered":"2023-09-23T15:20:51.436263+0000","last_clean":"2023-09-23T15:20:51.436263+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:20:51.436263+0000","last_undegraded":"2023-09-23T15:20:51.436263+0000","last_fullsized":"2023-09-23T15:20:51.436263+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":797,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436149+0000","last_change":"2023-09-23T15:12:41.858148+0000","last_active":"2023-09-23T15:20:51.436149+0000","last_peered":"2023-09-23T15:20:51.436149+0000","last_clean":"2023-09-23T15:20:51.436149+0000","last_became_active":"2023-09-23T15:12:41.857675+0000","last_became_peered":"2023-09-23T15:12:41.857675+0000","last_unstale":"2023-09-23T15:20:51.436149+0000","last_undegraded":"2023-09-23T15:20:51.436149+0000","last_fullsized":"2023-09-23T15:20:51.436149+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.396925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390662+0000","last_change":"2023-09-23T15:19:41.345625+0000","last_active":"2023-09-23T15:20:48.390662+0000","last_peered":"2023-09-23T15:20:48.390662+0000","last_clean":"2023-09-23T15:20:48.390662+0000","last_became_active":"2023-09-23T15:19:41.345256+0000","last_became_peered":"2023-09-23T15:19:41.345256+0000","last_unstale":"2023-09-23T15:20:48.390662+0000","last_undegraded":"2023-09-23T15:20:48.390662+0000","last_fullsized":"2023-09-23T15:20:48.390662+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:39.495384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":173,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457449+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:20:51.457449+0000","last_peered":"2023-09-23T15:20:51.457449+0000","last_clean":"2023-09-23T15:20:51.457449+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:20:51.457449+0000","last_undegraded":"2023-09-23T15:20:51.457449+0000","last_fullsized":"2023-09-23T15:20:51.457449+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"278'11472","reported_seq":15100,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722269+0000","last_change":"2023-09-23T15:20:14.742573+0000","last_active":"2023-09-23T15:20:53.722269+0000","last_peered":"2023-09-23T15:20:53.722269+0000","last_clean":"2023-09-23T15:20:53.722269+0000","last_became_active":"2023-09-23T15:20:14.670681+0000","last_became_peered":"2023-09-23T15:20:14.670681+0000","last_unstale":"2023-09-23T15:20:53.722269+0000","last_undegraded":"2023-09-23T15:20:53.722269+0000","last_fullsized":"2023-09-23T15:20:53.722269+0000","mapping_epoch":269,"log_start":"244'5666","ondisk_log_start":"244'5666","created":223,"last_epoch_clean":270,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5806,"log_dups_size":0,"ondisk_log_size":5806,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:51.982520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11472,"num_write_kb":367104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1767,"num_bytes_recovered":115605504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":603,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390712+0000","last_change":"2023-09-23T15:19:44.378313+0000","last_active":"2023-09-23T15:20:48.390712+0000","last_peered":"2023-09-23T15:20:48.390712+0000","last_clean":"2023-09-23T15:20:48.390712+0000","last_became_active":"2023-09-23T15:19:44.377464+0000","last_became_peered":"2023-09-23T15:19:44.377464+0000","last_unstale":"2023-09-23T15:20:48.390712+0000","last_undegraded":"2023-09-23T15:20:48.390712+0000","last_fullsized":"2023-09-23T15:20:48.390712+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:09:26.547533+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":893,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381924+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:20:48.381924+0000","last_peered":"2023-09-23T15:20:48.381924+0000","last_clean":"2023-09-23T15:20:48.381924+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:20:48.381924+0000","last_undegraded":"2023-09-23T15:20:48.381924+0000","last_fullsized":"2023-09-23T15:20:48.381924+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":154,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.445442+0000","last_change":"2023-09-23T15:12:41.922052+0000","last_active":"2023-09-23T15:20:51.445442+0000","last_peered":"2023-09-23T15:20:51.445442+0000","last_clean":"2023-09-23T15:20:51.445442+0000","last_became_active":"2023-09-23T15:12:41.921759+0000","last_became_peered":"2023-09-23T15:12:41.921759+0000","last_unstale":"2023-09-23T15:20:51.445442+0000","last_undegraded":"2023-09-23T15:20:51.445442+0000","last_fullsized":"2023-09-23T15:20:51.445442+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.950577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":177,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457436+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:20:51.457436+0000","last_peered":"2023-09-23T15:20:51.457436+0000","last_clean":"2023-09-23T15:20:51.457436+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:20:51.457436+0000","last_undegraded":"2023-09-23T15:20:51.457436+0000","last_fullsized":"2023-09-23T15:20:51.457436+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"278'11808","reported_seq":11873,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.718882+0000","last_change":"2023-09-23T15:16:44.257744+0000","last_active":"2023-09-23T15:20:53.718882+0000","last_peered":"2023-09-23T15:20:53.718882+0000","last_clean":"2023-09-23T15:20:53.718882+0000","last_became_active":"2023-09-23T15:16:44.257340+0000","last_became_peered":"2023-09-23T15:16:44.257340+0000","last_unstale":"2023-09-23T15:20:53.718882+0000","last_undegraded":"2023-09-23T15:20:53.718882+0000","last_fullsized":"2023-09-23T15:20:53.718882+0000","mapping_epoch":223,"log_start":"245'5984","ondisk_log_start":"245'5984","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5824,"log_dups_size":0,"ondisk_log_size":5824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:05:29.974460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11808,"num_write_kb":377856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":566,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436619+0000","last_change":"2023-09-23T15:11:37.198324+0000","last_active":"2023-09-23T15:20:51.436619+0000","last_peered":"2023-09-23T15:20:51.436619+0000","last_clean":"2023-09-23T15:20:51.436619+0000","last_became_active":"2023-09-23T15:11:37.197880+0000","last_became_peered":"2023-09-23T15:11:37.197880+0000","last_unstale":"2023-09-23T15:20:51.436619+0000","last_undegraded":"2023-09-23T15:20:51.436619+0000","last_fullsized":"2023-09-23T15:20:51.436619+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:45.074360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":569,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.404959+0000","last_change":"2023-09-23T15:12:45.079739+0000","last_active":"2023-09-23T15:20:48.404959+0000","last_peered":"2023-09-23T15:20:48.404959+0000","last_clean":"2023-09-23T15:20:48.404959+0000","last_became_active":"2023-09-23T15:12:45.079406+0000","last_became_peered":"2023-09-23T15:12:45.079406+0000","last_unstale":"2023-09-23T15:20:48.404959+0000","last_undegraded":"2023-09-23T15:20:48.404959+0000","last_fullsized":"2023-09-23T15:20:48.404959+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:27.861327+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":196,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.449292+0000","last_change":"2023-09-23T15:06:16.134611+0000","last_active":"2023-09-23T15:20:51.449292+0000","last_peered":"2023-09-23T15:20:51.449292+0000","last_clean":"2023-09-23T15:20:51.449292+0000","last_became_active":"2023-09-23T15:06:16.134329+0000","last_became_peered":"2023-09-23T15:06:16.134329+0000","last_unstale":"2023-09-23T15:20:51.449292+0000","last_undegraded":"2023-09-23T15:20:51.449292+0000","last_fullsized":"2023-09-23T15:20:51.449292+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:55:11.093424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":249,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447529+0000","last_change":"2023-09-23T15:20:30.274293+0000","last_active":"2023-09-23T15:20:51.447529+0000","last_peered":"2023-09-23T15:20:51.447529+0000","last_clean":"2023-09-23T15:20:51.447529+0000","last_became_active":"2023-09-23T15:20:30.273898+0000","last_became_peered":"2023-09-23T15:20:30.273898+0000","last_unstale":"2023-09-23T15:20:51.447529+0000","last_undegraded":"2023-09-23T15:20:51.447529+0000","last_fullsized":"2023-09-23T15:20:51.447529+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:13.354556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[6,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"278'11612","reported_seq":11678,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.722589+0000","last_change":"2023-09-23T15:16:44.265413+0000","last_active":"2023-09-23T15:20:53.722589+0000","last_peered":"2023-09-23T15:20:53.722589+0000","last_clean":"2023-09-23T15:20:53.722589+0000","last_became_active":"2023-09-23T15:16:44.265108+0000","last_became_peered":"2023-09-23T15:16:44.265108+0000","last_unstale":"2023-09-23T15:20:53.722589+0000","last_undegraded":"2023-09-23T15:20:53.722589+0000","last_fullsized":"2023-09-23T15:20:53.722589+0000","mapping_epoch":223,"log_start":"240'4870","ondisk_log_start":"240'4870","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":6742,"log_dups_size":0,"ondisk_log_size":6742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:33.573103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11612,"num_write_kb":371584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":551,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:56.774138+0000","last_change":"2023-09-23T15:19:44.386115+0000","last_active":"2023-09-23T15:20:56.774138+0000","last_peered":"2023-09-23T15:20:56.774138+0000","last_clean":"2023-09-23T15:20:56.774138+0000","last_became_active":"2023-09-23T15:19:44.385763+0000","last_became_peered":"2023-09-23T15:19:44.385763+0000","last_unstale":"2023-09-23T15:20:56.774138+0000","last_undegraded":"2023-09-23T15:20:56.774138+0000","last_fullsized":"2023-09-23T15:20:56.774138+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:11.988180+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":365,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.445274+0000","last_change":"2023-09-23T15:12:42.870488+0000","last_active":"2023-09-23T15:20:51.445274+0000","last_peered":"2023-09-23T15:20:51.445274+0000","last_clean":"2023-09-23T15:20:51.445274+0000","last_became_active":"2023-09-23T15:12:42.870011+0000","last_became_peered":"2023-09-23T15:12:42.870011+0000","last_unstale":"2023-09-23T15:20:51.445274+0000","last_undegraded":"2023-09-23T15:20:51.445274+0000","last_fullsized":"2023-09-23T15:20:51.445274+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:49.982649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":234,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.405695+0000","last_change":"2023-09-23T15:19:41.349771+0000","last_active":"2023-09-23T15:20:48.405695+0000","last_peered":"2023-09-23T15:20:48.405695+0000","last_clean":"2023-09-23T15:20:48.405695+0000","last_became_active":"2023-09-23T15:19:41.349349+0000","last_became_peered":"2023-09-23T15:19:41.349349+0000","last_unstale":"2023-09-23T15:20:48.405695+0000","last_undegraded":"2023-09-23T15:20:48.405695+0000","last_fullsized":"2023-09-23T15:20:48.405695+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:33:41.777395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.391748+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:20:48.391748+0000","last_peered":"2023-09-23T15:20:48.391748+0000","last_clean":"2023-09-23T15:20:48.391748+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:20:48.391748+0000","last_undegraded":"2023-09-23T15:20:48.391748+0000","last_fullsized":"2023-09-23T15:20:48.391748+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"278'11630","reported_seq":11694,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:53.723983+0000","last_change":"2023-09-23T15:16:44.257597+0000","last_active":"2023-09-23T15:20:53.723983+0000","last_peered":"2023-09-23T15:20:53.723983+0000","last_clean":"2023-09-23T15:20:53.723983+0000","last_became_active":"2023-09-23T15:16:44.257372+0000","last_became_peered":"2023-09-23T15:16:44.257372+0000","last_unstale":"2023-09-23T15:20:53.723983+0000","last_undegraded":"2023-09-23T15:20:53.723983+0000","last_fullsized":"2023-09-23T15:20:53.723983+0000","mapping_epoch":223,"log_start":"245'5931","ondisk_log_start":"245'5931","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:10.701673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11630,"num_write_kb":372160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":501,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.447256+0000","last_change":"2023-09-23T15:16:50.409267+0000","last_active":"2023-09-23T15:20:51.447256+0000","last_peered":"2023-09-23T15:20:51.447256+0000","last_clean":"2023-09-23T15:20:51.447256+0000","last_became_active":"2023-09-23T15:16:46.268302+0000","last_became_peered":"2023-09-23T15:16:46.268302+0000","last_unstale":"2023-09-23T15:20:51.447256+0000","last_undegraded":"2023-09-23T15:20:51.447256+0000","last_fullsized":"2023-09-23T15:20:51.447256+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:56.769263+0000","scrub_duration":0.0017923710000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":936,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.394769+0000","last_change":"2023-09-23T15:19:44.387907+0000","last_active":"2023-09-23T15:20:48.394769+0000","last_peered":"2023-09-23T15:20:48.394769+0000","last_clean":"2023-09-23T15:20:48.394769+0000","last_became_active":"2023-09-23T15:19:44.387402+0000","last_became_peered":"2023-09-23T15:19:44.387402+0000","last_unstale":"2023-09-23T15:20:48.394769+0000","last_undegraded":"2023-09-23T15:20:48.394769+0000","last_fullsized":"2023-09-23T15:20:48.394769+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:28.617856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,6],"acting":[0,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":153,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.457660+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:20:51.457660+0000","last_peered":"2023-09-23T15:20:51.457660+0000","last_clean":"2023-09-23T15:20:51.457660+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:20:51.457660+0000","last_undegraded":"2023-09-23T15:20:51.457660+0000","last_fullsized":"2023-09-23T15:20:51.457660+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":230,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390576+0000","last_change":"2023-09-23T15:12:43.161640+0000","last_active":"2023-09-23T15:20:48.390576+0000","last_peered":"2023-09-23T15:20:48.390576+0000","last_clean":"2023-09-23T15:20:48.390576+0000","last_became_active":"2023-09-23T15:12:43.161085+0000","last_became_peered":"2023-09-23T15:12:43.161085+0000","last_unstale":"2023-09-23T15:20:48.390576+0000","last_undegraded":"2023-09-23T15:20:48.390576+0000","last_fullsized":"2023-09-23T15:20:48.390576+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:56.655041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,1],"acting":[0,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"278'11946","reported_seq":16163,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:55.464474+0000","last_change":"2023-09-23T15:20:08.129441+0000","last_active":"2023-09-23T15:20:55.464474+0000","last_peered":"2023-09-23T15:20:55.464474+0000","last_clean":"2023-09-23T15:20:55.464474+0000","last_became_active":"2023-09-23T15:20:08.105976+0000","last_became_peered":"2023-09-23T15:20:08.105976+0000","last_unstale":"2023-09-23T15:20:55.464474+0000","last_undegraded":"2023-09-23T15:20:55.464474+0000","last_fullsized":"2023-09-23T15:20:55.464474+0000","mapping_epoch":267,"log_start":"248'7094","ondisk_log_start":"248'7094","created":223,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:42.365664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11946,"num_write_kb":382272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2062,"num_bytes_recovered":135135232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":557,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382917+0000","last_change":"2023-09-23T15:20:15.108514+0000","last_active":"2023-09-23T15:20:48.382917+0000","last_peered":"2023-09-23T15:20:48.382917+0000","last_clean":"2023-09-23T15:20:48.382917+0000","last_became_active":"2023-09-23T15:19:44.370417+0000","last_became_peered":"2023-09-23T15:19:44.370417+0000","last_unstale":"2023-09-23T15:20:48.382917+0000","last_undegraded":"2023-09-23T15:20:48.382917+0000","last_fullsized":"2023-09-23T15:20:48.382917+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:23.480089+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":19,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.381879+0000","last_change":"2023-09-23T15:20:30.309625+0000","last_active":"2023-09-23T15:20:48.381879+0000","last_peered":"2023-09-23T15:20:48.381879+0000","last_clean":"2023-09-23T15:20:48.381879+0000","last_became_active":"2023-09-23T15:20:30.261983+0000","last_became_peered":"2023-09-23T15:20:30.261983+0000","last_unstale":"2023-09-23T15:20:48.381879+0000","last_undegraded":"2023-09-23T15:20:48.381879+0000","last_fullsized":"2023-09-23T15:20:48.381879+0000","mapping_epoch":273,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":274,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:52.486099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":780,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.461466+0000","last_change":"2023-09-23T15:19:44.398589+0000","last_active":"2023-09-23T15:20:51.461466+0000","last_peered":"2023-09-23T15:20:51.461466+0000","last_clean":"2023-09-23T15:20:51.461466+0000","last_became_active":"2023-09-23T15:19:44.398077+0000","last_became_peered":"2023-09-23T15:19:44.398077+0000","last_unstale":"2023-09-23T15:20:51.461466+0000","last_undegraded":"2023-09-23T15:20:51.461466+0000","last_fullsized":"2023-09-23T15:20:51.461466+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:34.985355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":262,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.453152+0000","last_change":"2023-09-23T15:12:42.872072+0000","last_active":"2023-09-23T15:20:51.453152+0000","last_peered":"2023-09-23T15:20:51.453152+0000","last_clean":"2023-09-23T15:20:51.453152+0000","last_became_active":"2023-09-23T15:12:42.871605+0000","last_became_peered":"2023-09-23T15:12:42.871605+0000","last_unstale":"2023-09-23T15:20:51.453152+0000","last_undegraded":"2023-09-23T15:20:51.453152+0000","last_fullsized":"2023-09-23T15:20:51.453152+0000","mapping_epoch":184,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":185,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:24:40.062933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"278'11704","reported_seq":11771,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:55.464376+0000","last_change":"2023-09-23T15:16:44.268809+0000","last_active":"2023-09-23T15:20:55.464376+0000","last_peered":"2023-09-23T15:20:55.464376+0000","last_clean":"2023-09-23T15:20:55.464376+0000","last_became_active":"2023-09-23T15:16:44.261006+0000","last_became_peered":"2023-09-23T15:16:44.261006+0000","last_unstale":"2023-09-23T15:20:55.464376+0000","last_undegraded":"2023-09-23T15:20:55.464376+0000","last_fullsized":"2023-09-23T15:20:55.464376+0000","mapping_epoch":223,"log_start":"248'6851","ondisk_log_start":"248'6851","created":223,"last_epoch_clean":224,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:16:43.225650+0000","last_clean_scrub_stamp":"2023-09-23T15:16:43.225650+0000","objects_scrubbed":0,"log_size":4853,"log_dups_size":0,"ondisk_log_size":4853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:51.214545+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11704,"num_write_kb":374528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":579,"reported_epoch":279,"state":"active+clean","last_fresh":"2023-09-23T15:20:59.384825+0000","last_change":"2023-09-23T15:16:46.267143+0000","last_active":"2023-09-23T15:20:59.384825+0000","last_peered":"2023-09-23T15:20:59.384825+0000","last_clean":"2023-09-23T15:20:59.384825+0000","last_became_active":"2023-09-23T15:16:46.266728+0000","last_became_peered":"2023-09-23T15:16:46.266728+0000","last_unstale":"2023-09-23T15:20:59.384825+0000","last_undegraded":"2023-09-23T15:20:59.384825+0000","last_fullsized":"2023-09-23T15:20:59.384825+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.381851+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":755,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.448687+0000","last_change":"2023-09-23T15:11:37.196037+0000","last_active":"2023-09-23T15:20:51.448687+0000","last_peered":"2023-09-23T15:20:51.448687+0000","last_clean":"2023-09-23T15:20:51.448687+0000","last_became_active":"2023-09-23T15:11:37.195592+0000","last_became_peered":"2023-09-23T15:11:37.195592+0000","last_unstale":"2023-09-23T15:20:51.448687+0000","last_undegraded":"2023-09-23T15:20:51.448687+0000","last_fullsized":"2023-09-23T15:20:51.448687+0000","mapping_epoch":169,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":170,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:55.425624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":256,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.391837+0000","last_change":"2023-09-23T15:19:41.392698+0000","last_active":"2023-09-23T15:20:48.391837+0000","last_peered":"2023-09-23T15:20:48.391837+0000","last_clean":"2023-09-23T15:20:48.391837+0000","last_became_active":"2023-09-23T15:19:41.350174+0000","last_became_peered":"2023-09-23T15:19:41.350174+0000","last_unstale":"2023-09-23T15:20:48.391837+0000","last_undegraded":"2023-09-23T15:20:48.391837+0000","last_fullsized":"2023-09-23T15:20:48.391837+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:42:24.036011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":37,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.435998+0000","last_change":"2023-09-23T15:19:41.355318+0000","last_active":"2023-09-23T15:20:51.435998+0000","last_peered":"2023-09-23T15:20:51.435998+0000","last_clean":"2023-09-23T15:20:51.435998+0000","last_became_active":"2023-09-23T15:19:41.354940+0000","last_became_peered":"2023-09-23T15:19:41.354940+0000","last_unstale":"2023-09-23T15:20:51.435998+0000","last_undegraded":"2023-09-23T15:20:51.435998+0000","last_fullsized":"2023-09-23T15:20:51.435998+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:57:13.355798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":241,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.390788+0000","last_change":"2023-09-23T15:19:41.354183+0000","last_active":"2023-09-23T15:20:48.390788+0000","last_peered":"2023-09-23T15:20:48.390788+0000","last_clean":"2023-09-23T15:20:48.390788+0000","last_became_active":"2023-09-23T15:19:41.353520+0000","last_became_peered":"2023-09-23T15:19:41.353520+0000","last_unstale":"2023-09-23T15:20:48.390788+0000","last_undegraded":"2023-09-23T15:20:48.390788+0000","last_fullsized":"2023-09-23T15:20:48.390788+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:11.470291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.382743+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:20:48.382743+0000","last_peered":"2023-09-23T15:20:48.382743+0000","last_clean":"2023-09-23T15:20:48.382743+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:20:48.382743+0000","last_undegraded":"2023-09-23T15:20:48.382743+0000","last_fullsized":"2023-09-23T15:20:48.382743+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":271,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436872+0000","last_change":"2023-09-23T15:19:41.385820+0000","last_active":"2023-09-23T15:20:51.436872+0000","last_peered":"2023-09-23T15:20:51.436872+0000","last_clean":"2023-09-23T15:20:51.436872+0000","last_became_active":"2023-09-23T15:19:41.356653+0000","last_became_peered":"2023-09-23T15:19:41.356653+0000","last_unstale":"2023-09-23T15:20:51.436872+0000","last_undegraded":"2023-09-23T15:20:51.436872+0000","last_fullsized":"2023-09-23T15:20:51.436872+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:58.426360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.450178+0000","last_change":"2023-09-23T15:19:41.346796+0000","last_active":"2023-09-23T15:20:51.450178+0000","last_peered":"2023-09-23T15:20:51.450178+0000","last_clean":"2023-09-23T15:20:51.450178+0000","last_became_active":"2023-09-23T15:19:41.346289+0000","last_became_peered":"2023-09-23T15:19:41.346289+0000","last_unstale":"2023-09-23T15:20:51.450178+0000","last_undegraded":"2023-09-23T15:20:51.450178+0000","last_fullsized":"2023-09-23T15:20:51.450178+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:27.690863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":237,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.436872+0000","last_change":"2023-09-23T15:19:41.396637+0000","last_active":"2023-09-23T15:20:51.436872+0000","last_peered":"2023-09-23T15:20:51.436872+0000","last_clean":"2023-09-23T15:20:51.436872+0000","last_became_active":"2023-09-23T15:19:41.361792+0000","last_became_peered":"2023-09-23T15:19:41.361792+0000","last_unstale":"2023-09-23T15:20:51.436872+0000","last_undegraded":"2023-09-23T15:20:51.436872+0000","last_fullsized":"2023-09-23T15:20:51.436872+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:05:36.021130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":113,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.449423+0000","last_change":"2023-09-23T15:19:41.352434+0000","last_active":"2023-09-23T15:20:51.449423+0000","last_peered":"2023-09-23T15:20:51.449423+0000","last_clean":"2023-09-23T15:20:51.449423+0000","last_became_active":"2023-09-23T15:19:41.352220+0000","last_became_peered":"2023-09-23T15:19:41.352220+0000","last_unstale":"2023-09-23T15:20:51.449423+0000","last_undegraded":"2023-09-23T15:20:51.449423+0000","last_fullsized":"2023-09-23T15:20:51.449423+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:55.334410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":253,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444461+0000","last_change":"2023-09-23T15:06:16.129954+0000","last_active":"2023-09-23T15:20:51.444461+0000","last_peered":"2023-09-23T15:20:51.444461+0000","last_clean":"2023-09-23T15:20:51.444461+0000","last_became_active":"2023-09-23T15:06:16.129494+0000","last_became_peered":"2023-09-23T15:06:16.129494+0000","last_unstale":"2023-09-23T15:20:51.444461+0000","last_undegraded":"2023-09-23T15:20:51.444461+0000","last_fullsized":"2023-09-23T15:20:51.444461+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:31.638273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":267,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.444335+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:20:51.444335+0000","last_peered":"2023-09-23T15:20:51.444335+0000","last_clean":"2023-09-23T15:20:51.444335+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:20:51.444335+0000","last_undegraded":"2023-09-23T15:20:51.444335+0000","last_fullsized":"2023-09-23T15:20:51.444335+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":228,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T15:20:48.398802+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:20:48.398802+0000","last_peered":"2023-09-23T15:20:48.398802+0000","last_clean":"2023-09-23T15:20:48.398802+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:20:48.398802+0000","last_undegraded":"2023-09-23T15:20:48.398802+0000","last_fullsized":"2023-09-23T15:20:48.398802+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":102,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T15:20:51.446936+0000","last_change":"2023-09-23T15:19:41.355242+0000","last_active":"2023-09-23T15:20:51.446936+0000","last_peered":"2023-09-23T15:20:51.446936+0000","last_clean":"2023-09-23T15:20:51.446936+0000","last_became_active":"2023-09-23T15:19:41.354874+0000","last_became_peered":"2023-09-23T15:19:41.354874+0000","last_unstale":"2023-09-23T15:20:51.446936+0000","last_undegraded":"2023-09-23T15:20:51.446936+0000","last_fullsized":"2023-09-23T15:20:51.446936+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:50:25.245475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":187736,"num_write_kb":6007489,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13147,"num_bytes_recovered":861208576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126222336,"data_stored":126222336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":90055,"ondisk_log_size":90055,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":5040,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":126976,"data_stored":21414,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":236223201595,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":47,"seq":201863463231,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252644,"kb_used_data":3100,"kb_used_omap":0,"kb_used_meta":249536,"kb_avail":93492508,"statfs":{"total":95995035648,"available":95736328192,"internally_reserved":0,"allocated":3174400,"data_stored":2343039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":255524864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":2.2400000000000002,"15min":3.2440000000000002},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.19800000000000001},"max":{"1min":78.909000000000006,"5min":78.909000000000006,"15min":108.39400000000001},"last":0.58099999999999996},{"interface":"front","average":{"1min":6.923,"5min":2.8450000000000002,"15min":3.573},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":79.430000000000007,"5min":79.430000000000007,"15min":108.452},"last":0.55800000000000005}]},{"osd":1,"last update":"Sat Sep 23 15:20:56 2023","interfaces":[{"interface":"back","average":{"1min":9.5129999999999999,"5min":3.524,"15min":3.4300000000000002},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":111.90300000000001,"5min":111.90300000000001,"15min":111.90300000000001},"last":0.501},{"interface":"front","average":{"1min":9.3889999999999993,"5min":3.984,"15min":3.9550000000000001},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":111.917,"5min":111.917,"15min":111.917},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4560000000000004,"5min":4.7350000000000003,"15min":7.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.27500000000000002,"15min":0.255},"max":{"1min":80.875,"5min":86.569000000000003,"15min":180.374},"last":0.68700000000000006},{"interface":"front","average":{"1min":7.7530000000000001,"5min":4.4909999999999997,"15min":7.0229999999999997},"min":{"1min":0.39500000000000002,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":84.096000000000004,"5min":88.269000000000005,"15min":181.685},"last":0.53300000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.726,"15min":4.0529999999999999},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.827,"5min":84.004999999999995,"15min":102.254},"last":0.629},{"interface":"front","average":{"1min":2.8199999999999998,"5min":3.7240000000000002,"15min":4.1219999999999999},"min":{"1min":0.48299999999999998,"5min":0.36399999999999999,"15min":0.29799999999999999},"max":{"1min":6.9219999999999997,"5min":83.936000000000007,"15min":98.941999999999993},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52800000000000002,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":1.5569999999999999,"5min":1.7150000000000001,"15min":36.290999999999997},"last":0.76200000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.59199999999999997,"15min":1.129},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.337,"5min":2.6259999999999999,"15min":36.758000000000003},"last":0.751}]},{"osd":5,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":1.413},"min":{"1min":0.36199999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":1.4770000000000001,"5min":3.1789999999999998,"15min":131.90600000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.64300000000000002,"15min":1.4690000000000001},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.92800000000000005,"5min":3.431,"15min":131.61000000000001},"last":0.72699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.63300000000000001,"15min":1.032},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28999999999999998},"max":{"1min":2.4399999999999999,"5min":2.9449999999999998,"15min":17},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61799999999999999,"15min":0.97299999999999998},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.25800000000000001},"max":{"1min":1.581,"5min":1.8180000000000001,"15min":7.7809999999999997},"last":0.81399999999999995}]}]},{"osd":1,"up_from":183,"seq":785979015269,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":305220,"kb_used_data":7356,"kb_used_omap":6,"kb_used_meta":297849,"kb_avail":93439932,"statfs":{"total":95995035648,"available":95682490368,"internally_reserved":0,"allocated":7532544,"data_stored":6270504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":304997451},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":3.2709999999999999,"15min":3.4929999999999999},"min":{"1min":0.23200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":35.033000000000001,"5min":35.033000000000001,"15min":35.033000000000001},"last":0.36499999999999999},{"interface":"front","average":{"1min":4.0860000000000003,"5min":3.3679999999999999,"15min":4.1970000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":34.667999999999999,"5min":34.667999999999999,"15min":34.667999999999999},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":5.0970000000000004,"15min":4.4560000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.329000000000001,"5min":109.375,"15min":109.375},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.6319999999999999,"5min":3.673,"15min":3.9889999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":9.6219999999999999,"5min":105.16,"15min":105.16},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":6.5869999999999997,"5min":8.4589999999999996,"15min":9.2050000000000001},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":39.941000000000003,"5min":211.405,"15min":211.405},"last":0.48599999999999999},{"interface":"front","average":{"1min":7.3090000000000002,"5min":8.5109999999999992,"15min":10.135},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.27200000000000002},"max":{"1min":36.308,"5min":210.988,"15min":210.988},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":4.9829999999999997,"15min":3.4580000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.885,"5min":80.570999999999998,"15min":80.570999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.141,"5min":4.25,"15min":3.7109999999999999},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.9969999999999999,"5min":80.75,"15min":80.75},"last":0.71799999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.536,"15min":3.25},"min":{"1min":0.34200000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.833,"5min":34.512999999999998,"15min":34.512999999999998},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.952,"5min":3.1520000000000001,"15min":3.0310000000000001},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":12.795,"5min":37.911000000000001,"15min":37.911000000000001},"last":0.67600000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":4.6180000000000003,"15min":3.9500000000000002},"min":{"1min":0.38300000000000001,"5min":0.312,"15min":0.312},"max":{"1min":10.785,"5min":81.352000000000004,"15min":81.352000000000004},"last":0.627},{"interface":"front","average":{"1min":1.7889999999999999,"5min":5.2329999999999997,"15min":3.661},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.834,"5min":84.716999999999999,"15min":84.716999999999999},"last":0.55700000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.6619999999999999,"15min":2.9849999999999999},"min":{"1min":0.42799999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":12.861000000000001,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.7729999999999999,"15min":2.8159999999999998},"min":{"1min":0.40000000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.0279999999999996,"5min":13.282999999999999,"15min":16.841000000000001},"last":0.69099999999999995}]}]},{"osd":0,"up_from":95,"seq":408021893318,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":415188,"kb_used_data":7564,"kb_used_omap":8,"kb_used_meta":407607,"kb_avail":93329964,"statfs":{"total":95995035648,"available":95569883136,"internally_reserved":0,"allocated":7745536,"data_stored":6580460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":417390164},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:20:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.0430000000000001,"15min":3.8769999999999998},"min":{"1min":0.26500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":9.4909999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.476,"5min":3.21,"15min":3.8820000000000001},"min":{"1min":0.26500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.101000000000001,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4670000000000001,"5min":2.8039999999999998,"15min":4.1529999999999996},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.17000000000000001},"max":{"1min":6.9820000000000002,"5min":15.209,"15min":122.20099999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.1539999999999999,"5min":3.407,"15min":3.8900000000000001},"min":{"1min":0.28000000000000003,"5min":0.23499999999999999,"15min":0.18099999999999999},"max":{"1min":8.6679999999999993,"5min":64.466999999999999,"15min":122.071},"last":0.42299999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:20:43 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":6.3630000000000004,"15min":4.8250000000000002},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":14.785,"5min":146.315,"15min":146.315},"last":0.41199999999999998},{"interface":"front","average":{"1min":5.1890000000000001,"5min":7.5529999999999999,"15min":5.3849999999999998},"min":{"1min":0.39600000000000002,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":24.629000000000001,"5min":145.517,"15min":145.517},"last":0.47299999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.458,"5min":1.5820000000000001,"15min":1.9059999999999999},"min":{"1min":0.33900000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":4.609,"5min":7.0199999999999996,"15min":31.341999999999999},"last":0.623},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.008,"15min":2.0550000000000002},"min":{"1min":0.315,"5min":0.29699999999999999,"15min":0.27000000000000002},"max":{"1min":7.0460000000000003,"5min":13.566000000000001,"15min":99.168999999999997},"last":0.57999999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.9099999999999999,"5min":2.2770000000000001,"15min":2.1739999999999999},"min":{"1min":0.34799999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.5449999999999999,"5min":64.349999999999994,"15min":109.369},"last":0.66800000000000004},{"interface":"front","average":{"1min":1.589,"5min":1.6279999999999999,"15min":2.1040000000000001},"min":{"1min":0.38200000000000001,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":4.6390000000000002,"5min":6.9779999999999998,"15min":109.387},"last":0.56999999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":2.1840000000000002,"15min":2.1280000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.309},"max":{"1min":7.0739999999999998,"5min":13.529999999999999,"15min":99.114999999999995},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.4020000000000001,"15min":2.1309999999999998},"min":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":8.625,"5min":64.378,"15min":98.921999999999997},"last":0.73299999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":2.3450000000000002,"15min":2.2320000000000002},"min":{"1min":0.435,"5min":0.39300000000000002,"15min":0.36799999999999999},"max":{"1min":8.593,"5min":64.304000000000002,"15min":99.549999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":1.806,"5min":1.768,"15min":1.841},"min":{"1min":0.40400000000000003,"5min":0.39300000000000002,"15min":0.36399999999999999},"max":{"1min":8.6809999999999992,"5min":8.6809999999999992,"15min":31.225999999999999},"last":0.65800000000000003}]}]},{"osd":2,"up_from":105,"seq":450971566265,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":445944,"kb_used_data":3376,"kb_used_omap":11,"kb_used_meta":442548,"kb_avail":93299208,"statfs":{"total":95995035648,"available":95538388992,"internally_reserved":0,"allocated":3457024,"data_stored":2317371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":453169936},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":4.2169999999999996,"5min":3.6309999999999998,"15min":3.0070000000000001},"min":{"1min":0.45200000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":25.236000000000001,"5min":25.236000000000001,"15min":87.132999999999996},"last":0.39200000000000002},{"interface":"front","average":{"1min":3.6549999999999998,"5min":3.516,"15min":2.6389999999999998},"min":{"1min":0.28899999999999998,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":25.198,"5min":25.198,"15min":84.820999999999998},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:20:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.835,"15min":4.7889999999999997},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.522,"5min":31.513999999999999,"15min":84.641000000000005},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.1539999999999999,"5min":3.1280000000000001,"15min":3.153},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.763999999999999,"5min":19.983000000000001,"15min":84.751999999999995},"last":0.375}]},{"osd":3,"last update":"Sat Sep 23 15:20:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6479999999999997,"5min":5.532,"15min":4.3760000000000003},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":58.987000000000002,"5min":58.987000000000002,"15min":87.108000000000004},"last":0.30399999999999999},{"interface":"front","average":{"1min":8.4789999999999992,"5min":6.0510000000000002,"15min":4.4199999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":59.457999999999998,"5min":59.457999999999998,"15min":86.158000000000001},"last":0.35699999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":1.9630000000000001,"15min":2.202},"min":{"1min":0.375,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":6.665,"5min":16.831,"15min":84.608000000000004},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.351,"5min":2.29,"15min":2.7959999999999998},"min":{"1min":0.29699999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.6710000000000003,"5min":18.847999999999999,"15min":87.019999999999996},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8899999999999999,"5min":2.3860000000000001,"15min":2.4620000000000002},"min":{"1min":0.33400000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.9909999999999997,"5min":18.875,"15min":84.498000000000005},"last":0.435},{"interface":"front","average":{"1min":1.534,"5min":2.1749999999999998,"15min":2.3220000000000001},"min":{"1min":0.41399999999999998,"5min":0.34799999999999998,"15min":0.30199999999999999},"max":{"1min":6.7939999999999996,"5min":16.898,"15min":84.664000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.2149999999999999,"15min":2.4430000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32100000000000001},"max":{"1min":6.7220000000000004,"5min":16.872,"15min":84.715999999999994},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.53,"5min":2.472,"15min":2.5299999999999998},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":7.641,"5min":18.774000000000001,"15min":87.055000000000007},"last":0.47399999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.8879999999999999,"5min":2.4079999999999999,"15min":2.3010000000000002},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.31900000000000001},"max":{"1min":9.0289999999999999,"5min":18.954999999999998,"15min":84.596000000000004},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.905,"5min":2.2770000000000001,"15min":2.2469999999999999},"min":{"1min":0.42899999999999999,"5min":0.42099999999999999,"15min":0.34200000000000003},"max":{"1min":6.7590000000000003,"5min":16.920999999999999,"15min":84.783000000000001},"last":0.61599999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402739,"num_pgs":63,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":485936,"kb_used_data":7784,"kb_used_omap":3,"kb_used_meta":478140,"kb_avail":93259216,"statfs":{"total":95995035648,"available":95497437184,"internally_reserved":0,"allocated":7970816,"data_stored":6840492,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":489615529},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.3929999999999998,"15min":3.016},"min":{"1min":0.35199999999999998,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":9.077,"5min":33.220999999999997,"15min":79.611000000000004},"last":0.27000000000000002},{"interface":"front","average":{"1min":3.4489999999999998,"5min":3.8929999999999998,"15min":3.1930000000000001},"min":{"1min":0.318,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":12.186,"5min":33.084000000000003,"15min":79.656000000000006},"last":0.33400000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:20:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5179999999999998,"5min":5.359,"15min":4.8819999999999997},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":73.394000000000005,"5min":119.343,"15min":119.343},"last":0.23100000000000001},{"interface":"front","average":{"1min":7.2069999999999999,"5min":5.5380000000000003,"15min":4.9820000000000002},"min":{"1min":0.23999999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":75.918000000000006,"5min":119.505,"15min":119.505},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.0030000000000001,"15min":4.4850000000000003},"min":{"1min":0.34899999999999998,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":8.7699999999999996,"5min":18.893999999999998,"15min":109.077},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.7360000000000002,"5min":4.6699999999999999,"15min":5.3620000000000001},"min":{"1min":0.48699999999999999,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":12.35,"5min":119.449,"15min":119.72799999999999},"last":0.29699999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":2.343,"15min":2.0139999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.256,"5min":12.958,"15min":54.051000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.859,"5min":1.944,"15min":2.0369999999999999},"min":{"1min":0.36399999999999999,"5min":0.34999999999999998,"15min":0.26400000000000001},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":52.228999999999999},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":2.3919999999999999,"15min":2.1579999999999999},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":12.228999999999999,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":1.839,"5min":2.0750000000000002,"15min":2.23},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":12.478,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":2.52,"15min":2.3140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":12.294,"5min":13.029999999999999,"15min":78.102000000000004},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.105,"15min":2.3849999999999998},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.32300000000000001},"max":{"1min":12.446999999999999,"5min":12.446999999999999,"15min":78.028000000000006},"last":0.66500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.1920000000000002,"15min":2.177},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.28299999999999997},"max":{"1min":12.522,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.762,"5min":2.1890000000000001,"15min":1.875},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":8.8230000000000004,"5min":18.696000000000002,"15min":18.696000000000002},"last":0.68100000000000005}]}]},{"osd":4,"up_from":0,"seq":133143986504,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724867,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":291276,"kb_used_data":3012,"kb_used_omap":0,"kb_used_meta":288256,"kb_avail":93453876,"statfs":{"total":95995035648,"available":95696769024,"internally_reserved":0,"allocated":3084288,"data_stored":2265448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":295174144},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:09 2023","interfaces":[{"interface":"back","average":{"1min":7.7759999999999998,"5min":4.875,"15min":2.7349999999999999},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.20599999999999999},"max":{"1min":94.372,"5min":124.31,"15min":124.31},"last":0.42599999999999999},{"interface":"front","average":{"1min":6.6890000000000001,"5min":5.117,"15min":2.7530000000000001},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":94.396000000000001,"5min":124.374,"15min":124.374},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:20:01 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":2.7280000000000002,"15min":2.2130000000000001},"min":{"1min":0.26500000000000001,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":6.3360000000000003,"5min":124.38800000000001,"15min":124.38800000000001},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.363,"5min":3.0760000000000001,"15min":2.5830000000000002},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":5.8099999999999996,"5min":124.29000000000001,"15min":124.29000000000001},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:20:09 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":3.6419999999999999,"15min":3.3260000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23599999999999999},"max":{"1min":4.8780000000000001,"5min":193.00999999999999,"15min":193.00999999999999},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.4850000000000001,"5min":3.9169999999999998,"15min":3.4020000000000001},"min":{"1min":0.30599999999999999,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":5.1029999999999998,"5min":193.15700000000001,"15min":193.15700000000001},"last":0.50900000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":7.8499999999999996,"15min":4.9020000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":4.9320000000000004,"5min":186.56999999999999,"15min":186.56999999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.4630000000000001,"5min":7.6669999999999998,"15min":4.694},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.46,"5min":186.74600000000001,"15min":186.74600000000001},"last":0.58999999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:20:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":2.27,"15min":1.7649999999999999},"min":{"1min":0.29199999999999998,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":0.97899999999999998,"5min":138.68600000000001,"15min":138.68600000000001},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":2.3180000000000001,"15min":1.8340000000000001},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":1.1930000000000001,"5min":138.49600000000001,"15min":138.49600000000001},"last":0.375}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.54900000000000004,"15min":1.119},"min":{"1min":0.32700000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0129999999999999,"5min":3.5550000000000002,"15min":15.196},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58199999999999996,"15min":0.999},"min":{"1min":0.35999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.167,"5min":2.9990000000000001,"15min":14.965999999999999},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56999999999999995,"15min":1.276},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.599,"5min":2.1869999999999998,"15min":15.301},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63900000000000001,"15min":1.2589999999999999},"min":{"1min":0.42399999999999999,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":1.9550000000000001,"5min":3.5379999999999998,"15min":15.762},"last":0.54500000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":126222336,"data_stored":126222336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:21:04.741 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:21:04.741 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:21:04.741 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 2, 3, 4, 5, 6, 7, 1] 2023-09-23T15:21:04.741 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T15:21:04.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T15:21:04.763 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:04.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:04.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:04.905 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:05.099 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:21:04 smithi027 systemd[1]: Stopping Ceph osd.6 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:21:05.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:21:05 smithi027 conmon[131674]: 2023-09-23T15:21:05.006+0000 7fb090993700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:21:05.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:21:05 smithi027 conmon[131674]: 2023-09-23T15:21:05.006+0000 7fb090993700 -1 osd.6 280 *** Got signal Terminated *** 2023-09-23T15:21:05.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:21:05 smithi027 conmon[131674]: 2023-09-23T15:21:05.006+0000 7fb090993700 -1 osd.6 280 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:21:05.407 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:05.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:05.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:05.420 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:21:05.420 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:21:05.420 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T15:21:05.421 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T15:21:05.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[215646]: failed to encode map e280 with expected crc 2023-09-23T15:21:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:21:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[215646]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T15:21:05.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[215646]: osd.6 marked itself down and dead 2023-09-23T15:21:05.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[222878]: failed to encode map e280 with expected crc 2023-09-23T15:21:05.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:21:05.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[222878]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T15:21:05.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:05 smithi017 ceph-mon[222878]: osd.6 marked itself down and dead 2023-09-23T15:21:05.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:05.545 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:05 smithi027 ceph-mon[195233]: failed to encode map e280 with expected crc 2023-09-23T15:21:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:21:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:05 smithi027 ceph-mon[195233]: osdmap e280: 8 total, 8 up, 6 in 2023-09-23T15:21:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:05 smithi027 ceph-mon[195233]: osd.6 marked itself down and dead 2023-09-23T15:21:05.664 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:05.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:05.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:05.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:05.953 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:05.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:06.107 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:06.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: failed to encode map e280 with expected crc 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: pgmap v810: 93 pgs: 93 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[215646]: osdmap e281: 8 total, 7 up, 6 in 2023-09-23T15:21:06.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:06.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:06.311 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:06.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:06.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:06.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: failed to encode map e280 with expected crc 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: pgmap v810: 93 pgs: 93 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:06 smithi017 ceph-mon[222878]: osdmap e281: 8 total, 7 up, 6 in 2023-09-23T15:21:06.443 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:06.540 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:06.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:06.599 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:21:06 smithi027 bash[301542]: 37890724419e34c5f5048b5a04a94bb49428be2d0311dce8207b23b6c4cfbf41 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: failed to encode map e280 with expected crc 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: pgmap v810: 93 pgs: 93 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: failed to encode map e281 with expected crc 2023-09-23T15:21:06.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:06 smithi027 ceph-mon[195233]: osdmap e281: 8 total, 7 up, 6 in 2023-09-23T15:21:06.646 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:06.675 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:06.705 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:06.774 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:06.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:06.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:06.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:06.909 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:06.958 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:06.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:06.961 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:07.062 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:07.092 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:07.323 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:07 smithi027 ceph-mon[195233]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.323 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:07 smithi027 ceph-mon[195233]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.323 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:07 smithi027 ceph-mon[195233]: osdmap e282: 8 total, 7 up, 6 in 2023-09-23T15:21:07.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:21:07.396 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:07.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:07.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[215646]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[215646]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[215646]: osdmap e282: 8 total, 7 up, 6 in 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[222878]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[222878]: failed to encode map e282 with expected crc 2023-09-23T15:21:07.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:07 smithi017 ceph-mon[222878]: osdmap e282: 8 total, 7 up, 6 in 2023-09-23T15:21:07.532 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:07.594 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:07.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:07.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:07.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:07.729 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:07.730 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T15:21:07.784 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:21:07.784 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T15:21:08.083 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:08.086 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:08] "GET /metrics HTTP/1.1" 200 36340 "" "Prometheus/2.43.0" 2023-09-23T15:21:08.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:08.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:08.088 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:08.088 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:08.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:08.208 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:21:08.218 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:08.267 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:08.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:08.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:08.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:08.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[215646]: pgmap v813: 93 pgs: 10 peering, 13 stale+active+clean, 70 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 6.7 KiB/s rd, 3 op/s 2023-09-23T15:21:08.400 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/346403874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[222878]: pgmap v813: 93 pgs: 10 peering, 13 stale+active+clean, 70 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 6.7 KiB/s rd, 3 op/s 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/346403874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.403 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:08 smithi027 ceph-mon[195233]: pgmap v813: 93 pgs: 10 peering, 13 stale+active+clean, 70 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 6.7 KiB/s rd, 3 op/s 2023-09-23T15:21:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:08 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:21:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/346403874' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:08.893 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:09.004 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:09.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:09.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:09.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:09.135 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[215646]: failed to encode map e283 with expected crc 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[215646]: osdmap e283: 8 total, 7 up, 6 in 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[222878]: failed to encode map e283 with expected crc 2023-09-23T15:21:09.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:09.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:09 smithi017 ceph-mon[222878]: osdmap e283: 8 total, 7 up, 6 in 2023-09-23T15:21:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:09 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:09 smithi027 ceph-mon[195233]: failed to encode map e283 with expected crc 2023-09-23T15:21:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:09 smithi027 ceph-mon[195233]: osdmap e283: 8 total, 7 up, 6 in 2023-09-23T15:21:09.722 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:09.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:09.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:09.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:09.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:21:09.861 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:09.911 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:09.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:09.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:10.016 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:10.047 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:10.096 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:10.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:10.201 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:10.232 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:10.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:10 smithi017 ceph-mon[215646]: failed to encode map e283 with expected crc 2023-09-23T15:21:10.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:10 smithi017 ceph-mon[215646]: pgmap v815: 93 pgs: 9 active+undersized, 10 peering, 11 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 9.2 KiB/s rd, 25 KiB/s wr, 7 op/s; 18 KiB/s, 0 objects/s recovering 2023-09-23T15:21:10.263 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:10 smithi017 ceph-mon[222878]: failed to encode map e283 with expected crc 2023-09-23T15:21:10.263 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:10 smithi017 ceph-mon[222878]: pgmap v815: 93 pgs: 9 active+undersized, 10 peering, 11 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 9.2 KiB/s rd, 25 KiB/s wr, 7 op/s; 18 KiB/s, 0 objects/s recovering 2023-09-23T15:21:10.435 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:10.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:10.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:10.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:10.566 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:10 smithi027 ceph-mon[195233]: failed to encode map e283 with expected crc 2023-09-23T15:21:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:10 smithi027 ceph-mon[195233]: pgmap v815: 93 pgs: 9 active+undersized, 10 peering, 11 stale+active+clean, 63 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 9.2 KiB/s rd, 25 KiB/s wr, 7 op/s; 18 KiB/s, 0 objects/s recovering 2023-09-23T15:21:10.612 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:10.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:10.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:10.716 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:10.743 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:10.994 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:10.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:10.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:11.020 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:11.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:11.126 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:11.219 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' created 2023-09-23T15:21:11.237 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/470716225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.238 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.238 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/470716225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.238 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.239 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:11.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:11.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:11.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:11.373 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:11.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:11.588 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/470716225' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T15:21:11.612 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:11.633 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:11.652 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:11.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:11.693 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:11.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:21:11.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:21:11.964 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: failed to encode map e283 with expected crc 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: pgmap v816: 93 pgs: 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 17/84 objects degraded (20.238%); 39 KiB/s, 0 objects/s recovering 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 17/84 objects degraded (20.238%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: osdmap e284: 8 total, 7 up, 6 in 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.269 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/850870827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: failed to encode map e283 with expected crc 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: pgmap v816: 93 pgs: 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 17/84 objects degraded (20.238%); 39 KiB/s, 0 objects/s recovering 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 17/84 objects degraded (20.238%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: osdmap e284: 8 total, 7 up, 6 in 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.270 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/850870827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:12.276 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:12.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:12.382 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:12.412 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:12.461 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:12.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:12.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:12.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: failed to encode map e283 with expected crc 2023-09-23T15:21:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: pgmap v816: 93 pgs: 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 21 KiB/s rd, 55 KiB/s wr, 16 op/s; 17/84 objects degraded (20.238%); 39 KiB/s, 0 objects/s recovering 2023-09-23T15:21:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 17/84 objects degraded (20.238%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: osdmap e284: 8 total, 7 up, 6 in 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: failed to encode map e284 with expected crc 2023-09-23T15:21:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/850870827' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:12.600 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:12.603 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T15:21:12.746 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:12.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:12.785 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6] out_osds: [7, 4] dead_osds: [6] live_osds: [0, 2, 3, 4, 5, 7, 1] 2023-09-23T15:21:12.785 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:21:12.785 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T15:21:12.785 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T15:21:12.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:12.883 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:12.933 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:12.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:12.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:13.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:13.065 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:13.407 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:21:13.407 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:21:13.463 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:13.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:13.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:13.509 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:13.570 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:13.595 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.595 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: osdmap e285: 8 total, 7 up, 6 in 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: osdmap e285: 8 total, 7 up, 6 in 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:13.596 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:13.598 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: osdmap e285: 8 total, 7 up, 6 in 2023-09-23T15:21:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: failed to encode map e285 with expected crc 2023-09-23T15:21:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:13.709 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:13.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:13.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:13.817 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:13.849 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:13.897 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:13.901 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:13.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:13.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:13.950 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78608 6.b 6:da0fb1f2:::benchmark_data_smithi017_350392_object78607:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.555660+0000", 2023-09-23T15:21:13.960 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 512.401002189, 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7826397269999998, 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78608 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.961 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555660+0000", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555660+0000", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555660+0000", 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:21:13.962 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555687+0000", 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6996999999999999e-05 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555696+0000", 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8919999999999995e-06 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.963 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555705+0000", 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0490000000000004e-06 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555762+0000", 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7160999999999997e-05 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:13.964 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.555920+0000", 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158376 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 6", 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.556082+0000", 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016152000000000001 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.571878+0000", 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.015796634 2023-09-23T15:21:13.965 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.571894+0000", 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5583999999999999e-05 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852506+0000", 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28061221400000003 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.966 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852520+0000", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4219000000000001e-05 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.307861+0000", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4553402850000001 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.308486+0000", 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00062560800000000005 2023-09-23T15:21:13.967 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.338191+0000", 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029704939999999999 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.338300+0000", 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010905599999999999 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:13.968 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78636 6.4 6:2f1667ef:::benchmark_data_smithi017_350392_object78635:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.852651+0000", 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 512.104011507, 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3757485270000001, 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78636 2023-09-23T15:21:13.969 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852651+0000", 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852651+0000", 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.970 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852649+0000", 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.852680+0000", 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.112e-05 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:13.971 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854248+0000", 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015679610000000001 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854263+0000", 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5092000000000001e-05 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854303+0000", 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9548999999999997e-05 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.972 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854314+0000", 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0541e-05 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.204685+0000", 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.350371671 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:13.973 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.207200+0000", 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025151119999999999 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.207399+0000", 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019852300000000001 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.217441+0000", 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010042572 2023-09-23T15:21:13.974 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218191+0000", 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00074997200000000001 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.225350+0000", 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071583999999999997 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.975 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.225421+0000", 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0737999999999999e-05 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228400+0000", 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029790260000000001 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.976 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45074.0:78638 6.4 6:2c1a4d9b:::benchmark_data_smithi017_350392_object78637:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:12:41.854279+0000", 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 512.10238344599998, 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3778235599999999, 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45074", 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 78638 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.977 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854279+0000", 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854279+0000", 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.978 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854278+0000", 2023-09-23T15:21:13.979 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:13.979 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.979 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.979 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.980 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854336+0000", 2023-09-23T15:21:13.980 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8251999999999997e-05 2023-09-23T15:21:13.980 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.980 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854347+0000", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1168999999999999e-05 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854354+0000", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9299999999999997e-06 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854603+0000", 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024914000000000002 2023-09-23T15:21:13.981 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:41.854613+0000", 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4199999999999996e-06 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.213452+0000", 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3588396710000001 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.982 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.213649+0000", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019720999999999999 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.213897+0000", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024709500000000002 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218200+0000", 2023-09-23T15:21:13.983 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043039150000000002 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.218510+0000", 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00030988700000000001 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228724+0000", 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010213336 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.984 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.228803+0000", 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9524999999999995e-05 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:12:43.232103+0000", 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003299448 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.985 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165850 7.8 e271/261 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head v 271'10159, mlcod=271'10156)", 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.496175835999999, 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35471572400000001, 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.986 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460486+0000", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460491+0000", 2023-09-23T15:21:13.987 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4039999999999997e-06 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460497+0000", 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460502+0000", 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4759999999999997e-06 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.988 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460547+0000", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4233999999999997e-05 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460568+0000", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1189999999999999e-05 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468865+0000", 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082969110000000006 2023-09-23T15:21:13.989 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815202+0000", 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34633774499999997 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165851 7.1 7:8fdd4296:::benchmark_data_smithi017_422824_object71981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.495331297999996, 2023-09-23T15:21:13.990 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36053184399999999, 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165851 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.991 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461330+0000", 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.992 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461336+0000", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5899999999999996e-06 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461348+0000", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2065000000000001e-05 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461357+0000", 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8189999999999998e-06 2023-09-23T15:21:13.993 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9935999999999997e-05 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461554+0000", 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013704900000000001 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.994 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121172 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815161+0000", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35348590600000002 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815321+0000", 2023-09-23T15:21:13.995 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160006 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821782+0000", 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064606120000000001 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821827+0000", 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5030000000000001e-05 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:13.996 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821863+0000", 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6208999999999998e-05 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165853 7.3 7:c0488a25:::benchmark_data_smithi017_422824_object71983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.495290394999998, 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35803834699999998, 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:13.997 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165853 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:21:13.998 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461375+0000", 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2899999999999998e-06 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:13.999 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461393+0000", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8119000000000001e-05 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461404+0000", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0916e-05 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464554+0000", 2023-09-23T15:21:14.000 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003149799 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464660+0000", 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106097 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464785+0000", 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012551000000000001 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.001 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818430+0000", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35364495699999998 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818460+0000", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9578999999999999e-05 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819308+0000", 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084795300000000003 2023-09-23T15:21:14.002 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819358+0000", 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9982999999999999e-05 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819411+0000", 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2698000000000003e-05 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.003 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165854 7.a 7:57b0f785:::benchmark_data_smithi017_422824_object71984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.495244503999999, 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35469288599999999, 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.004 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165854 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.005 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461421+0000", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1570000000000002e-06 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461429+0000", 2023-09-23T15:21:14.006 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461435+0000", 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6699999999999999e-06 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461497+0000", 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1896999999999994e-05 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.007 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461605+0000", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108937 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9722999999999996e-05 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815166+0000", 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35349039599999998 2023-09-23T15:21:14.008 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4358999999999996e-05 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816030+0000", 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077006299999999998 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.009 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816073+0000", 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3195999999999998e-05 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816111+0000", 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7719999999999998e-05 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165856 7.2 7:45443312:::benchmark_data_smithi017_422824_object71986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:14.010 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.495216712999998, 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36007705000000001, 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165856 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.011 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461445+0000", 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461448+0000", 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1530000000000001e-06 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461456+0000", 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1609999999999997e-06 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461461+0000", 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4960000000000004e-06 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464405+0000", 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029436470000000002 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464508+0000", 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010341899999999999 2023-09-23T15:21:14.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464624+0000", 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115374 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815184+0000", 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35056013400000002 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815339+0000", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015575800000000001 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821434+0000", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060949569999999998 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821484+0000", 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9515999999999998e-05 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821523+0000", 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8986999999999998e-05 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165858 7.3 7:c518666c:::benchmark_data_smithi017_422824_object71988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.495196614000001, 2023-09-23T15:21:14.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36584072299999998, 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165858 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461468+0000", 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4530000000000001e-06 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461474+0000", 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7010000000000001e-06 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461486+0000", 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.201e-05 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467017+0000", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055312130000000001 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34830823100000002 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815477+0000", 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015180900000000001 2023-09-23T15:21:14.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819449+0000", 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039723479999999997 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.823533+0000", 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040831690000000002 2023-09-23T15:21:14.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827223+0000", 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036904910000000002 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827268+0000", 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5204000000000003e-05 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827307+0000", 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8433999999999998e-05 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165860 7.6 e271/223 7:667b3ce6:::benchmark_data_smithi017_422824_object71990:head v 271'10449, mlcod=271'10447)", 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.490166408999997, 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34876393300000003, 2023-09-23T15:21:14.024 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466495+0000", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466503+0000", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7440000000000007e-06 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466515+0000", 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2150999999999999e-05 2023-09-23T15:21:14.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466523+0000", 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466566+0000", 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2422000000000001e-05 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466587+0000", 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1475e-05 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815220+0000", 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34863227000000002 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0509000000000001e-05 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.028 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.489292857999999, 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34789368300000001, 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467369+0000", 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467375+0000", 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6760000000000004e-06 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815177+0000", 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.347802218 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2822e-05 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815244+0000", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4230000000000001e-05 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815253+0000", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1579999999999994e-06 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815263+0000", 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0268999999999999e-05 2023-09-23T15:21:14.032 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.487594412, 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34704774999999999, 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469067+0000", 2023-09-23T15:21:14.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469074+0000", 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2230000000000004e-06 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815176+0000", 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.346102403 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3959e-05 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815563+0000", 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037292800000000002 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816024+0000", 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046142900000000002 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816116+0000", 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1516999999999998e-05 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165851 7.1 e271/263 ondisk, result = 0)", 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 52.485088775000001, 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34375118199999999, 2023-09-23T15:21:14.037 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471573+0000", 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471579+0000", 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1940000000000003e-06 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815160+0000", 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.343580518 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815175+0000", 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5639000000000001e-05 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815306+0000", 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130894 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815315+0000", 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5769999999999994e-06 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0227999999999999e-05 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179348 7.1 7:83a21fe1:::benchmark_data_smithi017_422824_object85478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.274844057000003, 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32463872399999999, 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179348 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.042 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681817+0000", 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681825+0000", 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0039999999999999e-06 2023-09-23T15:21:14.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681838+0000", 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3648e-05 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681846+0000", 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0199999999999994e-06 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685481+0000", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00363489 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685642+0000", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016116599999999999 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685776+0000", 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133442 2023-09-23T15:21:14.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688382+0000", 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026061270000000002 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688535+0000", 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015359299999999999 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006352+0000", 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31781674399999998 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006412+0000", 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0061e-05 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006457+0000", 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5016999999999999e-05 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.049 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179350 7.1 7:87665378:::benchmark_data_smithi017_422824_object85480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267940277999998, 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31832527799999999, 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.050 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179350 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688721+0000", 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688727+0000", 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.806e-06 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688738+0000", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1192000000000001e-05 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688745+0000", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2049999999999996e-06 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688796+0000", 2023-09-23T15:21:14.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0779000000000003e-05 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688930+0000", 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013444299999999999 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689051+0000", 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120451 2023-09-23T15:21:14.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.691835+0000", 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027841459999999999 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694698+0000", 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028631170000000001 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.055 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006962+0000", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31226363000000001 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007010+0000", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8396999999999998e-05 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007048+0000", 2023-09-23T15:21:14.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7428999999999997e-05 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179357 7.2 7:4242eeb1:::benchmark_data_smithi017_422824_object85487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267893802000003, 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.317811762, 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.057 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179357 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688772+0000", 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.569e-06 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688782+0000", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8120000000000002e-06 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688788+0000", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6980000000000001e-06 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688840+0000", 2023-09-23T15:21:14.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1610000000000002e-05 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688932+0000", 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2485999999999995e-05 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048049699999999998 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006235+0000", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31682244500000001 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006406+0000", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017080200000000001 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006494+0000", 2023-09-23T15:21:14.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8374000000000005e-05 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006540+0000", 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5130000000000003e-05 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006581+0000", 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0924999999999999e-05 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.063 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179355 7.6 e276/223 7:615b3fe2:::benchmark_data_smithi017_422824_object85485:head v 276'11306, mlcod=276'11301)", 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267582228000002, 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31722602799999999, 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:21:14.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689087+0000", 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5159999999999997e-06 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689096+0000", 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689103+0000", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6159999999999998e-06 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692603+0000", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035004910000000001 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692623+0000", 2023-09-23T15:21:14.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0027999999999999e-05 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006273+0000", 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31364994099999999 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006306+0000", 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3408000000000003e-05 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.068 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179351 7.c e276/223 7:387cb04d:::benchmark_data_smithi017_422824_object85481:head v 276'11330, mlcod=276'11328)", 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267302471999997, 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31712001899999998, 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:21:14.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689359+0000", 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689369+0000", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2050000000000008e-06 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689378+0000", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3030000000000005e-06 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689397+0000", 2023-09-23T15:21:14.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8868999999999998e-05 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689491+0000", 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3949999999999993e-05 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689511+0000", 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0625999999999999e-05 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694710+0000", 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051988340000000003 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006480+0000", 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31176990500000001 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179354 7.7 e276/223 7:e702c870:::benchmark_data_smithi017_422824_object85484:head v 276'11003, mlcod=276'11000)", 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.267249188000001, 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31692908400000003, 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:21:14.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689418+0000", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7620000000000003e-06 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689425+0000", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2359999999999998e-06 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689430+0000", 2023-09-23T15:21:14.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3419999999999998e-06 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692321+0000", 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028904429999999999 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692340+0000", 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9295000000000001e-05 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.696068+0000", 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00372778 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006342+0000", 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31027458200000002 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179361 7.3 7:c730fffd:::benchmark_data_smithi017_422824_object85491:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.695882+0000", 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 32.260780818000001, 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31512405100000002, 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179361 2023-09-23T15:21:14.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695882+0000", 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695882+0000", 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:21:14.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695880+0000", 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695887+0000", 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7100000000000007e-06 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:21:14.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006244+0000", 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31035708400000001 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006261+0000", 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6792000000000001e-05 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006310+0000", 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8869000000000003e-05 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006471+0000", 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160989 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006613+0000", 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014166999999999999 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.010006+0000", 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033932509999999999 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.010905+0000", 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00089874499999999995 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.010921+0000", 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5968e-05 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.010967+0000", 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5768e-05 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.011006+0000", 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9292999999999998e-05 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:21:14.085 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:21:14.086 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:21:14.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:14.117 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:14.165 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:14.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:14.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[215646]: pgmap v819: 109 pgs: 4 creating+peering, 12 unknown, 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[215646]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[215646]: osdmap e286: 8 total, 7 up, 6 in 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[215646]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[222878]: pgmap v819: 109 pgs: 4 creating+peering, 12 unknown, 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[222878]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.221 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[222878]: osdmap e286: 8 total, 7 up, 6 in 2023-09-23T15:21:14.222 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:14 smithi017 ceph-mon[222878]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:14.298 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:14.342 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:14.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:14.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:14.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:14.482 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:14 smithi027 ceph-mon[195233]: pgmap v819: 109 pgs: 4 creating+peering, 12 unknown, 9 active+undersized+degraded, 39 active+undersized, 45 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:14 smithi027 ceph-mon[195233]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:14 smithi027 ceph-mon[195233]: osdmap e286: 8 total, 7 up, 6 in 2023-09-23T15:21:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:14 smithi027 ceph-mon[195233]: failed to encode map e286 with expected crc 2023-09-23T15:21:14.880 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:14.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:14.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:14.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:15.019 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:15.408 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:21:15.433 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:15 smithi017 ceph-mon[215646]: failed to encode map e286 with expected crc 2023-09-23T15:21:15.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:15 smithi017 ceph-mon[222878]: failed to encode map e286 with expected crc 2023-09-23T15:21:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:15 smithi027 ceph-mon[195233]: failed to encode map e286 with expected crc 2023-09-23T15:21:15.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:21:15.796 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:15.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:15.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:15.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:15.927 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:15.974 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:15.978 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:16.012 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:16.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:16.105 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:16.162 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:16.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:16.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:16.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:16.294 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:16.339 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:16.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:16.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:16.442 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:16 smithi017 ceph-mon[215646]: pgmap v821: 109 pgs: 4 creating+peering, 7 unknown, 9 active+undersized+degraded, 41 active+undersized, 48 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:16 smithi017 ceph-mon[222878]: pgmap v821: 109 pgs: 4 creating+peering, 7 unknown, 9 active+undersized+degraded, 41 active+undersized, 48 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:16.455 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:16 smithi027 ceph-mon[195233]: pgmap v821: 109 pgs: 4 creating+peering, 7 unknown, 9 active+undersized+degraded, 41 active+undersized, 48 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 14 KiB/s rd, 37 KiB/s wr, 10 op/s; 17/84 objects degraded (20.238%); 25 KiB/s, 0 objects/s recovering 2023-09-23T15:21:16.472 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:16.874 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:16.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:16.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:17.005 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:17.224 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:17.311 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:17.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:17.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:17.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:17.450 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:17.497 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:17.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:17.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:17.570 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:21:17.570 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:21:17.570 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:21:17.570 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:21:17.602 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:17.635 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:17.691 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:17.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:17.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:17.795 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:17.826 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:17.882 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:17.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:17 smithi017 conmon[215622]: 2023-09-23T15:21:17.705+0000 7f618a1a9700 -1 mon.a@0(leader).osd e286 definitely_dead 0 2023-09-23T15:21:17.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:17.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:17.987 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:18.016 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:18.111 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:18.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:18.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:18.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:21:18.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:18] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:21:18.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:18.260 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:21:18.260 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.7. 2023-09-23T15:21:18.262 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T15:21:18.265 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: pgmap v822: 109 pgs: 4 creating+peering, 9 active+undersized+degraded, 44 active+undersized, 52 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4179744386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2633992139' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/956958790' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:18.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:18.519 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: pgmap v822: 109 pgs: 4 creating+peering, 9 active+undersized+degraded, 44 active+undersized, 52 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4179744386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2633992139' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/956958790' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: pgmap v822: 109 pgs: 4 creating+peering, 9 active+undersized+degraded, 44 active+undersized, 52 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:18.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4179744386' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:18.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2633992139' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T15:21:18.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/956958790' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:18.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:21:19.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:19 smithi027 ceph-mon[195233]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T15:21:19.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[215646]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: osdmap e287: 8 total, 7 up, 7 in 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: failed to encode map e287 with expected crc 2023-09-23T15:21:19.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:19 smithi017 ceph-mon[222878]: osdmap e288: 8 total, 7 up, 7 in 2023-09-23T15:21:19.662 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:19.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:19.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:19.799 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:19.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:19.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:19.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:19.968 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:21:19.968 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:21:19.968 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:21:19.968 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:21:19.968 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:21:19.976 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:20.021 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:20.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:20.025 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:20.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:20.153 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:20.287 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T15:21:20.322 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:20.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:20.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:20.329 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: pgmap v824: 109 pgs: 9 active+undersized+degraded, 46 active+undersized, 54 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:20 smithi027 ceph-mon[195233]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T15:21:20.429 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:20.457 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:20.507 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:20.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:20.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: pgmap v824: 109 pgs: 9 active+undersized+degraded, 46 active+undersized, 54 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[215646]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: pgmap v824: 109 pgs: 9 active+undersized+degraded, 46 active+undersized, 54 active+clean; 4.6 MiB data, 2.1 GiB used, 534 GiB / 536 GiB avail; 17/84 objects degraded (20.238%) 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: failed to encode map e288 with expected crc 2023-09-23T15:21:20.519 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:20 smithi017 ceph-mon[222878]: osdmap e289: 8 total, 7 up, 7 in 2023-09-23T15:21:20.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:20.641 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:20.689 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:20.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:20.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:20.708 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:20.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:20.746 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:20.765 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:20.784 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:20.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:20.803 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:20.820 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:20.823 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:20.824 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.824 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:21:20.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:21:20.828 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:21:20.865 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:20.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:20.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:20.970 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:20.997 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:21.044 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:21.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:21.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:21.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:21.174 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:21.186 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:21.219 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:21.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:21.308 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T15:21:21.310 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:21:21.310 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:21:21.323 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:21.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:21.360 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:21:21.361 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:21:21.361 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_485823 2023-09-23T15:21:21.361 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:21:21.361 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/294418000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:21.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[215646]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/294418000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:21.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:21 smithi017 ceph-mon[222878]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: failed to encode map e289 with expected crc 2023-09-23T15:21:21.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/294418000' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:21.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:21 smithi027 ceph-mon[195233]: osdmap e290: 8 total, 7 up, 7 in 2023-09-23T15:21:21.642 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:21.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:21.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:21:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:21:21.717 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:21:21.747 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:21.783 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:21.944 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:21.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:21.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:22.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:22.075 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:22.119 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:22.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:22.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:22.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:22.236 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:22.249 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:22.313 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:22.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:22.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:22.362 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 2600 2584 161.356 161.5 0.00238749 0.00617684 2023-09-23T15:21:22.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:22.453 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: pgmap v827: 109 pgs: 1 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 5 active+undersized+degraded, 34 active+undersized, 46 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 8/63 objects degraded (12.698%); 682 KiB/s, 0 objects/s recovering 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 8/63 objects degraded (12.698%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[215646]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T15:21:22.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: pgmap v827: 109 pgs: 1 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 5 active+undersized+degraded, 34 active+undersized, 46 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 8/63 objects degraded (12.698%); 682 KiB/s, 0 objects/s recovering 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 8/63 objects degraded (12.698%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:22.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:22 smithi017 ceph-mon[222878]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T15:21:22.588 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: pgmap v827: 109 pgs: 1 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 5 active+undersized+degraded, 34 active+undersized, 46 active+clean; 4.6 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 8/63 objects degraded (12.698%); 682 KiB/s, 0 objects/s recovering 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: failed to encode map e290 with expected crc 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 8/63 objects degraded (12.698%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:22.591 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:22 smithi027 ceph-mon[195233]: osdmap e291: 8 total, 7 up, 7 in 2023-09-23T15:21:22.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:22.592 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:22.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:22.737 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:22.836 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:22.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:22.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:22.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:22.996 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:23.106 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:23.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:23.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:23.255 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:23.323 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:23.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:23.341 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:23.341 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:23.363 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 5683 5667 176.899 192.688 0.00524215 0.0056404 2023-09-23T15:21:23.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e290 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: 1.7 scrub starts 2023-09-23T15:21:23.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[215646]: 1.7 scrub ok 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e290 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: 1.7 scrub starts 2023-09-23T15:21:23.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:23 smithi017 ceph-mon[222878]: 1.7 scrub ok 2023-09-23T15:21:23.481 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e290 with expected crc 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.513 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.514 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.514 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: failed to encode map e291 with expected crc 2023-09-23T15:21:23.514 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: 1.7 scrub starts 2023-09-23T15:21:23.514 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:23 smithi027 ceph-mon[195233]: 1.7 scrub ok 2023-09-23T15:21:23.601 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:23.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:23.709 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:23.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:21:23.746 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:23.802 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:23.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:23.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:23.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:23.938 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:23.998 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:24.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:24.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:24.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:24.166 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:24.364 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 8466 8450 175.872 173.938 0.00534796 0.00567351 2023-09-23T15:21:24.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[215646]: pgmap v830: 109 pgs: 2 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 31 active+undersized, 46 active+clean; 28 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 5.8 MiB/s wr, 86 op/s; 184/1176 objects degraded (15.646%); 1024 KiB/s, 2 objects/s recovering 2023-09-23T15:21:24.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[215646]: 1.e scrub starts 2023-09-23T15:21:24.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[215646]: 1.e scrub ok 2023-09-23T15:21:24.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[222878]: pgmap v830: 109 pgs: 2 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 31 active+undersized, 46 active+clean; 28 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 5.8 MiB/s wr, 86 op/s; 184/1176 objects degraded (15.646%); 1024 KiB/s, 2 objects/s recovering 2023-09-23T15:21:24.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[222878]: 1.e scrub starts 2023-09-23T15:21:24.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:24 smithi017 ceph-mon[222878]: 1.e scrub ok 2023-09-23T15:21:24.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:21:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:24 smithi027 ceph-mon[195233]: pgmap v830: 109 pgs: 2 active+clean+remapped, 9 remapped+peering, 12 peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 31 active+undersized, 46 active+clean; 28 MiB data, 2.2 GiB used, 624 GiB / 626 GiB avail; 5.8 MiB/s wr, 86 op/s; 184/1176 objects degraded (15.646%); 1024 KiB/s, 2 objects/s recovering 2023-09-23T15:21:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:24 smithi027 ceph-mon[195233]: 1.e scrub starts 2023-09-23T15:21:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:24 smithi027 ceph-mon[195233]: 1.e scrub ok 2023-09-23T15:21:24.958 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:25.079 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:25.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:25.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:25.185 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:25.223 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:25.280 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:25.283 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:25.288 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [6] live_osds: [0, 2, 3, 4, 5, 7, 1] 2023-09-23T15:21:25.288 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:21:25.288 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T15:21:25.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:21:25.317 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:25.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 11109 11093 173.16 165.188 0.00153317 0.00576165 2023-09-23T15:21:25.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:25.424 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:25.533 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:25.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:25.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:25.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:25.709 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:25.819 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:25.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:25.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:25.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:25.959 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:26.015 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:26.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:26.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:26.119 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:26.189 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:26.301 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:26 smithi027 ceph-mon[195233]: pgmap v831: 109 pgs: 11 active+clean+remapped, 3 remapped+peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 27 active+undersized, 59 active+clean; 172 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 31 MiB/s wr, 496 op/s; 184/8124 objects degraded (2.265%); 951/8124 objects misplaced (11.706%); 759 KiB/s, 7 objects/s recovering 2023-09-23T15:21:26.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 13063 13047 162.958 122.125 0.00614093 0.00613188 2023-09-23T15:21:26.402 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:26.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:26.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:26.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:26 smithi017 ceph-mon[222878]: pgmap v831: 109 pgs: 11 active+clean+remapped, 3 remapped+peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 27 active+undersized, 59 active+clean; 172 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 31 MiB/s wr, 496 op/s; 184/8124 objects degraded (2.265%); 951/8124 objects misplaced (11.706%); 759 KiB/s, 7 objects/s recovering 2023-09-23T15:21:26.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:26 smithi017 ceph-mon[215646]: pgmap v831: 109 pgs: 11 active+clean+remapped, 3 remapped+peering, 2 active+undersized+remapped, 7 active+undersized+degraded, 27 active+undersized, 59 active+clean; 172 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 31 MiB/s wr, 496 op/s; 184/8124 objects degraded (2.265%); 951/8124 objects misplaced (11.706%); 759 KiB/s, 7 objects/s recovering 2023-09-23T15:21:26.512 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:26.549 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:26.625 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:26.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:26.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:26.734 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:26.778 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:26.892 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:26.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:26.997 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:27.036 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:27.108 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:27.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:27.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:27.146 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:27.146 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:27.213 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:27.255 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:27.351 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:21:27.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 15596 15580 162.182 158.312 0.00261136 0.00615913 2023-09-23T15:21:27.376 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:27.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:27.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:27.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:27.527 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:27.571 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:27.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4057560432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.573 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 184/8124 objects degraded (2.265%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:27.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4057560432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.573 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 184/8124 objects degraded (2.265%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:27.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4057560432' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 184/8124 objects degraded (2.265%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:27.676 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:27.703 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:27.747 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:27.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:27.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:27.877 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:28.088 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:28.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:28.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:28.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:21:28.192 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:28.272 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:28.312 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:28] "GET /metrics HTTP/1.1" 200 36306 "" "Prometheus/2.43.0" 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: pgmap v832: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 456 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 1433/21759 objects degraded (6.586%); 951/21759 objects misplaced (4.371%); 0 B/s, 1 keys/s, 6 objects/s recovering 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T15:21:28.314 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:21:28.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:28.315 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:28.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: pgmap v832: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 456 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 1433/21759 objects degraded (6.586%); 951/21759 objects misplaced (4.371%); 0 B/s, 1 keys/s, 6 objects/s recovering 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:28.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:28.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 16948 16932 151.09 84.5 0.00733034 0.00660959 2023-09-23T15:21:28.646 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: pgmap v832: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 456 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s; 1433/21759 objects degraded (6.586%); 951/21759 objects misplaced (4.371%); 0 B/s, 1 keys/s, 6 objects/s recovering 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: osdmap e292: 8 total, 7 up, 7 in 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: failed to encode map e292 with expected crc 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:28.913 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:29.148 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:29.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:29.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:29.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:29.294 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:29.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 18327 18311 142.98 86.1875 0.00301322 0.00698538 2023-09-23T15:21:29.841 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:29.884 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:29.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:29.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:30.010 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:30.011 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:30.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:30.047 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:30.083 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:21:30.084 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:21:30.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:30.179 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:30.319 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:30 smithi027 ceph-mon[195233]: pgmap v834: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 653 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 3013/31224 objects degraded (9.650%); 951/31224 objects misplaced (3.046%); 0 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T15:21:30.319 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1668879899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:30.365 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 19988 19972 138.628 103.812 0.0045325 0.00720483 2023-09-23T15:21:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:30 smithi017 ceph-mon[215646]: pgmap v834: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 653 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 3013/31224 objects degraded (9.650%); 951/31224 objects misplaced (3.046%); 0 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T15:21:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1668879899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:30 smithi017 ceph-mon[222878]: pgmap v834: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 653 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 88 MiB/s wr, 1.40k op/s; 3013/31224 objects degraded (9.650%); 951/31224 objects misplaced (3.046%); 0 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T15:21:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1668879899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:30.814 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:21:30.815 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:21:30.815 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:21:30.815 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:21:30.815 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:21:31.263 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:21:31.371 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 20999 20983 131.032 63.1875 0.0105437 0.00762217 2023-09-23T15:21:31.632 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:21:31.651 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:31.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:31.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:31.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:31.807 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:31.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:31.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:32.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:32.142 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:32.151 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:21:32.151 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:21:32.166 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:32.336 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:32 smithi017 ceph-mon[215646]: pgmap v835: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s; 4188/52734 objects degraded (7.942%); 2167/52734 objects misplaced (4.109%); 0 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T15:21:32.336 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 4188/52734 objects degraded (7.942%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:32.364 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:32.370 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 22314 22298 126.594 82.1875 0.00630609 0.00789078 2023-09-23T15:21:32.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:32.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:32.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:32.562 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:32.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:32 smithi027 ceph-mon[195233]: pgmap v835: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s; 4188/52734 objects degraded (7.942%); 2167/52734 objects misplaced (4.109%); 0 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T15:21:32.585 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 4188/52734 objects degraded (7.942%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:32.608 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:32.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:32 smithi017 ceph-mon[222878]: pgmap v835: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.1 GiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 128 MiB/s wr, 2.05k op/s; 4188/52734 objects degraded (7.942%); 2167/52734 objects misplaced (4.109%); 0 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T15:21:32.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 4188/52734 objects degraded (7.942%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:32.723 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:32.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:32.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:32.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:32.908 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:33.039 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:33.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:33.126 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:21:33.139 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:21:33.145 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:33.208 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:33.335 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:33.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:33.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:33.370 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 23738 23722 123.462 89 0.00415865 0.00809404 2023-09-23T15:21:33.451 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:33.512 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:34.152 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:21:34.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:34 smithi027 ceph-mon[195233]: pgmap v836: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.2 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.90k op/s; 5089/58083 objects degraded (8.762%); 2167/58083 objects misplaced (3.731%); 0 B/s, 0 keys/s, 3 objects/s recovering 2023-09-23T15:21:34.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:34 smithi027 ceph-mon[195233]: from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78612 6.d 6:bb45bf81:::benchmark_data_smithi017_350392_object78611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e182)", 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 532.79006326599995, 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6694258959999999, 2023-09-23T15:21:34.360 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78612 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567777+0000", 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567775+0000", 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567840+0000", 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4871999999999998e-05 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567851+0000", 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0589e-05 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.567857+0000", 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3589999999999996e-06 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568037+0000", 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017947699999999999 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568207+0000", 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000170431 2023-09-23T15:21:34.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.568367+0000", 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015976799999999999 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574569+0000", 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006201771 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.574582+0000", 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3179999999999999e-05 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.844996+0000", 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27041409399999999 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845010+0000", 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4292e-05 2023-09-23T15:21:34.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.201824+0000", 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.356813662 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203608+0000", 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017844499999999999 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237060+0000", 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.033452123 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237202+0000", 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014193699999999999 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.368 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78627 6.d 6:b5bf0d91:::benchmark_data_smithi017_350392_object78626:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 532.754984089, 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.637516355, 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78627 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602856+0000", 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602854+0000", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602946+0000", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1372999999999994e-05 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602960+0000", 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4171999999999999e-05 2023-09-23T15:21:34.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.602965+0000", 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3709999999999999e-06 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603026+0000", 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1115e-05 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.603042+0000", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5851000000000001e-05 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845043+0000", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24200096700000001 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845051+0000", 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9529999999999999e-06 2023-09-23T15:21:34.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203676+0000", 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.358624947 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.203805+0000", 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128534 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.204009+0000", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00020406299999999999 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.213055+0000", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090466460000000002 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.218171+0000", 2023-09-23T15:21:34.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051154820000000002 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237245+0000", 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019074157000000001 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.237307+0000", 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1816000000000006e-05 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240372+0000", 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030652040000000002 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45074.0:78647 6.d 6:b57ccd88:::benchmark_data_smithi017_350392_object78646:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e183)", 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 532.72747657299999, 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.61263826, 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.377 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45074", 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/2405110218", 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 78647 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630363+0000", 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630362+0000", 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630419+0000", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.698e-05 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630433+0000", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3936e-05 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630440+0000", 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5939999999999999e-06 2023-09-23T15:21:34.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630499+0000", 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9676000000000003e-05 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.630508+0000", 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7760000000000003e-06 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845613+0000", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21510510299999999 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:41.845622+0000", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8729999999999996e-06 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206337+0000", 2023-09-23T15:21:34.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3607147209999999 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206504+0000", 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016709 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.206715+0000", 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021077199999999999 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.217101+0000", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010385926 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.218207+0000", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011067220000000001 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240451+0000", 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022243489000000002 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.240524+0000", 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3198e-05 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:12:43.243001+0000", 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024774049999999998 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:165864 7.0 7:0e36f679:::benchmark_data_smithi017_422824_object71994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:21.469869+0000", 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 72.887970971000001, 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.354504824, 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 165864 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469869+0000", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469869+0000", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469867+0000", 2023-09-23T15:21:34.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469873+0000", 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1760000000000001e-06 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469884+0000", 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0866000000000001e-05 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469888+0000", 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4379999999999997e-06 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.469922+0000", 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3751000000000003e-05 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.470058+0000", 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013613300000000001 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.470211+0000", 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000153194 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.472935+0000", 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027240459999999999 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.473446+0000", 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00051095299999999997 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.824283+0000", 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.35083668000000001 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.824346+0000", 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3381999999999997e-05 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:21.824374+0000", 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7503000000000001e-05 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:178587 7.0 7:0b7f431e:::benchmark_data_smithi017_422824_object84717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:40.439828+0000", 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 53.918012222999998, 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20242469299999999, 2023-09-23T15:21:34.392 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178587 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439828+0000", 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439828+0000", 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439826+0000", 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439832+0000", 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0310000000000004e-06 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439850+0000", 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7829000000000002e-05 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439856+0000", 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4189999999999999e-06 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439900+0000", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3646000000000002e-05 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.440001+0000", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101254 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.440072+0000", 2023-09-23T15:21:34.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0228000000000006e-05 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.445255+0000", 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005182987 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.455601+0000", 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010346062 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.642150+0000", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.186549457 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.642206+0000", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5506e-05 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.642252+0000", 2023-09-23T15:21:34.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6693000000000002e-05 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:178589 7.0 7:01905014:::benchmark_data_smithi017_422824_object84719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:40.439872+0000", 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 53.917968150999997, 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.216050454, 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.399 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178589 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439872+0000", 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439872+0000", 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439871+0000", 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439874+0000", 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1300000000000001e-06 2023-09-23T15:21:34.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439881+0000", 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.742e-06 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.439885+0000", 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4619999999999998e-06 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.442579+0000", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002693385 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.442699+0000", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119938 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.442792+0000", 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2937999999999997e-05 2023-09-23T15:21:34.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.445403+0000", 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026112589999999999 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.455639+0000", 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010236250000000001 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655855+0000", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20021570499999999 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655892+0000", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6958000000000001e-05 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655922+0000", 2023-09-23T15:21:34.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0213e-05 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:178593 7.0 7:03691611:::benchmark_data_smithi017_422824_object84723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:40.446492+0000", 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 53.911347679999999, 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.209497131, 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.406 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178593 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446492+0000", 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446492+0000", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446490+0000", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446496+0000", 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8860000000000001e-06 2023-09-23T15:21:34.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446509+0000", 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2339e-05 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446514+0000", 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9230000000000001e-06 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446559+0000", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5630000000000002e-05 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446705+0000", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014530500000000001 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.446839+0000", 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013433 2023-09-23T15:21:34.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.449424+0000", 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025847999999999999 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.455720+0000", 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062961570000000001 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655945+0000", 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.200224872 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655971+0000", 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6047e-05 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:40.655989+0000", 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8479999999999999e-05 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.412 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:179360 7.0 7:05ed19d2:::benchmark_data_smithi017_422824_object85490:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:41.688902+0000", 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.668938163999997, 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.317854629, 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.413 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 179360 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688902+0000", 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688902+0000", 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688901+0000", 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688905+0000", 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6099999999999999e-06 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688915+0000", 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7019999999999996e-06 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688919+0000", 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3250000000000001e-06 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.688951+0000", 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1380000000000001e-05 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.689029+0000", 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8368000000000006e-05 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.689099+0000", 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9741999999999997e-05 2023-09-23T15:21:34.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.691585+0000", 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00248635 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.692627+0000", 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010412990000000001 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.006646+0000", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31401961699999997 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.006704+0000", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.783e-05 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.006756+0000", 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2207999999999999e-05 2023-09-23T15:21:34.419 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:179362 7.0 7:0ef1e638:::benchmark_data_smithi017_422824_object85492:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:20:41.695919+0000", 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.661920440000003, 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.324380697, 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:21:34.420 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 179362 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695919+0000", 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695919+0000", 2023-09-23T15:21:34.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695918+0000", 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695928+0000", 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0539e-05 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695940+0000", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1379000000000001e-05 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695947+0000", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6799999999999993e-06 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.695982+0000", 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5067000000000002e-05 2023-09-23T15:21:34.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.696068+0000", 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5450000000000003e-05 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.696164+0000", 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6145999999999997e-05 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.699009+0000", 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028445639999999999 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:41.699102+0000", 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3222999999999998e-05 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.020218+0000", 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32111578499999999 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.020266+0000", 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8096000000000002e-05 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:20:42.020300+0000", 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4418999999999997e-05 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.426 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:20690 8.9 8:9b59c9ea:::benchmark_data_smithi017_485823_object20689:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:30.897861+0000", 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.4599785989999998, 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14895550399999999, 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 20690 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897861+0000", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897861+0000", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897860+0000", 2023-09-23T15:21:34.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897957+0000", 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6935000000000006e-05 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897971+0000", 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3704e-05 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.897976+0000", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9470000000000002e-06 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.898019+0000", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3102000000000002e-05 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.898150+0000", 2023-09-23T15:21:34.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131118 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.898288+0000", 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137513 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.899072+0000", 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00078426099999999996 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046656+0000", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.147583726 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046725+0000", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9404000000000002e-05 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046817+0000", 2023-09-23T15:21:34.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1693000000000005e-05 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:20700 8.9 8:90af466c:::benchmark_data_smithi017_485823_object20699:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:30.901025+0000", 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.4568145609999998, 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14593888399999999, 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.433 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 20700 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901025+0000", 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901025+0000", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901024+0000", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901098+0000", 2023-09-23T15:21:34.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3856000000000006e-05 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901114+0000", 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5435999999999999e-05 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901123+0000", 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0140000000000004e-06 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901175+0000", 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2750999999999997e-05 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901279+0000", 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000103997 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.901371+0000", 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1062000000000003e-05 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:30.902207+0000", 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00083688199999999995 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046862+0000", 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14465508899999999 2023-09-23T15:21:34.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046908+0000", 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5389999999999997e-05 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.046964+0000", 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6215999999999998e-05 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:20875 8.8 8:11099a76:::benchmark_data_smithi017_485823_object20874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.146637+0000", 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.211202932, 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.165929195, 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 20875 2023-09-23T15:21:34.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146637+0000", 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146637+0000", 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146636+0000", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146690+0000", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3956000000000003e-05 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146697+0000", 2023-09-23T15:21:34.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1160000000000001e-06 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146702+0000", 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3329999999999998e-06 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146749+0000", 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7580000000000002e-05 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.146876+0000", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0001262 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.147014+0000", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138028 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148771+0000", 2023-09-23T15:21:34.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001757626 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312408+0000", 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16363707199999999 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312489+0000", 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0284999999999994e-05 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312531+0000", 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2917000000000003e-05 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312566+0000", 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4591000000000001e-05 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:20883 8.4 8:257c8810:::benchmark_data_smithi017_485823_object20882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.148355+0000", 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2094846289999999, 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16403601500000001, 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 20883 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.447 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148355+0000", 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148355+0000", 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148354+0000", 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148437+0000", 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2865e-05 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148446+0000", 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5320000000000006e-06 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.148450+0000", 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3640000000000001e-06 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.149039+0000", 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058856800000000003 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.149118+0000", 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9651e-05 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2", 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.149219+0000", 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100358 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.149923+0000", 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070448799999999997 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312280+0000", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16235655399999999 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312327+0000", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6941999999999999e-05 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312391+0000", 2023-09-23T15:21:34.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4442000000000004e-05 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:20885 8.8 8:1fc9c1fd:::benchmark_data_smithi017_485823_object20884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.150349+0000", 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2074905170000001, 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16342159000000001, 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.453 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 20885 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150349+0000", 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150349+0000", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150348+0000", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150407+0000", 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8805999999999999e-05 2023-09-23T15:21:34.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150416+0000", 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1060000000000006e-06 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150421+0000", 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5140000000000003e-06 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150458+0000", 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6826000000000003e-05 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150556+0000", 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8504000000000004e-05 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150645+0000", 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9038999999999999e-05 2023-09-23T15:21:34.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312445+0000", 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.161799744 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312621+0000", 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017645399999999999 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.313686+0000", 2023-09-23T15:21:34.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001064277 2023-09-23T15:21:34.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.313737+0000", 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1434000000000003e-05 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.313771+0000", 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3728e-05 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.461 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.45809.0:20883 8.4 e292/284 ondisk, result = 0)", 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.150972+0000", 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2068678959999999, 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.161423335, 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150972+0000", 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150972+0000", 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150971+0000", 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.150975+0000", 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0860000000000004e-06 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311278+0000", 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16030233299999999 2023-09-23T15:21:34.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311288+0000", 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0648999999999999e-05 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312265+0000", 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097696000000000002 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312274+0000", 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3000000000000002e-06 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312395+0000", 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012154699999999999 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.45809.0:20875 8.8 e292/287 ondisk, result = 0)", 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.152655+0000", 2023-09-23T15:21:34.466 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2051850919999998, 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15975906400000001, 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.152655+0000", 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.152655+0000", 2023-09-23T15:21:34.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.152654+0000", 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.152659+0000", 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5639999999999998e-06 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311296+0000", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15863766200000001 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311305+0000", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6079999999999996e-06 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312394+0000", 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010890170000000001 2023-09-23T15:21:34.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312402+0000", 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1370000000000002e-06 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312414+0000", 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1687e-05 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45809.0:20890 8.a e292/284 8:5a5c7d5a:::benchmark_data_smithi017_485823_object20889:head v 292'1281, mlcod=292'1279)", 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.154618+0000", 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 3.2032213500000002, 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15785358499999999, 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.154618+0000", 2023-09-23T15:21:34.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.154618+0000", 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.154618+0000", 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.154673+0000", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4917999999999998e-05 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311273+0000", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.156600296 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311285+0000", 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2255999999999999e-05 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311369+0000", 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4133000000000002e-05 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.311393+0000", 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3549000000000001e-05 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312448+0000", 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001054765 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.312472+0000", 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4286000000000001e-05 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:21118 8.9 8:911249d7:::benchmark_data_smithi017_485823_object21117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.455058+0000", 2023-09-23T15:21:34.475 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.9027821110000001, 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13406074500000001, 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 21118 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455058+0000", 2023-09-23T15:21:34.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455058+0000", 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455057+0000", 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455115+0000", 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7482000000000003e-05 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455126+0000", 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1224e-05 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455130+0000", 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3499999999999999e-06 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455185+0000", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4506000000000003e-05 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455281+0000", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6687000000000004e-05 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455375+0000", 2023-09-23T15:21:34.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3257999999999994e-05 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.456924+0000", 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001549669 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.588981+0000", 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.132056494 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.589032+0000", 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1746000000000002e-05 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.589118+0000", 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5927000000000006e-05 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:21119 8.9 8:9225ebcc:::benchmark_data_smithi017_485823_object21118:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:31.455147+0000", 2023-09-23T15:21:34.481 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 2.9026930279999998, 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.134134213, 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 21119 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455147+0000", 2023-09-23T15:21:34.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455147+0000", 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455146+0000", 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455178+0000", 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1390000000000003e-05 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455185+0000", 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9090000000000003e-06 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.455189+0000", 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2039999999999999e-06 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.456240+0000", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010507380000000001 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.456345+0000", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105115 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.456440+0000", 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5421999999999995e-05 2023-09-23T15:21:34.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.457047+0000", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00060638300000000005 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.589174+0000", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.132127311 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.589218+0000", 2023-09-23T15:21:34.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3847999999999998e-05 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:31.589281+0000", 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3281999999999995e-05 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:25056 8.9 8:95db4411:::benchmark_data_smithi017_485823_object25055:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:34.119055+0000", 2023-09-23T15:21:34.487 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.23878453699999999, 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13598876300000001, 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 25056 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119055+0000", 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119055+0000", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119055+0000", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119112+0000", 2023-09-23T15:21:34.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7256999999999997e-05 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119118+0000", 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9529999999999996e-06 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119121+0000", 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.286e-06 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119178+0000", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7089000000000002e-05 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119303+0000", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012456699999999999 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.119456+0000", 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015267899999999999 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.120514+0000", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010579929999999999 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.254891+0000", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.13437758499999999 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.254947+0000", 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5741e-05 2023-09-23T15:21:34.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:34.255044+0000", 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7084999999999996e-05 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:34.493 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:21:34.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:34.494 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:34.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:34.495 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 25153 25137 120.769 88.4375 0.00211332 0.00823343 2023-09-23T15:21:34.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:34 smithi017 ceph-mon[215646]: pgmap v836: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.2 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.90k op/s; 5089/58083 objects degraded (8.762%); 2167/58083 objects misplaced (3.731%); 0 B/s, 0 keys/s, 3 objects/s recovering 2023-09-23T15:21:34.495 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:34 smithi017 ceph-mon[215646]: from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:34.495 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:34 smithi017 ceph-mon[222878]: pgmap v836: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.2 GiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.90k op/s; 5089/58083 objects degraded (8.762%); 2167/58083 objects misplaced (3.731%); 0 B/s, 0 keys/s, 3 objects/s recovering 2023-09-23T15:21:34.495 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:34 smithi017 ceph-mon[222878]: from='client.55780 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:21:34.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:34.622 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:35.206 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":836,"stamp":"2023-09-23T15:21:32.633658+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1271851353,"num_objects":19361,"num_object_clones":0,"num_object_copies":58083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5089,"num_objects_misplaced":2167,"num_objects_unfound":0,"num_objects_dirty":19361,"num_whiteouts":0,"num_read":6039,"num_read_kb":13295,"num_write":20262,"num_write_kb":1250257,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":116,"num_bytes_recovered":9080035,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":23746,"ondisk_log_size":23746,"up":279,"acting":293,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":349,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5639900,"kb_used_data":3266468,"kb_used_omap":30,"kb_used_meta":2373345,"kb_avail":650576164,"statfs":{"total":671965249536,"available":666189991936,"internally_reserved":0,"allocated":3344863232,"data_stored":3337279545,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":31144,"internal_metadata":2430305880},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1242671635,"num_objects":18969,"num_object_clones":0,"num_object_copies":56907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4905,"num_objects_misplaced":2167,"num_objects_unfound":0,"num_objects_dirty":18969,"num_whiteouts":0,"num_read":-22,"num_read_kb":-41,"num_write":18977,"num_write_kb":1213484,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":-75596,"num_keys_recovered":10,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003314"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":49,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241002+0000","last_change":"2023-09-23T15:19:41.337833+0000","last_active":"2023-09-23T15:21:22.241002+0000","last_peered":"2023-09-23T15:21:22.241002+0000","last_clean":"2023-09-23T15:21:22.241002+0000","last_became_active":"2023-09-23T15:19:41.337459+0000","last_became_peered":"2023-09-23T15:19:41.337459+0000","last_unstale":"2023-09-23T15:21:22.241002+0000","last_undegraded":"2023-09-23T15:21:22.241002+0000","last_fullsized":"2023-09-23T15:21:22.241002+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:15:23.895299+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":238,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.235933+0000","last_change":"2023-09-23T15:21:06.192550+0000","last_active":"2023-09-23T15:21:22.235933+0000","last_peered":"2023-09-23T15:21:22.235933+0000","last_clean":"2023-09-23T15:21:04.551147+0000","last_became_active":"2023-09-23T15:21:06.192550+0000","last_became_peered":"2023-09-23T15:21:06.192550+0000","last_unstale":"2023-09-23T15:21:22.235933+0000","last_undegraded":"2023-09-23T15:21:22.235933+0000","last_fullsized":"2023-09-23T15:21:06.186555+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:48.947363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":186,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.241516+0000","last_change":"2023-09-23T15:21:06.191776+0000","last_active":"2023-09-23T15:21:22.241516+0000","last_peered":"2023-09-23T15:21:22.241516+0000","last_clean":"2023-09-23T15:21:04.172024+0000","last_became_active":"2023-09-23T15:21:06.191776+0000","last_became_peered":"2023-09-23T15:21:06.191776+0000","last_unstale":"2023-09-23T15:21:22.241516+0000","last_undegraded":"2023-09-23T15:21:22.241516+0000","last_fullsized":"2023-09-23T15:21:06.185682+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:41.340851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":269,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256797+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:21:22.256797+0000","last_peered":"2023-09-23T15:21:22.256797+0000","last_clean":"2023-09-23T15:21:22.256797+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:21:22.256797+0000","last_undegraded":"2023-09-23T15:21:22.256797+0000","last_fullsized":"2023-09-23T15:21:22.256797+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":292,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.235691+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:21:22.235691+0000","last_peered":"2023-09-23T15:21:22.235691+0000","last_clean":"2023-09-23T15:21:22.235691+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:21:22.235691+0000","last_undegraded":"2023-09-23T15:21:22.235691+0000","last_fullsized":"2023-09-23T15:21:22.235691+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":169,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.261782+0000","last_change":"2023-09-23T15:21:06.194531+0000","last_active":"2023-09-23T15:21:22.261782+0000","last_peered":"2023-09-23T15:21:22.261782+0000","last_clean":"2023-09-23T15:08:45.872889+0000","last_became_active":"2023-09-23T15:21:06.194531+0000","last_became_peered":"2023-09-23T15:21:06.194531+0000","last_unstale":"2023-09-23T15:21:22.261782+0000","last_undegraded":"2023-09-23T15:21:22.261782+0000","last_fullsized":"2023-09-23T15:21:06.187253+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:49.159201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":238,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256430+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:21:22.256430+0000","last_peered":"2023-09-23T15:21:22.256430+0000","last_clean":"2023-09-23T15:21:22.256430+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:21:22.256430+0000","last_undegraded":"2023-09-23T15:21:22.256430+0000","last_fullsized":"2023-09-23T15:21:22.256430+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":280,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.236159+0000","last_change":"2023-09-23T15:21:06.197389+0000","last_active":"2023-09-23T15:21:22.236159+0000","last_peered":"2023-09-23T15:21:22.236159+0000","last_clean":"2023-09-23T15:21:04.551534+0000","last_became_active":"2023-09-23T15:21:06.197389+0000","last_became_peered":"2023-09-23T15:21:06.197389+0000","last_unstale":"2023-09-23T15:21:22.236159+0000","last_undegraded":"2023-09-23T15:21:22.236159+0000","last_fullsized":"2023-09-23T15:21:06.185729+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:01:09.745742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":632,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:27.322551+0000","last_change":"2023-09-23T15:21:22.257978+0000","last_active":"2023-09-23T15:21:27.322551+0000","last_peered":"2023-09-23T15:21:27.322551+0000","last_clean":"2023-09-23T15:21:27.322551+0000","last_became_active":"2023-09-23T15:21:22.257638+0000","last_became_peered":"2023-09-23T15:21:22.257638+0000","last_unstale":"2023-09-23T15:21:27.322551+0000","last_undegraded":"2023-09-23T15:21:27.322551+0000","last_fullsized":"2023-09-23T15:21:27.322551+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:46.097885+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":292,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.263377+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:21:22.263377+0000","last_peered":"2023-09-23T15:21:22.263377+0000","last_clean":"2023-09-23T15:21:22.263377+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:21:22.263377+0000","last_undegraded":"2023-09-23T15:21:22.263377+0000","last_fullsized":"2023-09-23T15:21:22.263377+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":233,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.236062+0000","last_change":"2023-09-23T15:21:20.244606+0000","last_active":"2023-09-23T15:21:22.236062+0000","last_peered":"2023-09-23T15:21:22.236062+0000","last_clean":"2023-09-23T15:21:22.236062+0000","last_became_active":"2023-09-23T15:21:20.244179+0000","last_became_peered":"2023-09-23T15:21:20.244179+0000","last_unstale":"2023-09-23T15:21:22.236062+0000","last_undegraded":"2023-09-23T15:21:22.236062+0000","last_fullsized":"2023-09-23T15:21:22.236062+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:31.016963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":20,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256874+0000","last_change":"2023-09-23T15:21:19.265504+0000","last_active":"2023-09-23T15:21:22.256874+0000","last_peered":"2023-09-23T15:21:22.256874+0000","last_clean":"2023-09-23T15:21:22.256874+0000","last_became_active":"2023-09-23T15:21:19.253399+0000","last_became_peered":"2023-09-23T15:21:19.253399+0000","last_unstale":"2023-09-23T15:21:22.256874+0000","last_undegraded":"2023-09-23T15:21:22.256874+0000","last_fullsized":"2023-09-23T15:21:22.256874+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:24.892734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":172,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.241617+0000","last_change":"2023-09-23T15:21:06.196830+0000","last_active":"2023-09-23T15:21:22.241617+0000","last_peered":"2023-09-23T15:21:22.241617+0000","last_clean":"2023-09-23T15:21:04.172896+0000","last_became_active":"2023-09-23T15:21:06.196830+0000","last_became_peered":"2023-09-23T15:21:06.196830+0000","last_unstale":"2023-09-23T15:21:22.241617+0000","last_undegraded":"2023-09-23T15:21:22.241617+0000","last_fullsized":"2023-09-23T15:21:06.187415+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:50.116575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":26,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.256179+0000","last_change":"2023-09-23T15:21:20.254146+0000","last_active":"2023-09-23T15:21:22.256179+0000","last_peered":"2023-09-23T15:21:22.256179+0000","last_clean":"2023-09-23T15:21:22.256179+0000","last_became_active":"2023-09-23T15:21:20.245432+0000","last_became_peered":"2023-09-23T15:21:20.245432+0000","last_unstale":"2023-09-23T15:21:22.256179+0000","last_undegraded":"2023-09-23T15:21:22.256179+0000","last_fullsized":"2023-09-23T15:21:22.256179+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:34:16.673702+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,0],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":233,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.256097+0000","last_change":"2023-09-23T15:21:06.197341+0000","last_active":"2023-09-23T15:21:22.256097+0000","last_peered":"2023-09-23T15:21:22.256097+0000","last_clean":"2023-09-23T15:17:22.128643+0000","last_became_active":"2023-09-23T15:21:06.197341+0000","last_became_peered":"2023-09-23T15:21:06.197341+0000","last_unstale":"2023-09-23T15:21:22.256097+0000","last_undegraded":"2023-09-23T15:21:22.256097+0000","last_fullsized":"2023-09-23T15:21:06.193698+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:51.522919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":600,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256451+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:21:22.256451+0000","last_peered":"2023-09-23T15:21:22.256451+0000","last_clean":"2023-09-23T15:21:22.256451+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:21:22.256451+0000","last_undegraded":"2023-09-23T15:21:22.256451+0000","last_fullsized":"2023-09-23T15:21:22.256451+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":22,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.254689+0000","last_change":"2023-09-23T15:21:20.256750+0000","last_active":"2023-09-23T15:21:22.254689+0000","last_peered":"2023-09-23T15:21:22.254689+0000","last_clean":"2023-09-23T15:21:22.254689+0000","last_became_active":"2023-09-23T15:21:20.247521+0000","last_became_peered":"2023-09-23T15:21:20.247521+0000","last_unstale":"2023-09-23T15:21:22.254689+0000","last_undegraded":"2023-09-23T15:21:22.254689+0000","last_fullsized":"2023-09-23T15:21:22.254689+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:13.659110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,0],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":294,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.237456+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:21:22.237456+0000","last_peered":"2023-09-23T15:21:22.237456+0000","last_clean":"2023-09-23T15:21:22.237456+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:21:22.237456+0000","last_undegraded":"2023-09-23T15:21:22.237456+0000","last_fullsized":"2023-09-23T15:21:22.237456+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":638,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.268531+0000","last_change":"2023-09-23T15:21:22.266958+0000","last_active":"2023-09-23T15:21:22.268531+0000","last_peered":"2023-09-23T15:21:22.268531+0000","last_clean":"2023-09-23T15:21:22.268531+0000","last_became_active":"2023-09-23T15:21:22.266512+0000","last_became_peered":"2023-09-23T15:21:22.266512+0000","last_unstale":"2023-09-23T15:21:22.268531+0000","last_undegraded":"2023-09-23T15:21:22.268531+0000","last_fullsized":"2023-09-23T15:21:22.268531+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:57:01.687335+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,0],"avail_no_missing":["5","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.257515+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:21:22.257515+0000","last_peered":"2023-09-23T15:21:22.257515+0000","last_clean":"2023-09-23T15:21:22.257515+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:21:22.257515+0000","last_undegraded":"2023-09-23T15:21:22.257515+0000","last_fullsized":"2023-09-23T15:21:22.257515+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":277,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.263779+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:21:22.263779+0000","last_peered":"2023-09-23T15:21:22.263779+0000","last_clean":"2023-09-23T15:21:22.263779+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:21:22.263779+0000","last_undegraded":"2023-09-23T15:21:22.263779+0000","last_fullsized":"2023-09-23T15:21:22.263779+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:17.702231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":564,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:26.779761+0000","last_change":"2023-09-23T15:19:44.401028+0000","last_active":"2023-09-23T15:21:26.779761+0000","last_peered":"2023-09-23T15:21:26.779761+0000","last_clean":"2023-09-23T15:21:26.779761+0000","last_became_active":"2023-09-23T15:19:44.400692+0000","last_became_peered":"2023-09-23T15:19:44.400692+0000","last_unstale":"2023-09-23T15:21:26.779761+0000","last_undegraded":"2023-09-23T15:21:26.779761+0000","last_fullsized":"2023-09-23T15:21:26.779761+0000","mapping_epoch":257,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:35.692134+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.255795+0000","last_change":"2023-09-23T15:21:20.250248+0000","last_active":"2023-09-23T15:21:22.255795+0000","last_peered":"2023-09-23T15:21:22.255795+0000","last_clean":"2023-09-23T15:21:22.255795+0000","last_became_active":"2023-09-23T15:21:20.249815+0000","last_became_peered":"2023-09-23T15:21:20.249815+0000","last_unstale":"2023-09-23T15:21:22.255795+0000","last_undegraded":"2023-09-23T15:21:22.255795+0000","last_fullsized":"2023-09-23T15:21:22.255795+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:30.373078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,3],"avail_no_missing":["7","0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":256,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256805+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:21:22.256805+0000","last_peered":"2023-09-23T15:21:22.256805+0000","last_clean":"2023-09-23T15:21:22.256805+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:21:22.256805+0000","last_undegraded":"2023-09-23T15:21:22.256805+0000","last_fullsized":"2023-09-23T15:21:22.256805+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":598,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.261658+0000","last_change":"2023-09-23T15:21:06.194234+0000","last_active":"2023-09-23T15:21:22.261658+0000","last_peered":"2023-09-23T15:21:22.261658+0000","last_clean":"2023-09-23T15:21:04.684377+0000","last_became_active":"2023-09-23T15:21:06.194234+0000","last_became_peered":"2023-09-23T15:21:06.194234+0000","last_unstale":"2023-09-23T15:21:22.261658+0000","last_undegraded":"2023-09-23T15:21:22.261658+0000","last_fullsized":"2023-09-23T15:21:06.185561+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:53.934870+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":29,"reported_epoch":291,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:22.256180+0000","last_change":"2023-09-23T15:21:06.199110+0000","last_active":"2023-09-23T15:21:22.256180+0000","last_peered":"2023-09-23T15:21:22.256180+0000","last_clean":"2023-09-23T14:56:32.495894+0000","last_became_active":"2023-09-23T15:21:06.199110+0000","last_became_peered":"2023-09-23T15:21:06.199110+0000","last_unstale":"2023-09-23T15:21:22.256180+0000","last_undegraded":"2023-09-23T15:21:06.189952+0000","last_fullsized":"2023-09-23T15:21:06.189847+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":96,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:38.500457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":294,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.236965+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:21:22.236965+0000","last_peered":"2023-09-23T15:21:22.236965+0000","last_clean":"2023-09-23T15:21:22.236965+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:21:22.236965+0000","last_undegraded":"2023-09-23T15:21:22.236965+0000","last_fullsized":"2023-09-23T15:21:22.236965+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"292'1233","reported_seq":1246,"reported_epoch":292,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:29.793583+0000","last_change":"2023-09-23T15:21:21.374908+0000","last_active":"2023-09-23T15:21:29.793583+0000","last_peered":"2023-09-23T15:21:29.793583+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:12.276104+0000","last_became_peered":"2023-09-23T15:21:12.276104+0000","last_unstale":"2023-09-23T15:21:29.793583+0000","last_undegraded":"2023-09-23T15:21:21.366833+0000","last_fullsized":"2023-09-23T15:21:12.214393+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1233,"log_dups_size":0,"ondisk_log_size":1233,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:34.020824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80805888,"num_objects":1233,"num_object_clones":0,"num_object_copies":3699,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1233,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1233,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1233,"num_write_kb":78912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1233}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":534,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.266948+0000","last_change":"2023-09-23T15:21:22.263408+0000","last_active":"2023-09-23T15:21:22.266948+0000","last_peered":"2023-09-23T15:21:22.266948+0000","last_clean":"2023-09-23T15:21:22.266948+0000","last_became_active":"2023-09-23T15:21:22.262463+0000","last_became_peered":"2023-09-23T15:21:22.262463+0000","last_unstale":"2023-09-23T15:21:22.266948+0000","last_undegraded":"2023-09-23T15:21:22.266948+0000","last_fullsized":"2023-09-23T15:21:22.266948+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:31.989770+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":18,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256231+0000","last_change":"2023-09-23T15:21:19.269945+0000","last_active":"2023-09-23T15:21:22.256231+0000","last_peered":"2023-09-23T15:21:22.256231+0000","last_clean":"2023-09-23T15:21:22.256231+0000","last_became_active":"2023-09-23T15:21:19.254005+0000","last_became_peered":"2023-09-23T15:21:19.254005+0000","last_unstale":"2023-09-23T15:21:22.256231+0000","last_undegraded":"2023-09-23T15:21:22.256231+0000","last_fullsized":"2023-09-23T15:21:22.256231+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:50.900770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":272,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.264570+0000","last_change":"2023-09-23T15:21:20.244951+0000","last_active":"2023-09-23T15:21:22.264570+0000","last_peered":"2023-09-23T15:21:22.264570+0000","last_clean":"2023-09-23T15:21:22.264570+0000","last_became_active":"2023-09-23T15:21:20.244162+0000","last_became_peered":"2023-09-23T15:21:20.244162+0000","last_unstale":"2023-09-23T15:21:22.264570+0000","last_undegraded":"2023-09-23T15:21:22.264570+0000","last_fullsized":"2023-09-23T15:21:22.264570+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:27.138548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7,3],"avail_no_missing":["5","3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"292'1110","reported_seq":1126,"reported_epoch":292,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:29.303659+0000","last_change":"2023-09-23T15:21:20.250324+0000","last_active":"2023-09-23T15:21:29.303659+0000","last_peered":"2023-09-23T15:21:29.303659+0000","last_clean":"2023-09-23T15:21:29.303659+0000","last_became_active":"2023-09-23T15:21:20.248843+0000","last_became_peered":"2023-09-23T15:21:20.248843+0000","last_unstale":"2023-09-23T15:21:29.303659+0000","last_undegraded":"2023-09-23T15:21:29.303659+0000","last_fullsized":"2023-09-23T15:21:29.303659+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1110,"log_dups_size":0,"ondisk_log_size":1110,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:46.637917+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1110,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1110,"num_write_kb":71040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1110}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":602,"reported_epoch":291,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:22.237297+0000","last_change":"2023-09-23T15:21:06.202526+0000","last_active":"2023-09-23T15:21:22.237297+0000","last_peered":"2023-09-23T15:21:22.237297+0000","last_clean":"2023-09-23T15:21:04.561374+0000","last_became_active":"2023-09-23T15:21:06.202526+0000","last_became_peered":"2023-09-23T15:21:06.202526+0000","last_unstale":"2023-09-23T15:21:22.237297+0000","last_undegraded":"2023-09-23T15:21:06.186966+0000","last_fullsized":"2023-09-23T15:21:06.186844+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:13.758486+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0],"acting":[3,0],"avail_no_missing":["3","0"],"object_location_counts":[{"shards":"0,3","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":18,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.255357+0000","last_change":"2023-09-23T15:21:19.268148+0000","last_active":"2023-09-23T15:21:22.255357+0000","last_peered":"2023-09-23T15:21:22.255357+0000","last_clean":"2023-09-23T15:21:22.255357+0000","last_became_active":"2023-09-23T15:21:19.253663+0000","last_became_peered":"2023-09-23T15:21:19.253663+0000","last_unstale":"2023-09-23T15:21:22.255357+0000","last_undegraded":"2023-09-23T15:21:22.255357+0000","last_fullsized":"2023-09-23T15:21:22.255357+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:49:51.224645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":222,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.241338+0000","last_change":"2023-09-23T15:21:06.191733+0000","last_active":"2023-09-23T15:21:22.241338+0000","last_peered":"2023-09-23T15:21:22.241338+0000","last_clean":"2023-09-23T15:21:04.171954+0000","last_became_active":"2023-09-23T15:21:06.191733+0000","last_became_peered":"2023-09-23T15:21:06.191733+0000","last_unstale":"2023-09-23T15:21:22.241338+0000","last_undegraded":"2023-09-23T15:21:22.241338+0000","last_fullsized":"2023-09-23T15:21:06.185317+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:48.973089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"292'1050","reported_seq":1074,"reported_epoch":292,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:28.889913+0000","last_change":"2023-09-23T15:21:20.244903+0000","last_active":"2023-09-23T15:21:28.889913+0000","last_peered":"2023-09-23T15:21:28.889913+0000","last_clean":"2023-09-23T15:21:28.889913+0000","last_became_active":"2023-09-23T15:21:20.244448+0000","last_became_peered":"2023-09-23T15:21:20.244448+0000","last_unstale":"2023-09-23T15:21:28.889913+0000","last_undegraded":"2023-09-23T15:21:28.889913+0000","last_fullsized":"2023-09-23T15:21:28.889913+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1050,"log_dups_size":0,"ondisk_log_size":1050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:20.019769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68812800,"num_objects":1050,"num_object_clones":0,"num_object_copies":3150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1050,"num_objects_unfound":0,"num_objects_dirty":1050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1050,"num_write_kb":67200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1050}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":439,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.262238+0000","last_change":"2023-09-23T15:21:06.193205+0000","last_active":"2023-09-23T15:21:22.262238+0000","last_peered":"2023-09-23T15:21:22.262238+0000","last_clean":"2023-09-23T15:00:28.139526+0000","last_became_active":"2023-09-23T15:21:06.193205+0000","last_became_peered":"2023-09-23T15:21:06.193205+0000","last_unstale":"2023-09-23T15:21:22.262238+0000","last_undegraded":"2023-09-23T15:21:22.262238+0000","last_fullsized":"2023-09-23T15:21:06.186793+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":20,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.257414+0000","last_change":"2023-09-23T15:21:20.259943+0000","last_active":"2023-09-23T15:21:22.257414+0000","last_peered":"2023-09-23T15:21:22.257414+0000","last_clean":"2023-09-23T15:21:22.257414+0000","last_became_active":"2023-09-23T15:21:20.249781+0000","last_became_peered":"2023-09-23T15:21:20.249781+0000","last_unstale":"2023-09-23T15:21:22.257414+0000","last_undegraded":"2023-09-23T15:21:22.257414+0000","last_fullsized":"2023-09-23T15:21:22.257414+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:33.123099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":193,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.241094+0000","last_change":"2023-09-23T15:21:20.245200+0000","last_active":"2023-09-23T15:21:22.241094+0000","last_peered":"2023-09-23T15:21:22.241094+0000","last_clean":"2023-09-23T15:21:22.241094+0000","last_became_active":"2023-09-23T15:21:20.244781+0000","last_became_peered":"2023-09-23T15:21:20.244781+0000","last_unstale":"2023-09-23T15:21:22.241094+0000","last_undegraded":"2023-09-23T15:21:22.241094+0000","last_fullsized":"2023-09-23T15:21:22.241094+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:04.866341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,5],"avail_no_missing":["1","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.7","version":"292'1354","reported_seq":1368,"reported_epoch":292,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:31.771811+0000","last_change":"2023-09-23T15:21:21.382691+0000","last_active":"2023-09-23T15:21:31.771811+0000","last_peered":"2023-09-23T15:21:31.771811+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:12.221205+0000","last_became_peered":"2023-09-23T15:21:12.221205+0000","last_unstale":"2023-09-23T15:21:31.771811+0000","last_undegraded":"2023-09-23T15:21:21.375347+0000","last_fullsized":"2023-09-23T15:21:12.209817+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1354,"log_dups_size":0,"ondisk_log_size":1354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:27.324808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88735744,"num_objects":1354,"num_object_clones":0,"num_object_copies":4062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1354,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1354,"num_write_kb":86656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":1354}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":531,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:23.410069+0000","last_change":"2023-09-23T15:21:23.410000+0000","last_active":"2023-09-23T15:21:23.410069+0000","last_peered":"2023-09-23T15:21:23.410069+0000","last_clean":"2023-09-23T15:21:23.410069+0000","last_became_active":"2023-09-23T15:21:22.258358+0000","last_became_peered":"2023-09-23T15:21:22.258358+0000","last_unstale":"2023-09-23T15:21:23.410069+0000","last_undegraded":"2023-09-23T15:21:23.410069+0000","last_fullsized":"2023-09-23T15:21:23.410069+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.248962+0000","scrub_duration":0.003114112,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":258,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.262300+0000","last_change":"2023-09-23T15:21:06.191338+0000","last_active":"2023-09-23T15:21:22.262300+0000","last_peered":"2023-09-23T15:21:22.262300+0000","last_clean":"2023-09-23T15:21:04.683808+0000","last_became_active":"2023-09-23T15:21:06.191338+0000","last_became_peered":"2023-09-23T15:21:06.191338+0000","last_unstale":"2023-09-23T15:21:22.262300+0000","last_undegraded":"2023-09-23T15:21:22.262300+0000","last_fullsized":"2023-09-23T15:21:06.186007+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:22.808277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":263,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256797+0000","last_change":"2023-09-23T15:19:41.353927+0000","last_active":"2023-09-23T15:21:22.256797+0000","last_peered":"2023-09-23T15:21:22.256797+0000","last_clean":"2023-09-23T15:21:22.256797+0000","last_became_active":"2023-09-23T15:19:41.353066+0000","last_became_peered":"2023-09-23T15:19:41.353066+0000","last_unstale":"2023-09-23T15:21:22.256797+0000","last_undegraded":"2023-09-23T15:21:22.256797+0000","last_fullsized":"2023-09-23T15:21:22.256797+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:28.877030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"292'1264","reported_seq":1281,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:30.462023+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:21:30.462023+0000","last_peered":"2023-09-23T15:21:30.462023+0000","last_clean":"2023-09-23T15:21:30.462023+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:21:30.462023+0000","last_undegraded":"2023-09-23T15:21:30.462023+0000","last_fullsized":"2023-09-23T15:21:30.462023+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1264,"log_dups_size":0,"ondisk_log_size":1264,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82837504,"num_objects":1264,"num_object_clones":0,"num_object_copies":3792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1264,"num_write_kb":80896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":528,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.259582+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:21:22.259582+0000","last_peered":"2023-09-23T15:21:22.259582+0000","last_clean":"2023-09-23T15:21:22.259582+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:21:22.259582+0000","last_undegraded":"2023-09-23T15:21:22.259582+0000","last_fullsized":"2023-09-23T15:21:22.259582+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":270,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.237681+0000","last_change":"2023-09-23T15:19:41.387781+0000","last_active":"2023-09-23T15:21:22.237681+0000","last_peered":"2023-09-23T15:21:22.237681+0000","last_clean":"2023-09-23T15:21:22.237681+0000","last_became_active":"2023-09-23T15:19:41.346358+0000","last_became_peered":"2023-09-23T15:19:41.346358+0000","last_unstale":"2023-09-23T15:21:22.237681+0000","last_undegraded":"2023-09-23T15:21:22.237681+0000","last_fullsized":"2023-09-23T15:21:22.237681+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:52.625203+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":115,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.237611+0000","last_change":"2023-09-23T15:21:06.196809+0000","last_active":"2023-09-23T15:21:22.237611+0000","last_peered":"2023-09-23T15:21:22.237611+0000","last_clean":"2023-09-23T15:21:04.561497+0000","last_became_active":"2023-09-23T15:21:06.196809+0000","last_became_peered":"2023-09-23T15:21:06.196809+0000","last_unstale":"2023-09-23T15:21:22.237611+0000","last_undegraded":"2023-09-23T15:21:22.237611+0000","last_fullsized":"2023-09-23T15:21:06.187302+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:25.621647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.1","version":"292'1353","reported_seq":1370,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:31.763871+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:21:31.763871+0000","last_peered":"2023-09-23T15:21:31.763871+0000","last_clean":"2023-09-23T15:21:31.763871+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:21:31.763871+0000","last_undegraded":"2023-09-23T15:21:31.763871+0000","last_fullsized":"2023-09-23T15:21:31.763871+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1353,"log_dups_size":0,"ondisk_log_size":1353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88670208,"num_objects":1353,"num_object_clones":0,"num_object_copies":4059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1353,"num_write_kb":86592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":596,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.264890+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:21:22.264890+0000","last_peered":"2023-09-23T15:21:22.264890+0000","last_clean":"2023-09-23T15:21:22.264890+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:21:22.264890+0000","last_undegraded":"2023-09-23T15:21:22.264890+0000","last_fullsized":"2023-09-23T15:21:22.264890+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":250,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.241491+0000","last_change":"2023-09-23T15:21:06.197306+0000","last_active":"2023-09-23T15:21:22.241491+0000","last_peered":"2023-09-23T15:21:22.241491+0000","last_clean":"2023-09-23T15:19:32.236798+0000","last_became_active":"2023-09-23T15:21:06.197306+0000","last_became_peered":"2023-09-23T15:21:06.197306+0000","last_unstale":"2023-09-23T15:21:22.241491+0000","last_undegraded":"2023-09-23T15:21:22.241491+0000","last_fullsized":"2023-09-23T15:21:06.187204+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:43.965322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":222,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.239328+0000","last_change":"2023-09-23T15:21:06.195902+0000","last_active":"2023-09-23T15:21:22.239328+0000","last_peered":"2023-09-23T15:21:22.239328+0000","last_clean":"2023-09-23T15:21:04.691954+0000","last_became_active":"2023-09-23T15:21:06.195902+0000","last_became_peered":"2023-09-23T15:21:06.195902+0000","last_unstale":"2023-09-23T15:21:22.239328+0000","last_undegraded":"2023-09-23T15:21:22.239328+0000","last_fullsized":"2023-09-23T15:21:06.188607+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:24:07.049098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.2","version":"292'1131","reported_seq":1148,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:28.882092+0000","last_change":"2023-09-23T15:21:12.216500+0000","last_active":"2023-09-23T15:21:28.882092+0000","last_peered":"2023-09-23T15:21:28.882092+0000","last_clean":"2023-09-23T15:21:28.882092+0000","last_became_active":"2023-09-23T15:21:12.216188+0000","last_became_peered":"2023-09-23T15:21:12.216188+0000","last_unstale":"2023-09-23T15:21:28.882092+0000","last_undegraded":"2023-09-23T15:21:28.882092+0000","last_fullsized":"2023-09-23T15:21:28.882092+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1131,"log_dups_size":0,"ondisk_log_size":1131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:17:35.048614+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74121216,"num_objects":1131,"num_object_clones":0,"num_object_copies":3393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1131,"num_write_kb":72384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":547,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241658+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:21:22.241658+0000","last_peered":"2023-09-23T15:21:22.241658+0000","last_clean":"2023-09-23T15:21:22.241658+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:21:22.241658+0000","last_undegraded":"2023-09-23T15:21:22.241658+0000","last_fullsized":"2023-09-23T15:21:22.241658+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":217,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.238662+0000","last_change":"2023-09-23T15:21:19.254459+0000","last_active":"2023-09-23T15:21:22.238662+0000","last_peered":"2023-09-23T15:21:22.238662+0000","last_clean":"2023-09-23T15:21:22.238662+0000","last_became_active":"2023-09-23T15:21:19.254107+0000","last_became_peered":"2023-09-23T15:21:19.254107+0000","last_unstale":"2023-09-23T15:21:22.238662+0000","last_undegraded":"2023-09-23T15:21:22.238662+0000","last_fullsized":"2023-09-23T15:21:22.238662+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:14.614032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.262743+0000","last_change":"2023-09-23T15:21:06.199763+0000","last_active":"2023-09-23T15:21:22.262743+0000","last_peered":"2023-09-23T15:21:22.262743+0000","last_clean":"2023-09-23T15:17:22.128865+0000","last_became_active":"2023-09-23T15:21:06.199763+0000","last_became_peered":"2023-09-23T15:21:06.199763+0000","last_unstale":"2023-09-23T15:21:22.262743+0000","last_undegraded":"2023-09-23T15:21:22.262743+0000","last_fullsized":"2023-09-23T15:21:06.186414+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:35:15.232058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.3","version":"292'1303","reported_seq":1317,"reported_epoch":292,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:31.773008+0000","last_change":"2023-09-23T15:21:21.375353+0000","last_active":"2023-09-23T15:21:31.773008+0000","last_peered":"2023-09-23T15:21:31.773008+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:12.216455+0000","last_became_peered":"2023-09-23T15:21:12.216455+0000","last_unstale":"2023-09-23T15:21:31.773008+0000","last_undegraded":"2023-09-23T15:21:21.367670+0000","last_fullsized":"2023-09-23T15:21:12.210488+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1303,"log_dups_size":0,"ondisk_log_size":1303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:56.456457+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85393408,"num_objects":1303,"num_object_clones":0,"num_object_copies":3909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1303,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1303,"num_write_kb":83392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":1303}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":598,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.261816+0000","last_change":"2023-09-23T15:21:06.196329+0000","last_active":"2023-09-23T15:21:22.261816+0000","last_peered":"2023-09-23T15:21:22.261816+0000","last_clean":"2023-09-23T15:21:04.684289+0000","last_became_active":"2023-09-23T15:21:06.196329+0000","last_became_peered":"2023-09-23T15:21:06.196329+0000","last_unstale":"2023-09-23T15:21:22.261816+0000","last_undegraded":"2023-09-23T15:21:22.261816+0000","last_fullsized":"2023-09-23T15:21:06.185190+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:04.937652+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":305,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:25.470093+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:21:25.470093+0000","last_peered":"2023-09-23T15:21:25.470093+0000","last_clean":"2023-09-23T15:21:25.470093+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:21:25.470093+0000","last_undegraded":"2023-09-23T15:21:25.470093+0000","last_fullsized":"2023-09-23T15:21:25.470093+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":189,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241253+0000","last_change":"2023-09-23T15:21:19.247340+0000","last_active":"2023-09-23T15:21:22.241253+0000","last_peered":"2023-09-23T15:21:22.241253+0000","last_clean":"2023-09-23T15:21:22.241253+0000","last_became_active":"2023-09-23T15:21:19.246958+0000","last_became_peered":"2023-09-23T15:21:19.246958+0000","last_unstale":"2023-09-23T15:21:22.241253+0000","last_undegraded":"2023-09-23T15:21:22.241253+0000","last_fullsized":"2023-09-23T15:21:22.241253+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:34.494051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"292'1092","reported_seq":1118,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:28.878345+0000","last_change":"2023-09-23T15:21:19.255482+0000","last_active":"2023-09-23T15:21:28.878345+0000","last_peered":"2023-09-23T15:21:28.878345+0000","last_clean":"2023-09-23T15:21:28.878345+0000","last_became_active":"2023-09-23T15:21:19.255150+0000","last_became_peered":"2023-09-23T15:21:19.255150+0000","last_unstale":"2023-09-23T15:21:28.878345+0000","last_undegraded":"2023-09-23T15:21:28.878345+0000","last_fullsized":"2023-09-23T15:21:28.878345+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1092,"log_dups_size":0,"ondisk_log_size":1092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:38.854214+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71565312,"num_objects":1092,"num_object_clones":0,"num_object_copies":3276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1092,"num_write_kb":69888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":467,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.489820+0000","last_change":"2023-09-23T15:21:22.489820+0000","last_active":"2023-09-23T15:21:22.489820+0000","last_peered":"2023-09-23T15:21:22.489820+0000","last_clean":"2023-09-23T15:21:22.489820+0000","last_became_active":"2023-09-23T15:21:22.264568+0000","last_became_peered":"2023-09-23T15:21:22.264568+0000","last_unstale":"2023-09-23T15:21:22.489820+0000","last_undegraded":"2023-09-23T15:21:22.489820+0000","last_fullsized":"2023-09-23T15:21:22.489820+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:17:28.054591+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,5],"avail_no_missing":["3","5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":490,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241276+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:21:22.241276+0000","last_peered":"2023-09-23T15:21:22.241276+0000","last_clean":"2023-09-23T15:21:22.241276+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:21:22.241276+0000","last_undegraded":"2023-09-23T15:21:22.241276+0000","last_fullsized":"2023-09-23T15:21:22.241276+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":268,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.263702+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:21:22.263702+0000","last_peered":"2023-09-23T15:21:22.263702+0000","last_clean":"2023-09-23T15:21:22.263702+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:21:22.263702+0000","last_undegraded":"2023-09-23T15:21:22.263702+0000","last_fullsized":"2023-09-23T15:21:22.263702+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":225,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.238702+0000","last_change":"2023-09-23T15:19:41.350708+0000","last_active":"2023-09-23T15:21:22.238702+0000","last_peered":"2023-09-23T15:21:22.238702+0000","last_clean":"2023-09-23T15:21:22.238702+0000","last_became_active":"2023-09-23T15:19:41.350435+0000","last_became_peered":"2023-09-23T15:19:41.350435+0000","last_unstale":"2023-09-23T15:21:22.238702+0000","last_undegraded":"2023-09-23T15:21:22.238702+0000","last_fullsized":"2023-09-23T15:21:22.238702+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:15.309375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"292'1180","reported_seq":1206,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:29.784294+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:21:29.784294+0000","last_peered":"2023-09-23T15:21:29.784294+0000","last_clean":"2023-09-23T15:21:29.784294+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:21:29.784294+0000","last_undegraded":"2023-09-23T15:21:29.784294+0000","last_fullsized":"2023-09-23T15:21:29.784294+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1180,"log_dups_size":0,"ondisk_log_size":1180,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77332480,"num_objects":1180,"num_object_clones":0,"num_object_copies":3540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1180,"num_write_kb":75520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":589,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.263363+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:21:22.263363+0000","last_peered":"2023-09-23T15:21:22.263363+0000","last_clean":"2023-09-23T15:21:22.263363+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:21:22.263363+0000","last_undegraded":"2023-09-23T15:21:22.263363+0000","last_fullsized":"2023-09-23T15:21:22.263363+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":817,"reported_epoch":291,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:22.261633+0000","last_change":"2023-09-23T15:21:06.194583+0000","last_active":"2023-09-23T15:21:22.261633+0000","last_peered":"2023-09-23T15:21:22.261633+0000","last_clean":"2023-09-23T15:21:04.684549+0000","last_became_active":"2023-09-23T15:21:06.194583+0000","last_became_peered":"2023-09-23T15:21:06.194583+0000","last_unstale":"2023-09-23T15:21:22.261633+0000","last_undegraded":"2023-09-23T15:21:06.186207+0000","last_fullsized":"2023-09-23T15:21:06.186092+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:08:42.208124+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":101,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.256614+0000","last_change":"2023-09-23T15:21:06.202493+0000","last_active":"2023-09-23T15:21:22.256614+0000","last_peered":"2023-09-23T15:21:22.256614+0000","last_clean":"2023-09-23T15:21:04.685533+0000","last_became_active":"2023-09-23T15:21:06.202493+0000","last_became_peered":"2023-09-23T15:21:06.202493+0000","last_unstale":"2023-09-23T15:21:22.256614+0000","last_undegraded":"2023-09-23T15:21:22.256614+0000","last_fullsized":"2023-09-23T15:21:06.194026+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:52.619791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":187,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241216+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:21:22.241216+0000","last_peered":"2023-09-23T15:21:22.241216+0000","last_clean":"2023-09-23T15:21:22.241216+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:21:22.241216+0000","last_undegraded":"2023-09-23T15:21:22.241216+0000","last_fullsized":"2023-09-23T15:21:22.241216+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"292'1237","reported_seq":1254,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:30.470252+0000","last_change":"2023-09-23T15:21:12.276548+0000","last_active":"2023-09-23T15:21:30.470252+0000","last_peered":"2023-09-23T15:21:30.470252+0000","last_clean":"2023-09-23T15:21:30.470252+0000","last_became_active":"2023-09-23T15:21:12.276239+0000","last_became_peered":"2023-09-23T15:21:12.276239+0000","last_unstale":"2023-09-23T15:21:30.470252+0000","last_undegraded":"2023-09-23T15:21:30.470252+0000","last_fullsized":"2023-09-23T15:21:30.470252+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1237,"log_dups_size":0,"ondisk_log_size":1237,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:15.416113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81068032,"num_objects":1237,"num_object_clones":0,"num_object_copies":3711,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1237,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1237,"num_write_kb":79168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":627,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.263088+0000","last_change":"2023-09-23T15:21:22.261285+0000","last_active":"2023-09-23T15:21:22.263088+0000","last_peered":"2023-09-23T15:21:22.263088+0000","last_clean":"2023-09-23T15:21:22.263088+0000","last_became_active":"2023-09-23T15:21:22.260953+0000","last_became_peered":"2023-09-23T15:21:22.260953+0000","last_unstale":"2023-09-23T15:21:22.263088+0000","last_undegraded":"2023-09-23T15:21:22.263088+0000","last_fullsized":"2023-09-23T15:21:22.263088+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:08.039094+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":907,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.238593+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:21:22.238593+0000","last_peered":"2023-09-23T15:21:22.238593+0000","last_clean":"2023-09-23T15:21:22.238593+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:21:22.238593+0000","last_undegraded":"2023-09-23T15:21:22.238593+0000","last_fullsized":"2023-09-23T15:21:22.238593+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.240863+0000","last_change":"2023-09-23T15:21:06.196751+0000","last_active":"2023-09-23T15:21:22.240863+0000","last_peered":"2023-09-23T15:21:22.240863+0000","last_clean":"2023-09-23T15:21:04.171793+0000","last_became_active":"2023-09-23T15:21:06.196751+0000","last_became_peered":"2023-09-23T15:21:06.196751+0000","last_unstale":"2023-09-23T15:21:22.240863+0000","last_undegraded":"2023-09-23T15:21:22.240863+0000","last_fullsized":"2023-09-23T15:21:06.185897+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:25:50.670427+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":191,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:26.779875+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:21:26.779875+0000","last_peered":"2023-09-23T15:21:26.779875+0000","last_clean":"2023-09-23T15:21:26.779875+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:21:26.779875+0000","last_undegraded":"2023-09-23T15:21:26.779875+0000","last_fullsized":"2023-09-23T15:21:26.779875+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"292'1191","reported_seq":1204,"reported_epoch":292,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:29.781706+0000","last_change":"2023-09-23T15:21:21.389621+0000","last_active":"2023-09-23T15:21:29.781706+0000","last_peered":"2023-09-23T15:21:29.781706+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:12.217479+0000","last_became_peered":"2023-09-23T15:21:12.217479+0000","last_unstale":"2023-09-23T15:21:29.781706+0000","last_undegraded":"2023-09-23T15:21:21.365743+0000","last_fullsized":"2023-09-23T15:21:12.212828+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1191,"log_dups_size":0,"ondisk_log_size":1191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:41.527978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1191,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1191,"num_write_kb":76224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[{"shards":"0,5","objects":1191}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":621,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.268247+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:21:22.268247+0000","last_peered":"2023-09-23T15:21:22.268247+0000","last_clean":"2023-09-23T15:21:22.268247+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:21:22.268247+0000","last_undegraded":"2023-09-23T15:21:22.268247+0000","last_fullsized":"2023-09-23T15:21:22.268247+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":599,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.261443+0000","last_change":"2023-09-23T15:21:22.261159+0000","last_active":"2023-09-23T15:21:22.261443+0000","last_peered":"2023-09-23T15:21:22.261443+0000","last_clean":"2023-09-23T15:21:22.261443+0000","last_became_active":"2023-09-23T15:21:22.260817+0000","last_became_peered":"2023-09-23T15:21:22.260817+0000","last_unstale":"2023-09-23T15:21:22.261443+0000","last_undegraded":"2023-09-23T15:21:22.261443+0000","last_fullsized":"2023-09-23T15:21:22.261443+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:26.728200+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,1],"avail_no_missing":["7","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":215,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.235901+0000","last_change":"2023-09-23T15:21:06.192937+0000","last_active":"2023-09-23T15:21:22.235901+0000","last_peered":"2023-09-23T15:21:22.235901+0000","last_clean":"2023-09-23T15:21:04.557355+0000","last_became_active":"2023-09-23T15:21:06.192937+0000","last_became_peered":"2023-09-23T15:21:06.192937+0000","last_unstale":"2023-09-23T15:21:22.235901+0000","last_undegraded":"2023-09-23T15:21:22.235901+0000","last_fullsized":"2023-09-23T15:21:06.185273+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:29.111158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":16,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.257561+0000","last_change":"2023-09-23T15:21:20.249983+0000","last_active":"2023-09-23T15:21:22.257561+0000","last_peered":"2023-09-23T15:21:22.257561+0000","last_clean":"2023-09-23T15:21:22.257561+0000","last_became_active":"2023-09-23T15:21:20.248739+0000","last_became_peered":"2023-09-23T15:21:20.248739+0000","last_unstale":"2023-09-23T15:21:22.257561+0000","last_undegraded":"2023-09-23T15:21:22.257561+0000","last_fullsized":"2023-09-23T15:21:22.257561+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:20.982721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,2],"avail_no_missing":["7","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.a","version":"292'1299","reported_seq":1316,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:31.764280+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:21:31.764280+0000","last_peered":"2023-09-23T15:21:31.764280+0000","last_clean":"2023-09-23T15:21:31.764280+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:21:31.764280+0000","last_undegraded":"2023-09-23T15:21:31.764280+0000","last_fullsized":"2023-09-23T15:21:31.764280+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1299,"log_dups_size":0,"ondisk_log_size":1299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85131264,"num_objects":1299,"num_object_clones":0,"num_object_copies":3897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1299,"num_write_kb":83136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":570,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.241633+0000","last_change":"2023-09-23T15:21:06.194361+0000","last_active":"2023-09-23T15:21:22.241633+0000","last_peered":"2023-09-23T15:21:22.241633+0000","last_clean":"2023-09-23T15:21:04.173566+0000","last_became_active":"2023-09-23T15:21:06.194361+0000","last_became_peered":"2023-09-23T15:21:06.194361+0000","last_unstale":"2023-09-23T15:21:22.241633+0000","last_undegraded":"2023-09-23T15:21:22.241633+0000","last_fullsized":"2023-09-23T15:21:06.185777+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:39.196389+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":36,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.254333+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:21:22.254333+0000","last_peered":"2023-09-23T15:21:22.254333+0000","last_clean":"2023-09-23T15:21:22.254333+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:21:22.254333+0000","last_undegraded":"2023-09-23T15:21:22.254333+0000","last_fullsized":"2023-09-23T15:21:22.254333+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":249,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256351+0000","last_change":"2023-09-23T15:19:41.349771+0000","last_active":"2023-09-23T15:21:22.256351+0000","last_peered":"2023-09-23T15:21:22.256351+0000","last_clean":"2023-09-23T15:21:22.256351+0000","last_became_active":"2023-09-23T15:19:41.349349+0000","last_became_peered":"2023-09-23T15:19:41.349349+0000","last_unstale":"2023-09-23T15:21:22.256351+0000","last_undegraded":"2023-09-23T15:21:22.256351+0000","last_fullsized":"2023-09-23T15:21:22.256351+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:33:41.777395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":269,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256713+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:21:22.256713+0000","last_peered":"2023-09-23T15:21:22.256713+0000","last_clean":"2023-09-23T15:21:22.256713+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:21:22.256713+0000","last_undegraded":"2023-09-23T15:21:22.256713+0000","last_fullsized":"2023-09-23T15:21:22.256713+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"292'1230","reported_seq":1246,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:30.456446+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:21:30.456446+0000","last_peered":"2023-09-23T15:21:30.456446+0000","last_clean":"2023-09-23T15:21:30.456446+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:21:30.456446+0000","last_undegraded":"2023-09-23T15:21:30.456446+0000","last_fullsized":"2023-09-23T15:21:30.456446+0000","mapping_epoch":284,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1230,"log_dups_size":0,"ondisk_log_size":1230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609280,"num_objects":1230,"num_object_clones":0,"num_object_copies":3690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1230,"num_write_kb":78720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":516,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.257505+0000","last_change":"2023-09-23T15:21:06.201815+0000","last_active":"2023-09-23T15:21:22.257505+0000","last_peered":"2023-09-23T15:21:22.257505+0000","last_clean":"2023-09-23T15:16:38.968946+0000","last_became_active":"2023-09-23T15:21:06.201815+0000","last_became_peered":"2023-09-23T15:21:06.201815+0000","last_unstale":"2023-09-23T15:21:22.257505+0000","last_undegraded":"2023-09-23T15:21:22.257505+0000","last_fullsized":"2023-09-23T15:21:06.196509+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3],"acting":[0,3],"avail_no_missing":["0","3"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":966,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.266967+0000","last_change":"2023-09-23T15:21:22.263612+0000","last_active":"2023-09-23T15:21:22.266967+0000","last_peered":"2023-09-23T15:21:22.266967+0000","last_clean":"2023-09-23T15:21:22.266967+0000","last_became_active":"2023-09-23T15:21:22.262971+0000","last_became_peered":"2023-09-23T15:21:22.262971+0000","last_unstale":"2023-09-23T15:21:22.266967+0000","last_undegraded":"2023-09-23T15:21:22.266967+0000","last_fullsized":"2023-09-23T15:21:22.266967+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:45.545584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":167,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.241025+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:21:22.241025+0000","last_peered":"2023-09-23T15:21:22.241025+0000","last_clean":"2023-09-23T15:21:22.241025+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:21:22.241025+0000","last_undegraded":"2023-09-23T15:21:22.241025+0000","last_fullsized":"2023-09-23T15:21:22.241025+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":254,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256508+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:21:22.256508+0000","last_peered":"2023-09-23T15:21:22.256508+0000","last_clean":"2023-09-23T15:21:22.256508+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:21:22.256508+0000","last_undegraded":"2023-09-23T15:21:22.256508+0000","last_fullsized":"2023-09-23T15:21:22.256508+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"292'1075","reported_seq":1101,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:28.889677+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:21:28.889677+0000","last_peered":"2023-09-23T15:21:28.889677+0000","last_clean":"2023-09-23T15:21:28.889677+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:21:28.889677+0000","last_undegraded":"2023-09-23T15:21:28.889677+0000","last_fullsized":"2023-09-23T15:21:28.889677+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1075,"log_dups_size":0,"ondisk_log_size":1075,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70451200,"num_objects":1075,"num_object_clones":0,"num_object_copies":3225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1075,"num_write_kb":68800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":577,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.239351+0000","last_change":"2023-09-23T15:21:06.193636+0000","last_active":"2023-09-23T15:21:22.239351+0000","last_peered":"2023-09-23T15:21:22.239351+0000","last_clean":"2023-09-23T15:21:04.692542+0000","last_became_active":"2023-09-23T15:21:06.193636+0000","last_became_peered":"2023-09-23T15:21:06.193636+0000","last_unstale":"2023-09-23T15:21:22.239351+0000","last_undegraded":"2023-09-23T15:21:22.239351+0000","last_fullsized":"2023-09-23T15:21:06.188913+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:07.858934+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":46,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.239520+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:21:22.239520+0000","last_peered":"2023-09-23T15:21:22.239520+0000","last_clean":"2023-09-23T15:21:22.239520+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:21:22.239520+0000","last_undegraded":"2023-09-23T15:21:22.239520+0000","last_fullsized":"2023-09-23T15:21:22.239520+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":800,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:26.779824+0000","last_change":"2023-09-23T15:21:06.200831+0000","last_active":"2023-09-23T15:21:26.779824+0000","last_peered":"2023-09-23T15:21:26.779824+0000","last_clean":"2023-09-23T15:21:04.172426+0000","last_became_active":"2023-09-23T15:21:06.200831+0000","last_became_peered":"2023-09-23T15:21:06.200831+0000","last_unstale":"2023-09-23T15:21:26.779824+0000","last_undegraded":"2023-09-23T15:21:26.779824+0000","last_fullsized":"2023-09-23T15:21:06.190998+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:43:20.095557+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":284,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.235772+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:21:22.235772+0000","last_peered":"2023-09-23T15:21:22.235772+0000","last_clean":"2023-09-23T15:21:22.235772+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:21:22.235772+0000","last_undegraded":"2023-09-23T15:21:22.235772+0000","last_fullsized":"2023-09-23T15:21:22.235772+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"292'1231","reported_seq":1254,"reported_epoch":292,"state":"active+clean","last_fresh":"2023-09-23T15:21:29.780484+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:21:29.780484+0000","last_peered":"2023-09-23T15:21:29.780484+0000","last_clean":"2023-09-23T15:21:29.780484+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:21:29.780484+0000","last_undegraded":"2023-09-23T15:21:29.780484+0000","last_fullsized":"2023-09-23T15:21:29.780484+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":1231,"log_dups_size":0,"ondisk_log_size":1231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674816,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1231,"num_write_kb":78784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":615,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.266045+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:21:22.266045+0000","last_peered":"2023-09-23T15:21:22.266045+0000","last_clean":"2023-09-23T15:21:22.266045+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:21:22.266045+0000","last_undegraded":"2023-09-23T15:21:22.266045+0000","last_fullsized":"2023-09-23T15:21:22.266045+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:42.163013+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":525,"reported_epoch":291,"state":"active+clean+remapped","last_fresh":"2023-09-23T15:21:22.260516+0000","last_change":"2023-09-23T15:21:22.259898+0000","last_active":"2023-09-23T15:21:22.260516+0000","last_peered":"2023-09-23T15:21:22.260516+0000","last_clean":"2023-09-23T15:21:22.260516+0000","last_became_active":"2023-09-23T15:21:22.259307+0000","last_became_peered":"2023-09-23T15:21:22.259307+0000","last_unstale":"2023-09-23T15:21:22.260516+0000","last_undegraded":"2023-09-23T15:21:22.260516+0000","last_fullsized":"2023-09-23T15:21:22.260516+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:23.324011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2,3],"avail_no_missing":["7","2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":283,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256707+0000","last_change":"2023-09-23T15:21:19.265143+0000","last_active":"2023-09-23T15:21:22.256707+0000","last_peered":"2023-09-23T15:21:22.256707+0000","last_clean":"2023-09-23T15:21:22.256707+0000","last_became_active":"2023-09-23T15:21:19.262038+0000","last_became_peered":"2023-09-23T15:21:19.262038+0000","last_unstale":"2023-09-23T15:21:22.256707+0000","last_undegraded":"2023-09-23T15:21:22.256707+0000","last_fullsized":"2023-09-23T15:21:22.256707+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:41.979019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.263457+0000","last_change":"2023-09-23T15:21:06.198297+0000","last_active":"2023-09-23T15:21:22.263457+0000","last_peered":"2023-09-23T15:21:22.263457+0000","last_clean":"2023-09-23T15:21:04.684073+0000","last_became_active":"2023-09-23T15:21:06.198297+0000","last_became_peered":"2023-09-23T15:21:06.198297+0000","last_unstale":"2023-09-23T15:21:22.263457+0000","last_undegraded":"2023-09-23T15:21:22.263457+0000","last_fullsized":"2023-09-23T15:21:06.185172+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:49.093530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":264,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256399+0000","last_change":"2023-09-23T15:21:19.262438+0000","last_active":"2023-09-23T15:21:22.256399+0000","last_peered":"2023-09-23T15:21:22.256399+0000","last_clean":"2023-09-23T15:21:22.256399+0000","last_became_active":"2023-09-23T15:21:19.262042+0000","last_became_peered":"2023-09-23T15:21:19.262042+0000","last_unstale":"2023-09-23T15:21:22.256399+0000","last_undegraded":"2023-09-23T15:21:22.256399+0000","last_fullsized":"2023-09-23T15:21:22.256399+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:52.470397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":294,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.239259+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:21:22.239259+0000","last_peered":"2023-09-23T15:21:22.239259+0000","last_clean":"2023-09-23T15:21:22.239259+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:21:22.239259+0000","last_undegraded":"2023-09-23T15:21:22.239259+0000","last_fullsized":"2023-09-23T15:21:22.239259+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":291,"reported_epoch":291,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:22.262123+0000","last_change":"2023-09-23T15:21:06.194936+0000","last_active":"2023-09-23T15:21:22.262123+0000","last_peered":"2023-09-23T15:21:22.262123+0000","last_clean":"2023-09-23T15:21:04.684704+0000","last_became_active":"2023-09-23T15:21:06.194936+0000","last_became_peered":"2023-09-23T15:21:06.194936+0000","last_unstale":"2023-09-23T15:21:22.262123+0000","last_undegraded":"2023-09-23T15:21:06.186428+0000","last_fullsized":"2023-09-23T15:21:06.186320+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:58:54.701633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":216,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.237762+0000","last_change":"2023-09-23T15:19:41.346796+0000","last_active":"2023-09-23T15:21:22.237762+0000","last_peered":"2023-09-23T15:21:22.237762+0000","last_clean":"2023-09-23T15:21:22.237762+0000","last_became_active":"2023-09-23T15:19:41.346289+0000","last_became_peered":"2023-09-23T15:19:41.346289+0000","last_unstale":"2023-09-23T15:21:22.237762+0000","last_undegraded":"2023-09-23T15:21:22.237762+0000","last_fullsized":"2023-09-23T15:21:22.237762+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:27.690863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":257,"reported_epoch":291,"state":"active+undersized+degraded","last_fresh":"2023-09-23T15:21:22.261914+0000","last_change":"2023-09-23T15:21:06.194740+0000","last_active":"2023-09-23T15:21:22.261914+0000","last_peered":"2023-09-23T15:21:22.261914+0000","last_clean":"2023-09-23T15:21:04.684679+0000","last_became_active":"2023-09-23T15:21:06.194740+0000","last_became_peered":"2023-09-23T15:21:06.194740+0000","last_unstale":"2023-09-23T15:21:22.261914+0000","last_undegraded":"2023-09-23T15:21:06.185649+0000","last_fullsized":"2023-09-23T15:21:06.185540+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:39.465001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":126,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.236065+0000","last_change":"2023-09-23T15:19:41.352434+0000","last_active":"2023-09-23T15:21:22.236065+0000","last_peered":"2023-09-23T15:21:22.236065+0000","last_clean":"2023-09-23T15:21:22.236065+0000","last_became_active":"2023-09-23T15:19:41.352220+0000","last_became_peered":"2023-09-23T15:19:41.352220+0000","last_unstale":"2023-09-23T15:21:22.236065+0000","last_undegraded":"2023-09-23T15:21:22.236065+0000","last_fullsized":"2023-09-23T15:21:22.236065+0000","mapping_epoch":254,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:55.334410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":272,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.236591+0000","last_change":"2023-09-23T15:21:06.197347+0000","last_active":"2023-09-23T15:21:22.236591+0000","last_peered":"2023-09-23T15:21:22.236591+0000","last_clean":"2023-09-23T15:21:04.551428+0000","last_became_active":"2023-09-23T15:21:06.197347+0000","last_became_peered":"2023-09-23T15:21:06.197347+0000","last_unstale":"2023-09-23T15:21:22.236591+0000","last_undegraded":"2023-09-23T15:21:22.236591+0000","last_fullsized":"2023-09-23T15:21:06.187423+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:50.682255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":280,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.235978+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:21:22.235978+0000","last_peered":"2023-09-23T15:21:22.235978+0000","last_clean":"2023-09-23T15:21:22.235978+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:21:22.235978+0000","last_undegraded":"2023-09-23T15:21:22.235978+0000","last_fullsized":"2023-09-23T15:21:22.235978+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":243,"reported_epoch":291,"state":"active+clean","last_fresh":"2023-09-23T15:21:22.256573+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:21:22.256573+0000","last_peered":"2023-09-23T15:21:22.256573+0000","last_clean":"2023-09-23T15:21:22.256573+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:21:22.256573+0000","last_undegraded":"2023-09-23T15:21:22.256573+0000","last_fullsized":"2023-09-23T15:21:22.256573+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":88,"reported_epoch":291,"state":"active+undersized","last_fresh":"2023-09-23T15:21:22.238490+0000","last_change":"2023-09-23T15:21:06.192450+0000","last_active":"2023-09-23T15:21:22.238490+0000","last_peered":"2023-09-23T15:21:22.238490+0000","last_clean":"2023-09-23T15:05:12.637802+0000","last_became_active":"2023-09-23T15:21:06.192450+0000","last_became_peered":"2023-09-23T15:21:06.192450+0000","last_unstale":"2023-09-23T15:21:22.238490+0000","last_undegraded":"2023-09-23T15:21:22.238490+0000","last_fullsized":"2023-09-23T15:21:06.186473+0000","mapping_epoch":281,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":255,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:56:43.427012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":81,"acting":85,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":7,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":5042,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":147456,"data_stored":26320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":26927,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":82,"acting":86,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1267007488,"num_objects":19333,"num_object_clones":0,"num_object_copies":57999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5081,"num_objects_misplaced":2160,"num_objects_unfound":0,"num_objects_dirty":19333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":19333,"num_write_kb":1237312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3433889792,"data_stored":3433889792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":19333,"ondisk_log_size":19333,"up":42,"acting":44,"num_store_stats":6},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":53,"acting":56,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201601,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":510708,"kb_used_data":414252,"kb_used_omap":0,"kb_used_meta":96448,"kb_avail":93234444,"statfs":{"total":95995035648,"available":95472070656,"internally_reserved":0,"allocated":424194048,"data_stored":423323907,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":98762752},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:21:15 2023","interfaces":[{"interface":"back","average":{"1min":10.670999999999999,"5min":4.1289999999999996,"15min":2.7370000000000001},"min":{"1min":0.251,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":167.39500000000001,"5min":167.39500000000001,"15min":167.39500000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":10.614000000000001,"5min":4.7140000000000004,"15min":3.044},"min":{"1min":0.312,"5min":0.26800000000000002,"15min":0.24399999999999999},"max":{"1min":167.512,"5min":167.512,"15min":167.512},"last":2.8279999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.2889999999999999,"15min":2.2989999999999999},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.4299999999999997,"5min":9.0459999999999994,"15min":9.0459999999999994},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.2210000000000001,"5min":1.9930000000000001,"15min":2.8679999999999999},"min":{"1min":0.35299999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":7.2510000000000003,"5min":16.065999999999999,"15min":16.065999999999999},"last":0.46500000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:21:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":2.5539999999999998,"15min":3.7719999999999998},"min":{"1min":0.315,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":5.359,"5min":90.227999999999994,"15min":170.63800000000001},"last":0.37},{"interface":"front","average":{"1min":1.5840000000000001,"5min":3.2189999999999999,"15min":4.3700000000000001},"min":{"1min":0.38500000000000001,"5min":0.32400000000000001,"15min":0.311},"max":{"1min":4.4020000000000001,"5min":90.197999999999993,"15min":170.47999999999999},"last":0.54800000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:20:54 2023","interfaces":[{"interface":"back","average":{"1min":10.353,"5min":5.8280000000000003,"15min":3.5920000000000001},"min":{"1min":0.436,"5min":0.316,"15min":0.28899999999999998},"max":{"1min":146.166,"5min":146.166,"15min":146.166},"last":0.309},{"interface":"front","average":{"1min":10.367000000000001,"5min":5.8730000000000002,"15min":3.5790000000000002},"min":{"1min":0.46200000000000002,"5min":0.36599999999999999,"15min":0.32400000000000001},"max":{"1min":146.18299999999999,"5min":146.18299999999999,"15min":146.18299999999999},"last":0.40799999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":1.3999999999999999,"15min":2.411},"min":{"1min":0.33200000000000002,"5min":0.28499999999999998,"15min":0.27000000000000002},"max":{"1min":0.83999999999999997,"5min":87.513000000000005,"15min":150.119},"last":0.44},{"interface":"front","average":{"1min":0.41299999999999998,"5min":1.393,"15min":2.6800000000000002},"min":{"1min":0.29899999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.53000000000000003,"5min":87.951999999999998,"15min":150.596},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":1.4650000000000001,"15min":2.1200000000000001},"min":{"1min":0.29499999999999998,"5min":0.28999999999999998,"15min":0.27800000000000002},"max":{"1min":0.878,"5min":86.013999999999996,"15min":141.53800000000001},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.52700000000000002,"5min":1.4810000000000001,"15min":2.1339999999999999},"min":{"1min":0.38800000000000001,"5min":0.31,"15min":0.29599999999999999},"max":{"1min":0.89200000000000002,"5min":85.841999999999999,"15min":141.30199999999999},"last":0.33300000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:20:36 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":2.6040000000000001,"15min":2.2989999999999999},"min":{"1min":0.40799999999999997,"5min":0.35899999999999999,"15min":0.315},"max":{"1min":1.0880000000000001,"5min":97.224000000000004,"15min":153.47300000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.47799999999999998,"5min":2.649,"15min":2.5289999999999999},"min":{"1min":0.315,"5min":0.30499999999999999,"15min":0.29299999999999998},"max":{"1min":0.65200000000000002,"5min":100.666,"15min":153.07900000000001},"last":0.71299999999999997}]}]},{"osd":6,"up_from":47,"seq":201863463232,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252644,"kb_used_data":3100,"kb_used_omap":0,"kb_used_meta":249536,"kb_avail":93492508,"statfs":{"total":95995035648,"available":95736328192,"internally_reserved":0,"allocated":3174400,"data_stored":2343039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":255524864},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4400000000000004,"5min":2.2400000000000002,"15min":3.2440000000000002},"min":{"1min":0.255,"5min":0.22800000000000001,"15min":0.19800000000000001},"max":{"1min":78.909000000000006,"5min":78.909000000000006,"15min":108.39400000000001},"last":0.497},{"interface":"front","average":{"1min":6.923,"5min":2.8450000000000002,"15min":3.573},"min":{"1min":0.251,"5min":0.24099999999999999,"15min":0.223},"max":{"1min":79.430000000000007,"5min":79.430000000000007,"15min":108.452},"last":0.60499999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:20:56 2023","interfaces":[{"interface":"back","average":{"1min":9.5129999999999999,"5min":3.524,"15min":3.4300000000000002},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":111.90300000000001,"5min":111.90300000000001,"15min":111.90300000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":9.3889999999999993,"5min":3.984,"15min":3.9550000000000001},"min":{"1min":0.28799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":111.917,"5min":111.917,"15min":111.917},"last":0.46999999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:20:08 2023","interfaces":[{"interface":"back","average":{"1min":6.4560000000000004,"5min":4.7350000000000003,"15min":7.0739999999999998},"min":{"1min":0.28100000000000003,"5min":0.27500000000000002,"15min":0.255},"max":{"1min":80.875,"5min":86.569000000000003,"15min":180.374},"last":0.51500000000000001},{"interface":"front","average":{"1min":7.7530000000000001,"5min":4.4909999999999997,"15min":7.0229999999999997},"min":{"1min":0.39500000000000002,"5min":0.28699999999999998,"15min":0.23999999999999999},"max":{"1min":84.096000000000004,"5min":88.269000000000005,"15min":181.685},"last":0.52900000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5209999999999999,"5min":3.726,"15min":4.0529999999999999},"min":{"1min":0.39500000000000002,"5min":0.29999999999999999,"15min":0.249},"max":{"1min":5.827,"5min":84.004999999999995,"15min":102.254},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.8199999999999998,"5min":3.7240000000000002,"15min":4.1219999999999999},"min":{"1min":0.48299999999999998,"5min":0.36399999999999999,"15min":0.29799999999999999},"max":{"1min":6.9219999999999997,"5min":83.936000000000007,"15min":98.941999999999993},"last":0.58499999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.52800000000000002,"15min":1.0329999999999999},"min":{"1min":0.30199999999999999,"5min":0.26400000000000001,"15min":0.23000000000000001},"max":{"1min":1.5569999999999999,"5min":1.7150000000000001,"15min":36.290999999999997},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.59199999999999997,"15min":1.129},"min":{"1min":0.28899999999999998,"5min":0.253,"15min":0.253},"max":{"1min":1.337,"5min":2.6259999999999999,"15min":36.758000000000003},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:20:17 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.58199999999999996,"15min":1.413},"min":{"1min":0.36199999999999999,"5min":0.30299999999999999,"15min":0.28100000000000003},"max":{"1min":1.4770000000000001,"5min":3.1789999999999998,"15min":131.90600000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.64300000000000002,"15min":1.4690000000000001},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.92800000000000005,"5min":3.431,"15min":131.61000000000001},"last":0.70999999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:20:50 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.63300000000000001,"15min":1.032},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.28999999999999998},"max":{"1min":2.4399999999999999,"5min":2.9449999999999998,"15min":17},"last":0.64800000000000002},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.61799999999999999,"15min":0.97299999999999998},"min":{"1min":0.38,"5min":0.32300000000000001,"15min":0.25800000000000001},"max":{"1min":1.581,"5min":1.8180000000000001,"15min":7.7809999999999997},"last":0.68100000000000005}]}]},{"osd":1,"up_from":183,"seq":785979015275,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":979720,"kb_used_data":667904,"kb_used_omap":6,"kb_used_meta":311801,"kb_avail":92765432,"statfs":{"total":95995035648,"available":94991802368,"internally_reserved":0,"allocated":683933696,"data_stored":682589995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":319284299},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":3.3090000000000002,"5min":3.2709999999999999,"15min":3.4929999999999999},"min":{"1min":0.23200000000000001,"5min":0.191,"15min":0.191},"max":{"1min":35.033000000000001,"5min":35.033000000000001,"15min":35.033000000000001},"last":0.19400000000000001},{"interface":"front","average":{"1min":4.0860000000000003,"5min":3.3679999999999999,"15min":4.1970000000000001},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":34.667999999999999,"5min":34.667999999999999,"15min":34.667999999999999},"last":2.3170000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":5.0970000000000004,"15min":4.4560000000000004},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.329000000000001,"5min":109.375,"15min":109.375},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.6319999999999999,"5min":3.673,"15min":3.9889999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.253},"max":{"1min":9.6219999999999999,"5min":105.16,"15min":105.16},"last":0.71199999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":6.5869999999999997,"5min":8.4589999999999996,"15min":9.2050000000000001},"min":{"1min":0.33300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":39.941000000000003,"5min":211.405,"15min":211.405},"last":0.68899999999999995},{"interface":"front","average":{"1min":7.3090000000000002,"5min":8.5109999999999992,"15min":10.135},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.27200000000000002},"max":{"1min":36.308,"5min":210.988,"15min":210.988},"last":0.59899999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.7789999999999999,"5min":4.9829999999999997,"15min":3.4580000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":12.885,"5min":80.570999999999998,"15min":80.570999999999998},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.141,"5min":4.25,"15min":3.7109999999999999},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.308},"max":{"1min":4.9969999999999999,"5min":80.75,"15min":80.75},"last":0.65500000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.536,"15min":3.25},"min":{"1min":0.34200000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":10.833,"5min":34.512999999999998,"15min":34.512999999999998},"last":0.629},{"interface":"front","average":{"1min":1.952,"5min":3.1520000000000001,"15min":3.0310000000000001},"min":{"1min":0.35599999999999998,"5min":0.311,"15min":0.311},"max":{"1min":12.795,"5min":37.911000000000001,"15min":37.911000000000001},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":4.6180000000000003,"15min":3.9500000000000002},"min":{"1min":0.38300000000000001,"5min":0.312,"15min":0.312},"max":{"1min":10.785,"5min":81.352000000000004,"15min":81.352000000000004},"last":5.806},{"interface":"front","average":{"1min":1.7889999999999999,"5min":5.2329999999999997,"15min":3.661},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":12.834,"5min":84.716999999999999,"15min":84.716999999999999},"last":0.79600000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2,"5min":2.6619999999999999,"15min":2.9849999999999999},"min":{"1min":0.42799999999999999,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":12.861000000000001,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.53200000000000003},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.7729999999999999,"15min":2.8159999999999998},"min":{"1min":0.40000000000000002,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":5.0279999999999996,"5min":13.282999999999999,"15min":16.841000000000001},"last":0.73699999999999999}]}]},{"osd":0,"up_from":95,"seq":408021893324,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917216,"kb_used_data":479064,"kb_used_omap":8,"kb_used_meta":438135,"kb_avail":92827936,"statfs":{"total":95995035648,"available":95055806464,"internally_reserved":0,"allocated":490561536,"data_stored":489314370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":448650836},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:20:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6440000000000001,"5min":4.0430000000000001,"15min":3.8769999999999998},"min":{"1min":0.26500000000000001,"5min":0.18099999999999999,"15min":0.18099999999999999},"max":{"1min":9.4909999999999997,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.21199999999999999},{"interface":"front","average":{"1min":2.476,"5min":3.21,"15min":3.8820000000000001},"min":{"1min":0.26500000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.101000000000001,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.30599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.292,"5min":2.3769999999999998,"15min":4.2119999999999997},"min":{"1min":0.223,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":8.1419999999999995,"5min":15.209,"15min":122.20099999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.835,"5min":3.0880000000000001,"15min":3.9849999999999999},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.18099999999999999},"max":{"1min":7.8879999999999999,"5min":64.466999999999999,"15min":122.071},"last":0.65100000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:20:43 2023","interfaces":[{"interface":"back","average":{"1min":3.6920000000000002,"5min":6.3630000000000004,"15min":4.8250000000000002},"min":{"1min":0.27400000000000002,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":14.785,"5min":146.315,"15min":146.315},"last":0.28899999999999998},{"interface":"front","average":{"1min":5.1890000000000001,"5min":7.5529999999999999,"15min":5.3849999999999998},"min":{"1min":0.39600000000000002,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":24.629000000000001,"5min":145.517,"15min":145.517},"last":0.68799999999999994}]},{"osd":4,"last update":"Sat Sep 23 15:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.3859999999999999,"5min":1.569,"15min":1.9550000000000001},"min":{"1min":0.34000000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":10.99,"5min":10.99,"15min":31.341999999999999},"last":0.504},{"interface":"front","average":{"1min":1.149,"5min":1.7430000000000001,"15min":2.089},"min":{"1min":0.32900000000000001,"5min":0.311,"15min":0.27000000000000002},"max":{"1min":8.4960000000000004,"5min":8.4960000000000004,"15min":99.168999999999997},"last":0.33700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5289999999999999,"5min":2.0819999999999999,"15min":2.23},"min":{"1min":0.35199999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.9420000000000002,"5min":64.349999999999994,"15min":109.369},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.413,"5min":1.6100000000000001,"15min":2.149},"min":{"1min":0.35299999999999998,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":11.036,"5min":11.036,"15min":109.387},"last":0.57499999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:20:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1539999999999999,"5min":2.1840000000000002,"15min":2.1280000000000001},"min":{"1min":0.39100000000000001,"5min":0.378,"15min":0.309},"max":{"1min":7.0739999999999998,"5min":13.529999999999999,"15min":99.114999999999995},"last":0.43099999999999999},{"interface":"front","average":{"1min":2.2959999999999998,"5min":2.4020000000000001,"15min":2.1309999999999998},"min":{"1min":0.38400000000000001,"5min":0.35399999999999998,"15min":0.313},"max":{"1min":8.625,"5min":64.378,"15min":98.921999999999997},"last":0.5}]},{"osd":7,"last update":"Sat Sep 23 15:21:19 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":2.141,"15min":2.2909999999999999},"min":{"1min":0.375,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":7.9960000000000004,"5min":64.304000000000002,"15min":99.549999999999997},"last":0.74199999999999999},{"interface":"front","average":{"1min":1.45,"5min":1.718,"15min":1.8899999999999999},"min":{"1min":0.42299999999999999,"5min":0.39300000000000002,"15min":0.36399999999999999},"max":{"1min":11.065,"5min":11.065,"15min":31.225999999999999},"last":0.54300000000000004}]}]},{"osd":2,"up_from":105,"seq":450971566271,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1042308,"kb_used_data":571836,"kb_used_omap":11,"kb_used_meta":470452,"kb_avail":92702844,"statfs":{"total":95995035648,"available":94927712256,"internally_reserved":0,"allocated":585560064,"data_stored":584338321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":481743632},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:21:17 2023","interfaces":[{"interface":"back","average":{"1min":9.9990000000000006,"5min":4.673,"15min":3.6469999999999998},"min":{"1min":0.26500000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":81.218999999999994,"5min":81.218999999999994,"15min":87.132999999999996},"last":1.6080000000000001},{"interface":"front","average":{"1min":10.144,"5min":4.758,"15min":3.2890000000000001},"min":{"1min":0.29499999999999998,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":81.352000000000004,"5min":81.352000000000004,"15min":84.820999999999998},"last":1.679}]},{"osd":1,"last update":"Sat Sep 23 15:20:55 2023","interfaces":[{"interface":"back","average":{"1min":2.6179999999999999,"5min":3.835,"15min":4.7889999999999997},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.522,"5min":31.513999999999999,"15min":84.641000000000005},"last":1.1160000000000001},{"interface":"front","average":{"1min":2.1539999999999999,"5min":3.1280000000000001,"15min":3.153},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":10.763999999999999,"5min":19.983000000000001,"15min":84.751999999999995},"last":1.4370000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:20:40 2023","interfaces":[{"interface":"back","average":{"1min":7.6479999999999997,"5min":5.532,"15min":4.3760000000000003},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.219},"max":{"1min":58.987000000000002,"5min":58.987000000000002,"15min":87.108000000000004},"last":0.65700000000000003},{"interface":"front","average":{"1min":8.4789999999999992,"5min":6.0510000000000002,"15min":4.4199999999999999},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":59.457999999999998,"5min":59.457999999999998,"15min":86.158000000000001},"last":1.351}]},{"osd":4,"last update":"Sat Sep 23 15:21:17 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.9339999999999999,"15min":2.2589999999999999},"min":{"1min":0.36399999999999999,"5min":0.32400000000000001,"15min":0.27200000000000002},"max":{"1min":10.837,"5min":16.831,"15min":84.608000000000004},"last":1.5349999999999999},{"interface":"front","average":{"1min":1.625,"5min":2.0379999999999998,"15min":2.8599999999999999},"min":{"1min":0.32800000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":7.7880000000000003,"5min":18.847999999999999,"15min":87.019999999999996},"last":1.4219999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:21:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.149,"15min":2.5630000000000002},"min":{"1min":0.34599999999999997,"5min":0.31900000000000001,"15min":0.28899999999999998},"max":{"1min":7.6289999999999996,"5min":18.875,"15min":84.498000000000005},"last":1.175},{"interface":"front","average":{"1min":1.5189999999999999,"5min":2.0609999999999999,"15min":2.3780000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":10.808999999999999,"5min":16.898,"15min":84.664000000000001},"last":1.4870000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":1.5760000000000001,"5min":2.2149999999999999,"15min":2.4430000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.32100000000000001},"max":{"1min":6.7220000000000004,"5min":16.872,"15min":84.715999999999994},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.53,"5min":2.472,"15min":2.5299999999999998},"min":{"1min":0.36599999999999999,"5min":0.311,"15min":0.311},"max":{"1min":7.641,"5min":18.774000000000001,"15min":87.055000000000007},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:21:17 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":2.1600000000000001,"15min":2.399},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":7.6749999999999998,"5min":18.954999999999998,"15min":84.596000000000004},"last":1.151},{"interface":"front","average":{"1min":1.5620000000000001,"5min":2.1970000000000001,"15min":2.3050000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":10.861000000000001,"5min":16.920999999999999,"15min":84.783000000000001},"last":1.512}]}]},{"osd":3,"up_from":110,"seq":472446402745,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1072372,"kb_used_data":566316,"kb_used_omap":3,"kb_used_meta":506044,"kb_avail":92672780,"statfs":{"total":95995035648,"available":94896926720,"internally_reserved":0,"allocated":579907584,"data_stored":578695599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":518189225},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9830000000000001,"5min":3.3929999999999998,"15min":3.016},"min":{"1min":0.35199999999999998,"5min":0.20100000000000001,"15min":0.17899999999999999},"max":{"1min":9.077,"5min":33.220999999999997,"15min":79.611000000000004},"last":0.66300000000000003},{"interface":"front","average":{"1min":3.4489999999999998,"5min":3.8929999999999998,"15min":3.1930000000000001},"min":{"1min":0.318,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":12.186,"5min":33.084000000000003,"15min":79.656000000000006},"last":0.215}]},{"osd":1,"last update":"Sat Sep 23 15:20:59 2023","interfaces":[{"interface":"back","average":{"1min":6.5179999999999998,"5min":5.359,"15min":4.8819999999999997},"min":{"1min":0.23100000000000001,"5min":0.214,"15min":0.214},"max":{"1min":73.394000000000005,"5min":119.343,"15min":119.343},"last":0.33800000000000002},{"interface":"front","average":{"1min":7.2069999999999999,"5min":5.5380000000000003,"15min":4.9820000000000002},"min":{"1min":0.23999999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":75.918000000000006,"5min":119.505,"15min":119.505},"last":0.75800000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.6040000000000001,"5min":3.0030000000000001,"15min":4.4850000000000003},"min":{"1min":0.34899999999999998,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":8.7699999999999996,"5min":18.893999999999998,"15min":109.077},"last":0.79000000000000004},{"interface":"front","average":{"1min":3.7360000000000002,"5min":4.6699999999999999,"15min":5.3620000000000001},"min":{"1min":0.48699999999999999,"5min":0.27800000000000002,"15min":0.25700000000000001},"max":{"1min":12.35,"5min":119.449,"15min":119.72799999999999},"last":0.84199999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.375,"5min":2.343,"15min":2.0139999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":12.256,"5min":12.958,"15min":54.051000000000002},"last":0.80700000000000005},{"interface":"front","average":{"1min":1.859,"5min":1.944,"15min":2.0369999999999999},"min":{"1min":0.36399999999999999,"5min":0.34999999999999998,"15min":0.26400000000000001},"max":{"1min":12.500999999999999,"5min":12.500999999999999,"15min":52.228999999999999},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.3959999999999999,"5min":2.3919999999999999,"15min":2.1579999999999999},"min":{"1min":0.32800000000000001,"5min":0.29599999999999999,"15min":0.25600000000000001},"max":{"1min":12.228999999999999,"5min":12.999000000000001,"15min":56.960999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.839,"5min":2.0750000000000002,"15min":2.23},"min":{"1min":0.33300000000000002,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":12.478,"5min":12.667999999999999,"15min":56.981999999999999},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.4420000000000002,"5min":2.52,"15min":2.3140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":12.294,"5min":13.029999999999999,"15min":78.102000000000004},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.8919999999999999,"5min":2.105,"15min":2.3849999999999998},"min":{"1min":0.39300000000000002,"5min":0.35599999999999998,"15min":0.32300000000000001},"max":{"1min":12.446999999999999,"5min":12.446999999999999,"15min":78.028000000000006},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:20:45 2023","interfaces":[{"interface":"back","average":{"1min":1.885,"5min":2.1920000000000002,"15min":2.177},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.28299999999999997},"max":{"1min":12.522,"5min":12.701000000000001,"15min":14.885999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.762,"5min":2.1890000000000001,"15min":1.875},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":8.8230000000000004,"5min":18.696000000000002,"15min":18.696000000000002},"last":0.70699999999999996}]}]},{"osd":4,"up_from":0,"seq":133143986510,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724873,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864932,"kb_used_data":563996,"kb_used_omap":0,"kb_used_meta":300928,"kb_avail":92880220,"statfs":{"total":95995035648,"available":95109345280,"internally_reserved":0,"allocated":577531904,"data_stored":576674314,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":308150272},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:21:12 2023","interfaces":[{"interface":"back","average":{"1min":2.766,"5min":4.9720000000000004,"15min":2.883},"min":{"1min":0.252,"5min":0.22500000000000001,"15min":0.20599999999999999},"max":{"1min":16.725999999999999,"5min":124.31,"15min":124.31},"last":1.3069999999999999},{"interface":"front","average":{"1min":1.7809999999999999,"5min":4.6790000000000003,"15min":2.8359999999999999},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":16.908999999999999,"5min":124.374,"15min":124.374},"last":1.53}]},{"osd":1,"last update":"Sat Sep 23 15:21:03 2023","interfaces":[{"interface":"back","average":{"1min":7.1479999999999997,"5min":3.6880000000000002,"15min":2.5779999999999998},"min":{"1min":0.28199999999999997,"5min":0.248,"15min":0.23699999999999999},"max":{"1min":66.75,"5min":124.38800000000001,"15min":124.38800000000001},"last":0.72599999999999998},{"interface":"front","average":{"1min":8.3550000000000004,"5min":3.9249999999999998,"15min":2.9889999999999999},"min":{"1min":0.29199999999999998,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":66.825000000000003,"5min":124.29000000000001,"15min":124.29000000000001},"last":1.3400000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:21:12 2023","interfaces":[{"interface":"back","average":{"1min":5.2619999999999996,"5min":4.0949999999999998,"15min":3.6389999999999998},"min":{"1min":0.313,"5min":0.25700000000000001,"15min":0.23599999999999999},"max":{"1min":74.019000000000005,"5min":193.00999999999999,"15min":193.00999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":4.819,"5min":4.3799999999999999,"15min":3.6840000000000002},"min":{"1min":0.38200000000000001,"5min":0.27900000000000003,"15min":0.254},"max":{"1min":73.712000000000003,"5min":193.15700000000001,"15min":193.15700000000001},"last":1.381}]},{"osd":3,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":7.8499999999999996,"15min":4.9020000000000001},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":4.9320000000000004,"5min":186.56999999999999,"15min":186.56999999999999},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.4630000000000001,"5min":7.6669999999999998,"15min":4.694},"min":{"1min":0.30599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.46,"5min":186.74600000000001,"15min":186.74600000000001},"last":0.61499999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:21:03 2023","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.61599999999999999,"15min":1.762},"min":{"1min":0.29499999999999998,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":1.893,"5min":12.202,"15min":138.68600000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.66800000000000004,"15min":1.831},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":1.9330000000000001,"5min":12.481,"15min":138.49600000000001},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 15:20:15 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.54900000000000004,"15min":1.119},"min":{"1min":0.32700000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0129999999999999,"5min":3.5550000000000002,"15min":15.196},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":0.58199999999999996,"15min":0.999},"min":{"1min":0.35999999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.167,"5min":2.9990000000000001,"15min":14.965999999999999},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:20:41 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.56999999999999995,"15min":1.276},"min":{"1min":0.35799999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.599,"5min":2.1869999999999998,"15min":15.301},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":0.63900000000000001,"15min":1.2589999999999999},"min":{"1min":0.42399999999999999,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":1.9550000000000001,"5min":3.5379999999999998,"15min":15.762},"last":0.46500000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":500105216,"data_stored":500105216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":691732480,"data_stored":691732480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":586088448,"data_stored":586088448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":580386816,"data_stored":580386816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":629932032,"data_stored":629932032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":445644800,"data_stored":445644800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:21:35.213 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T15:21:35.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:21:35.216 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:35.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:35.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:35.325 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:35.370 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 26341 26325 117.447 74.25 0.0187693 0.00850851 2023-09-23T15:21:35.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.473 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:35.498 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:35.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:35.631 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:35.709 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:35.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:35.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:35.874 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:35.991 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:35.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:35.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:36.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:36.173 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:36.360 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:36.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:36.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:36.371 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 27603 27587 114.868 78.875 0.00715504 0.00869954 2023-09-23T15:21:36.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:36.494 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:36 smithi027 ceph-mon[195233]: pgmap v837: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 5089/64776 objects degraded (7.856%); 3077/64776 objects misplaced (4.750%); 0 B/s, 0 objects/s recovering 2023-09-23T15:21:36.498 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:36.558 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:36.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:36.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:36.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:36.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:36 smithi017 ceph-mon[222878]: pgmap v837: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 5089/64776 objects degraded (7.856%); 3077/64776 objects misplaced (4.750%); 0 B/s, 0 objects/s recovering 2023-09-23T15:21:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:36 smithi017 ceph-mon[215646]: pgmap v837: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 118 MiB/s wr, 1.89k op/s; 5089/64776 objects degraded (7.856%); 3077/64776 objects misplaced (4.750%); 0 B/s, 0 objects/s recovering 2023-09-23T15:21:36.725 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:37.022 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:37.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:37.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:37.125 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:37.174 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:37.316 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:21:37.316 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:28572 8.5 8:a34e7d28:::benchmark_data_smithi017_485823_object28571:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e292)", 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:37.311436+0000", 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0046719279999999997, 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047068170000000003, 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:21:37.317 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 28572 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311436+0000", 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311436+0000", 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311435+0000", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311533+0000", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7422000000000003e-05 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311546+0000", 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2977e-05 2023-09-23T15:21:37.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311552+0000", 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6880000000000002e-06 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311599+0000", 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6743999999999998e-05 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311685+0000", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6167000000000004e-05 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.311797+0000", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112044 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.312795+0000", 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00099735600000000002 2023-09-23T15:21:37.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:37.314987+0000", 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002192903 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 1 2023-09-23T15:21:37.322 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:21:37.375 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 28687 28671 111.9 67.75 0.0104356 0.00893177 2023-09-23T15:21:37.430 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:21:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2148214452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:37 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5089/64776 objects degraded (7.856%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:37.692 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:37.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2148214452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5089/64776 objects degraded (7.856%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:37.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2148214452' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:21:37.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5089/64776 objects degraded (7.856%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:37.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:37.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:37.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:37.843 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:38.109 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:21:38.381 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 29790 29774 109.361 68.9375 0.00419613 0.00913837 2023-09-23T15:21:38.410 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:38] "GET /metrics HTTP/1.1" 200 36364 "" "Prometheus/2.43.0" 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: pgmap v838: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.5 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s; 5967/73359 objects degraded (8.134%); 3077/73359 objects misplaced (4.194%) 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:38.411 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[215646]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T15:21:38.499 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:38.559 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:38.685 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:38.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:38.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: pgmap v838: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.5 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s; 5967/73359 objects degraded (8.134%); 3077/73359 objects misplaced (4.194%) 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:38.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:38 smithi017 ceph-mon[222878]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T15:21:38.782 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: pgmap v838: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.5 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 107 MiB/s wr, 1.72k op/s; 5967/73359 objects degraded (8.134%); 3077/73359 objects misplaced (4.194%) 2023-09-23T15:21:38.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:38.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: failed to encode map e293 with expected crc 2023-09-23T15:21:38.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:21:38.783 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:38 smithi027 ceph-mon[195233]: osdmap e293: 8 total, 7 up, 7 in 2023-09-23T15:21:38.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:38.833 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:38.891 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:38.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:38.996 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:39.142 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:39.377 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 31076 31060 107.752 80.375 0.00200491 0.00922312 2023-09-23T15:21:39.392 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:39.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:39.397 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:39.410 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:39.452 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:39.497 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:39.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:39.564 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:39.576 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:39.576 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:39.605 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.641 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.642 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.642 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:21:39.642 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:21:39.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:39 smithi017 ceph-mon[215646]: pgmap v840: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.6 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 6820/78477 objects degraded (8.690%); 3077/78477 objects misplaced (3.921%) 2023-09-23T15:21:39.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/114617895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:39.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:39 smithi017 ceph-mon[222878]: pgmap v840: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.6 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 6820/78477 objects degraded (8.690%); 3077/78477 objects misplaced (3.921%) 2023-09-23T15:21:39.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/114617895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:39.689 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:39.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:39.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:39.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:39.829 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:39 smithi027 ceph-mon[195233]: pgmap v840: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.6 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 98 MiB/s wr, 1.57k op/s; 6820/78477 objects degraded (8.690%); 3077/78477 objects misplaced (3.921%) 2023-09-23T15:21:39.829 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/114617895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:39.834 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:39.948 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:39.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:39.953 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:40.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:40.102 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:40.166 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:40.166 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:40.377 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 32245 32229 105.927 73.0625 0.00225005 0.00933858 2023-09-23T15:21:40.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:21:40.821 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:40.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:40.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:40.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:40.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:21:41.075 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:41.191 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:41.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:41.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:41.294 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:41.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:41.377 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:21:41.378146+0000 min lat: 0.00125293 max lat: 0.231236 avg lat: 0.00961424 2023-09-23T15:21:41.377 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:21:41.377 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 33310 33294 103.96 66.5625 0.00972412 0.00961424 2023-09-23T15:21:41.393 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:41.396 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:21:41.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:41.399 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:41.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:41.534 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:41.645 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:41.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:41.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:41.752 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:41.787 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:21:41.911 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:21:41.927 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:41 smithi027 ceph-mon[195233]: pgmap v841: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 7548/91803 objects degraded (8.222%); 3791/91803 objects misplaced (4.129%) 2023-09-23T15:21:41.946 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:41.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:41 smithi017 ceph-mon[215646]: pgmap v841: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 7548/91803 objects degraded (8.222%); 3791/91803 objects misplaced (4.129%) 2023-09-23T15:21:41.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:41 smithi017 ceph-mon[222878]: pgmap v841: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 1.9 GiB data, 7.4 GiB used, 618 GiB / 626 GiB avail; 81 MiB/s wr, 1.30k op/s; 7548/91803 objects degraded (8.222%); 3791/91803 objects misplaced (4.129%) 2023-09-23T15:21:41.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:41.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:42.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:42.081 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:42.130 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:42.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:42.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:42.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:42.309 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:42.382 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 34608 34592 102.848 81.125 0.01952 0.00971458 2023-09-23T15:21:42.415 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:42.483 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:42.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:42.592 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 7548/91803 objects degraded (8.222%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.730 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:42.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 7548/91803 objects degraded (8.222%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:42 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.861 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:42.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:42.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:42.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 7548/91803 objects degraded (8.222%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:42.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:42 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:42.973 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:43.009 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:43.071 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:43.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:43.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:43.123 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:43.123 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:43.175 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:43.250 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:43.371 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:43.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:43.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:43.388 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 35864 35848 101.725 78.5 0.00971635 0.009825 2023-09-23T15:21:43.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:43.510 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:43.554 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:43.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:43.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:43.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:43.729 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pgp_num to 21 2023-09-23T15:21:43.732 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:43.795 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:43.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: pgmap v842: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 8321/96348 objects degraded (8.636%); 3791/96348 objects misplaced (3.935%) 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3981568301' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:21:43.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: pgmap v842: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 8321/96348 objects degraded (8.636%); 3791/96348 objects misplaced (3.935%) 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3981568301' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:21:43.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:43.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:43.866 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:21:43.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:43.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:21:43.933 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:43.989 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: pgmap v842: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.0 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 80 MiB/s wr, 1.27k op/s; 8321/96348 objects degraded (8.636%); 3791/96348 objects misplaced (3.935%) 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3981568301' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:21:43.996 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:21:43.997 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:21:43.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:43.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:44.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:44.217 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:44.386 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 36914 36898 100.157 65.625 0.00681566 0.00997827 2023-09-23T15:21:44.827 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:44.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:44.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:44.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[222878]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T15:21:44.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T15:21:44.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: failed to encode map e294 with expected crc 2023-09-23T15:21:44.966 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:44 smithi017 ceph-mon[215646]: failed to encode map e294 with expected crc 2023-09-23T15:21:45.018 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: failed to encode map e294 with expected crc 2023-09-23T15:21:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T15:21:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: osdmap e294: 8 total, 7 up, 7 in 2023-09-23T15:21:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: failed to encode map e294 with expected crc 2023-09-23T15:21:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: failed to encode map e294 with expected crc 2023-09-23T15:21:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:44 smithi027 ceph-mon[195233]: failed to encode map e294 with expected crc 2023-09-23T15:21:45.386 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 38068 38052 98.99 72.125 0.0101595 0.0100961 2023-09-23T15:21:45.489 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:45.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:45.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:45.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:45.634 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:45.691 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:45.695 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:45.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:45.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:45.701 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:45.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:45.899 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:46.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:45 smithi017 ceph-mon[215646]: pgmap v844: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.18k op/s; 8321/101589 objects degraded (8.191%); 4482/101589 objects misplaced (4.412%) 2023-09-23T15:21:46.037 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:45 smithi017 ceph-mon[222878]: pgmap v844: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.18k op/s; 8321/101589 objects degraded (8.191%); 4482/101589 objects misplaced (4.412%) 2023-09-23T15:21:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:45 smithi027 ceph-mon[195233]: pgmap v844: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 73 MiB/s wr, 1.18k op/s; 8321/101589 objects degraded (8.191%); 4482/101589 objects misplaced (4.412%) 2023-09-23T15:21:46.159 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:46.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:46.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:46.269 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:46.346 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:46.386 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 39348 39332 98.2308 80 0.0111063 0.0101744 2023-09-23T15:21:46.770 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:21:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2317535115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:46 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2317535115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:46 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2317535115' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:46 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:21:47.388 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 40855 40839 98.0706 94.1875 0.010902 0.0101919 2023-09-23T15:21:47.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:47.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:47.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:47.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:47.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:48.015 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: pgmap v845: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 9035/109071 objects degraded (8.284%); 4482/109071 objects misplaced (4.109%) 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.019 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 9035/109071 objects degraded (8.284%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:48.020 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: pgmap v845: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 9035/109071 objects degraded (8.284%); 4482/109071 objects misplaced (4.109%) 2023-09-23T15:21:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 9035/109071 objects degraded (8.284%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: pgmap v845: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 69 MiB/s wr, 1.10k op/s; 9035/109071 objects degraded (8.284%); 4482/109071 objects misplaced (4.109%) 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: osdmap e295: 8 total, 7 up, 7 in 2023-09-23T15:21:48.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: failed to encode map e295 with expected crc 2023-09-23T15:21:48.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 9035/109071 objects degraded (8.284%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:48.121 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:48.151 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:48.210 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:48.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:48.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:48.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:48.375 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:48] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T15:21:48.375 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:48.389 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 41864 41848 96.7714 63.0625 0.00607284 0.0103294 2023-09-23T15:21:49.258 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:49.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:49.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:49.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:49.398 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 43163 43147 96.1919 81.1875 0.0107147 0.01039 2023-09-23T15:21:49.467 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:49.741 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:49.746 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:49.847 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:49.914 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:50.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:49 smithi017 ceph-mon[215646]: pgmap v847: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s; 9823/113706 objects degraded (8.639%); 4482/113706 objects misplaced (3.942%) 2023-09-23T15:21:50.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:49 smithi017 ceph-mon[222878]: pgmap v847: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s; 9823/113706 objects degraded (8.639%); 4482/113706 objects misplaced (3.942%) 2023-09-23T15:21:50.042 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:50.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:50.076 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:21:50.077 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:21:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:49 smithi027 ceph-mon[195233]: pgmap v847: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.3 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 57 MiB/s wr, 912 op/s; 9823/113706 objects degraded (8.639%); 4482/113706 objects misplaced (3.942%) 2023-09-23T15:21:50.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:50.222 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:50.317 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [6] live_osds: [0, 2, 3, 4, 5, 7, 1] 2023-09-23T15:21:50.317 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:21:50.317 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T15:21:50.317 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T15:21:50.395 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 15 44367 44352 95.4724 75.3125 0.0112134 0.0104671 2023-09-23T15:21:50.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:21:50.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:21:50.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:21:50.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:21:50.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:21:50.665 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:50.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:21:50.742 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:21:50.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:50.843 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/144721323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:50.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/144721323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/144721323' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:21:51.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:21:51.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 45252 45236 94.133 55.25 0.0191398 0.0106173 2023-09-23T15:21:51.546 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:51.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:51.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:51.717 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:51.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:51 smithi017 ceph-mon[215646]: pgmap v848: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 10603/127728 objects degraded (8.301%); 5271/127728 objects misplaced (4.127%) 2023-09-23T15:21:51.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:51 smithi017 ceph-mon[222878]: pgmap v848: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 10603/127728 objects degraded (8.301%); 5271/127728 objects misplaced (4.127%) 2023-09-23T15:21:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:51 smithi027 ceph-mon[195233]: pgmap v848: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 82 MiB/s wr, 1.31k op/s; 10603/127728 objects degraded (8.301%); 5271/127728 objects misplaced (4.127%) 2023-09-23T15:21:52.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:21:52.265 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:52.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:52.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:52.299 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:21:52.300 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:21:52.388 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:52.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 46395 46379 93.4014 71.4375 0.00524546 0.0107002 2023-09-23T15:21:52.418 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:52.444 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:52.463 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:52.470 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:52.479 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:52.579 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:52.647 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:52.768 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:52.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:52.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:52.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:52.904 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:52.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 10603/127728 objects degraded (8.301%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:52.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 10603/127728 objects degraded (8.301%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:52.962 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:52.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:53.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 10603/127728 objects degraded (8.301%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:53.167 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:53.321 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:53.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:53.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:53.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 47555 47539 92.7487 72.5 0.00898811 0.0107762 2023-09-23T15:21:53.430 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:53.460 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:53.514 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:53.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:53.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:53.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:53.648 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:53.700 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:21:53.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:53.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:21:53.805 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:53.838 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:53.896 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:53.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:54.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:54.035 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:54.090 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:54.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:54.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:54.196 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:21:54.218 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:54.269 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:54.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:21:54.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:54 smithi027 ceph-mon[195233]: pgmap v849: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 11296/132012 objects degraded (8.557%); 5271/132012 objects misplaced (3.993%) 2023-09-23T15:21:54.377 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:21:54.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:54 smithi017 ceph-mon[215646]: pgmap v849: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 11296/132012 objects degraded (8.557%); 5271/132012 objects misplaced (3.993%) 2023-09-23T15:21:54.381 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:54 smithi017 ceph-mon[222878]: pgmap v849: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 83 MiB/s wr, 1.33k op/s; 11296/132012 objects degraded (8.557%); 5271/132012 objects misplaced (3.993%) 2023-09-23T15:21:54.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:54.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:21:54.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 48551 48535 91.8252 62.25 0.00231626 0.0108779 2023-09-23T15:21:54.484 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:54.519 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:54.572 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:54.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:54.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:54.681 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:54.714 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:54.765 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:54.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:54.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:54.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:54.959 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:55.205 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:55.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:55.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:55.380 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:55.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 49715 49699 91.2645 72.75 0.0161921 0.0109503 2023-09-23T15:21:55.551 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:55.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:55.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:55.663 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:55.694 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:55.745 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:55.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:55.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:55.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:21:55.883 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:55.944 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:21:55.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:55.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:21:56.050 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:56.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:56.172 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.4. 2023-09-23T15:21:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:56 smithi027 ceph-mon[195233]: pgmap v850: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 11296/136935 objects degraded (8.249%); 5949/136935 objects misplaced (4.344%) 2023-09-23T15:21:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3031413686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:56.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:56 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:56.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 50612 50596 90.2595 56.0625 0.00334146 0.0110153 2023-09-23T15:21:56.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:56 smithi017 conmon[215622]: 2023-09-23T15:21:56.028+0000 7f618a1a9700 -1 mon.a@0(leader).osd e295 definitely_dead 0 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[215646]: pgmap v850: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 11296/136935 objects degraded (8.249%); 5949/136935 objects misplaced (4.344%) 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3031413686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[222878]: pgmap v850: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 11296/136935 objects degraded (8.249%); 5949/136935 objects misplaced (4.344%) 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3031413686' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:56.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:56 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:21:57.134 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:21:57.162 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.162 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:21:57.162 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: osdmap e296: 8 total, 7 up, 8 in 2023-09-23T15:21:57.162 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4047543619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 11296/136935 objects degraded (8.249%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[215646]: osdmap e297: 8 total, 7 up, 8 in 2023-09-23T15:21:57.163 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: osdmap e296: 8 total, 7 up, 8 in 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4047543619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 11296/136935 objects degraded (8.249%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:57.164 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:57.165 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:57 smithi017 ceph-mon[222878]: osdmap e297: 8 total, 7 up, 8 in 2023-09-23T15:21:57.182 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:57.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:57.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:57.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:21:57.314 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:57.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 50612 50596 87.7545 0 - 0.0110153 2023-09-23T15:21:57.460 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:21:57.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:57.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:21:57.565 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:57.594 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: osdmap e296: 8 total, 7 up, 8 in 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: failed to encode map e296 with expected crc 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4047543619' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 11296/136935 objects degraded (8.249%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:21:57.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:57 smithi027 ceph-mon[195233]: osdmap e297: 8 total, 7 up, 8 in 2023-09-23T15:21:57.597 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:57.648 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:57.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:57.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:57.669 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T15:21:57.754 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:21:57.783 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:57.830 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:21:57.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:57.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:21:57.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:57.963 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:58.007 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:58.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:58.067 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:21:58.112 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:21:58.159 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:21:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:21:58] "GET /metrics HTTP/1.1" 200 36365 "" "Prometheus/2.43.0" 2023-09-23T15:21:58.163 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:58.231 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:21:58.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:58.234 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:21:58.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: pgmap v852: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 12028/144186 objects degraded (8.342%); 5949/144186 objects misplaced (4.126%) 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:58 smithi027 ceph-mon[195233]: osdmap e298: 8 total, 7 up, 8 in 2023-09-23T15:21:58.381 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:58.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 50695 50679 85.5249 2.59375 0.0156619 0.0116807 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: pgmap v852: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 12028/144186 objects degraded (8.342%); 5949/144186 objects misplaced (4.126%) 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[215646]: osdmap e298: 8 total, 7 up, 8 in 2023-09-23T15:21:58.425 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: pgmap v852: 109 pgs: 14 active+clean+remapped, 9 active+undersized+degraded, 25 active+undersized, 61 active+clean; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 74 MiB/s wr, 1.18k op/s; 12028/144186 objects degraded (8.342%); 5949/144186 objects misplaced (4.126%) 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.426 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: failed to encode map e297 with expected crc 2023-09-23T15:21:58.427 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:58 smithi017 ceph-mon[222878]: osdmap e298: 8 total, 7 up, 8 in 2023-09-23T15:21:58.430 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:21:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:21:58.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:21:58.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:21:58.577 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:21:58.578 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:21:58.578 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:21:58.578 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:21:58.578 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:21:58.602 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:21:59.396 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 51757 51741 85.0212 66.375 0.0147458 0.0117564 2023-09-23T15:21:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[215646]: osdmap e299: 8 total, 7 up, 8 in 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:59.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:21:59 smithi017 ceph-mon[222878]: osdmap e299: 8 total, 7 up, 8 in 2023-09-23T15:21:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: failed to encode map e298 with expected crc 2023-09-23T15:21:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:21:59.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:21:59 smithi027 ceph-mon[195233]: osdmap e299: 8 total, 7 up, 8 in 2023-09-23T15:22:00.019 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:00.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:00.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:00.041 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:22:00.124 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:00.170 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:00.398 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 52710 52694 84.3655 59.5625 0.00984933 0.0118467 2023-09-23T15:22:00.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[222878]: pgmap v855: 109 pgs: 5 peering, 14 active+clean+remapped, 9 active+undersized+degraded, 22 active+undersized, 59 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 12631/147921 objects degraded (8.539%); 5949/147921 objects misplaced (4.022%) 2023-09-23T15:22:00.433 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[222878]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[222878]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[222878]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.434 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[222878]: osdmap e300: 8 total, 7 up, 8 in 2023-09-23T15:22:00.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[215646]: pgmap v855: 109 pgs: 5 peering, 14 active+clean+remapped, 9 active+undersized+degraded, 22 active+undersized, 59 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 12631/147921 objects degraded (8.539%); 5949/147921 objects misplaced (4.022%) 2023-09-23T15:22:00.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[215646]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.435 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[215646]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[215646]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.436 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:00 smithi017 ceph-mon[215646]: osdmap e300: 8 total, 7 up, 8 in 2023-09-23T15:22:00.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:22:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:00 smithi027 ceph-mon[195233]: pgmap v855: 109 pgs: 5 peering, 14 active+clean+remapped, 9 active+undersized+degraded, 22 active+undersized, 59 active+clean; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 55 MiB/s wr, 883 op/s; 12631/147921 objects degraded (8.539%); 5949/147921 objects misplaced (4.022%) 2023-09-23T15:22:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:00 smithi027 ceph-mon[195233]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:00 smithi027 ceph-mon[195233]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:00 smithi027 ceph-mon[195233]: failed to encode map e299 with expected crc 2023-09-23T15:22:00.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:00 smithi027 ceph-mon[195233]: osdmap e300: 8 total, 7 up, 8 in 2023-09-23T15:22:00.625 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:00.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:00.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:00.738 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:00.780 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:00.830 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:00.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:00.938 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:00.973 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:01.050 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:01.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:01.070 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:01.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:01.199 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:01.270 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:01.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:01.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:01 smithi027 ceph-mon[195233]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:01 smithi027 ceph-mon[195233]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:01 smithi027 ceph-mon[195233]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:01 smithi027 ceph-mon[195233]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:01.398 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:22:01.399037+0000 min lat: 0.00125293 max lat: 2.15996 avg lat: 0.0119358 2023-09-23T15:22:01.398 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:22:01.398 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 53661 53645 83.7427 59.4375 0.00908611 0.0119358 2023-09-23T15:22:01.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[222878]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[222878]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[222878]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[222878]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[215646]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.453 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[215646]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[215646]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.454 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:01 smithi017 ceph-mon[215646]: failed to encode map e300 with expected crc 2023-09-23T15:22:01.489 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:01.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:22:01.622 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:22:01.630 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:01.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:01.662 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:22:01.706 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:22:01.723 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:01.738 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:01.744 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:22:01.779 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:22:01.826 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:22:01.827 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:22:01.838 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:02.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 54645 54629 83.1962 61.5 0.0176531 0.012013 2023-09-23T15:22:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[215646]: pgmap v858: 109 pgs: 2 remapped+peering, 9 peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 58 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 10 KiB/s rd, 56 MiB/s wr, 903 op/s; 22197/154926 objects degraded (14.327%); 9529/154926 objects misplaced (6.151%); 484 KiB/s, 2 keys/s, 9 objects/s recovering 2023-09-23T15:22:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:22:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/730159912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22197/154926 objects degraded (14.327%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:02.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[222878]: pgmap v858: 109 pgs: 2 remapped+peering, 9 peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 58 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 10 KiB/s rd, 56 MiB/s wr, 903 op/s; 22197/154926 objects degraded (14.327%); 9529/154926 objects misplaced (6.151%); 484 KiB/s, 2 keys/s, 9 objects/s recovering 2023-09-23T15:22:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:22:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/730159912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22197/154926 objects degraded (14.327%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:02 smithi027 ceph-mon[195233]: pgmap v858: 109 pgs: 2 remapped+peering, 9 peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 10 active+undersized, 58 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 10 KiB/s rd, 56 MiB/s wr, 903 op/s; 22197/154926 objects degraded (14.327%); 9529/154926 objects misplaced (6.151%); 484 KiB/s, 2 keys/s, 9 objects/s recovering 2023-09-23T15:22:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:02 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:22:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/730159912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:02.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22197/154926 objects degraded (14.327%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:02.670 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6] live_osds: [0, 2, 3, 4, 5, 7, 1] 2023-09-23T15:22:02.670 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:22:02.671 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T15:22:02.671 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:22:02.671 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:22:02.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T15:22:03.224 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:22:03.400 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 55734 55718 82.836 68.0625 0.00398831 0.0120503 2023-09-23T15:22:04.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 56386 56370 81.858 40.75 0.0185499 0.0122111 2023-09-23T15:22:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:04 smithi027 ceph-mon[195233]: pgmap v859: 109 pgs: 2 remapped+peering, 4 peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 61 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 7.6 KiB/s rd, 36 MiB/s wr, 580 op/s; 22601/157461 objects degraded (14.353%); 9538/157461 objects misplaced (6.057%); 350 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T15:22:04.612 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:04.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:04.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:04.631 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:04 smithi017 ceph-mon[222878]: pgmap v859: 109 pgs: 2 remapped+peering, 4 peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 61 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 7.6 KiB/s rd, 36 MiB/s wr, 580 op/s; 22601/157461 objects degraded (14.353%); 9538/157461 objects misplaced (6.057%); 350 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T15:22:04.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:04 smithi017 ceph-mon[215646]: pgmap v859: 109 pgs: 2 remapped+peering, 4 peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 61 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 7.6 KiB/s rd, 36 MiB/s wr, 580 op/s; 22601/157461 objects degraded (14.353%); 9538/157461 objects misplaced (6.057%); 350 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T15:22:04.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:04.756 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:04.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:04.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:04.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:04.847 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:22:04.847 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:22:04.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:05.275 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:05.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 57445 57429 81.502 66.1875 0.00382931 0.0122639 2023-09-23T15:22:05.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:05 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:22:05.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:05 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:22:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:05 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T15:22:06.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 58268 58252 80.8343 51.4375 0.00833873 0.0123654 2023-09-23T15:22:06.556 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:06.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:06.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:06 smithi017 ceph-mon[215646]: pgmap v860: 109 pgs: 1 remapped+peering, 20 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6.4 KiB/s rd, 46 MiB/s wr, 734 op/s; 22601/162192 objects degraded (13.935%); 9893/162192 objects misplaced (6.100%); 2.9 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:22:06.564 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:06 smithi017 ceph-mon[222878]: pgmap v860: 109 pgs: 1 remapped+peering, 20 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6.4 KiB/s rd, 46 MiB/s wr, 734 op/s; 22601/162192 objects degraded (13.935%); 9893/162192 objects misplaced (6.100%); 2.9 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:22:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:06 smithi027 ceph-mon[195233]: pgmap v860: 109 pgs: 1 remapped+peering, 20 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 6.4 KiB/s rd, 46 MiB/s wr, 734 op/s; 22601/162192 objects degraded (13.935%); 9893/162192 objects misplaced (6.100%); 2.9 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:22:06.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:06.771 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:06.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:22:06.931 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:06.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:06.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:07.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:07.068 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:22:07.068 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "" 2023-09-23T15:22:07.068 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:22:07.087 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:07.310 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:07.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:07.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22601/162192 objects degraded (13.935%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:07.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 59192 59176 80.3327 57.75 0.00649178 0.0124429 2023-09-23T15:22:07.415 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:07.630 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22601/162192 objects degraded (13.935%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:07.630 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22601/162192 objects degraded (13.935%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:07.881 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:08.168 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:22:08.169 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:22:08.320 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:08] "GET /metrics HTTP/1.1" 200 36375 "" "Prometheus/2.43.0" 2023-09-23T15:22:08.329 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:22:08.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 59950 59934 79.6319 47.375 0.00429237 0.0125097 2023-09-23T15:22:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:08 smithi027 ceph-mon[195233]: pgmap v861: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5.2 KiB/s rd, 53 MiB/s wr, 849 op/s; 22894/168216 objects degraded (13.610%); 9893/168216 objects misplaced (5.881%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T15:22:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/337969624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[215646]: pgmap v861: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5.2 KiB/s rd, 53 MiB/s wr, 849 op/s; 22894/168216 objects degraded (13.610%); 9893/168216 objects misplaced (5.881%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T15:22:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/337969624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.659 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[222878]: pgmap v861: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.4 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 5.2 KiB/s rd, 53 MiB/s wr, 849 op/s; 22894/168216 objects degraded (13.610%); 9893/168216 objects misplaced (5.881%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T15:22:08.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/337969624' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.660 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:08.690 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:09.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:09.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:09.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:09.174 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:09.249 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:09.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:09.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:09.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:09.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 60941 60925 79.2634 61.9375 0.0212249 0.0126104 2023-09-23T15:22:09.412 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:09.474 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:09.479 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:09 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:09.479 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:09 smithi027 ceph-mon[195233]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.479 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:09 smithi027 ceph-mon[195233]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.479 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:09.479 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:09 smithi027 ceph-mon[195233]: osdmap e301: 8 total, 7 up, 8 in 2023-09-23T15:22:09.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:09.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:09.581 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:09.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[215646]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[215646]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[215646]: osdmap e301: 8 total, 7 up, 8 in 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[222878]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[222878]: failed to encode map e301 with expected crc 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:09.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:09 smithi017 ceph-mon[222878]: osdmap e301: 8 total, 7 up, 8 in 2023-09-23T15:22:09.897 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:10.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:22:10.401 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 61913 61897 78.8858 60.75 0.0141873 0.01267 2023-09-23T15:22:10.453 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:10.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:10.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:10.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:10.571 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:10 smithi027 ceph-mon[195233]: pgmap v863: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 23459/171627 objects degraded (13.669%); 9893/171627 objects misplaced (5.764%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:22:10.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:10 smithi017 ceph-mon[215646]: pgmap v863: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 23459/171627 objects degraded (13.669%); 9893/171627 objects misplaced (5.764%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:22:10.682 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:10 smithi017 ceph-mon[222878]: pgmap v863: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 45 MiB/s wr, 718 op/s; 23459/171627 objects degraded (13.669%); 9893/171627 objects misplaced (5.764%); 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:22:10.682 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:10.877 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:10.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:10.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:11.055 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:11.182 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:11.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:11.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:11.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:11.384 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:11.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 62811 62795 78.4274 56.125 0.0180537 0.0127431 2023-09-23T15:22:11.473 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:11.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:11.583 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:11.636 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:11.709 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:11.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:11.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:11.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:11.899 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:12.384 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:12.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:12.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:12.403 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 15 63607 63592 77.8666 49.8125 0.00892683 0.0128369 2023-09-23T15:22:12.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:22:12.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:12.595 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:12 smithi017 ceph-mon[215646]: pgmap v864: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 23721/181575 objects degraded (13.064%); 10244/181575 objects misplaced (5.642%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23721/181575 objects degraded (13.064%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:12.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:12 smithi017 ceph-mon[222878]: pgmap v864: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 23721/181575 objects degraded (13.064%); 10244/181575 objects misplaced (5.642%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:12.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23721/181575 objects degraded (13.064%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:12.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:12 smithi027 ceph-mon[195233]: pgmap v864: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 23721/181575 objects degraded (13.064%); 10244/181575 objects misplaced (5.642%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:12.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23721/181575 objects degraded (13.064%), 11 pgs degraded, 13 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:12.891 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:12.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:12.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:12.966 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6] live_osds: [0, 2, 3, 4, 5, 7, 1] 2023-09-23T15:22:12.967 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T15:22:12.967 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T15:22:12.967 DEBUG:teuthology.orchestra.run.smithi027:osd.6> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T15:22:12.969 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl start ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T15:22:13.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:13.009 INFO:journalctl@ceph.osd.6.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T15:22:13.044 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:13.091 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:13.351 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:13 smithi027 systemd[1]: Starting Ceph osd.6 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:22:13.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 64418 64402 77.3426 50.625 0.0257364 0.0129231 2023-09-23T15:22:13.792 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:13.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:13.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:13.914 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:13.964 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:14.035 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:14.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:14.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:14.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:14.166 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T15:22:14.226 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T15:22:14.227 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:14 smithi027 bash[311289]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T15:22:14.227 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:14 smithi027 ceph-mon[195233]: pgmap v865: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 24311/185037 objects degraded (13.138%); 10244/185037 objects misplaced (5.536%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:14.227 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:14.227 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:14.227 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3486171540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:14.229 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:22:14.230 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:14.286 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:22:14.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:22:14.386 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:22:14.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 65268 65252 76.8859 53.125 0.0130223 0.0130006 2023-09-23T15:22:14.428 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:22:14.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[215646]: pgmap v865: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 24311/185037 objects degraded (13.138%); 10244/185037 objects misplaced (5.536%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:14.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3486171540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[222878]: pgmap v865: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 24311/185037 objects degraded (13.138%); 10244/185037 objects misplaced (5.536%); 3.1 MiB/s, 50 objects/s recovering 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3486171540' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:14.475 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:22:14.476 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:22:14.498 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:14.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:14.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:14.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:14.663 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:15.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 65921 65905 76.2183 40.8125 0.0110606 0.0131151 2023-09-23T15:22:16.280 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:16.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:16.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:16.306 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:22:16.355 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:16.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:16.387 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:22:16.387 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:22:16.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:16 smithi017 ceph-mon[215646]: pgmap v866: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 24311/189261 objects degraded (12.845%); 10494/189261 objects misplaced (5.545%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:16.388 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:16 smithi017 ceph-mon[222878]: pgmap v866: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 24311/189261 objects degraded (12.845%); 10494/189261 objects misplaced (5.545%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:16.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 66934 66918 75.9837 63.3125 0.0122562 0.0131549 2023-09-23T15:22:16.437 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:16 smithi027 ceph-mon[195233]: pgmap v866: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 24311/189261 objects degraded (12.845%); 10494/189261 objects misplaced (5.545%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:16.439 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:16 smithi027 ceph-osd[311825]: -- 172.21.15.27:0/364510302 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5610fe726b60 con 0x5610fe6ecc00 2023-09-23T15:22:16.439 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:16 smithi027 bash[311763]: 1a318744f186dcb14734b7fc136197ab73a5c3997ba2e222a9643bb461db7a96 2023-09-23T15:22:16.439 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:16 smithi027 systemd[1]: Started Ceph osd.6 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:22:16.443 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:16.539 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:16.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:16.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:16.654 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:16.698 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:16.842 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:16.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:16.886 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:22:16.927 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:22:16.927 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:22:16.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.927 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(8.2 301/297 [PushReplyOp(8:4529951a:::benchmark_data_smithi017_485823_object14638:head)])", 2023-09-23T15:22:16.927 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:16.912831+0000", 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.002155508, 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021824209999999999, 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912831+0000", 2023-09-23T15:22:16.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912831+0000", 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912830+0000", 2023-09-23T15:22:16.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912836+0000", 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7769999999999999e-06 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912847+0000", 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1422e-05 2023-09-23T15:22:16.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.912855+0000", 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5719999999999997e-06 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:16.914865+0000", 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002009832 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:16.931 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:16.932 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:22:16.932 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 1 2023-09-23T15:22:16.932 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:22:16.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:16.996 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:16 smithi027 conmon[311800]: 2023-09-23T15:22:16.799+0000 7f98ceb40540 -1 Falling back to public interface 2023-09-23T15:22:16.999 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:17.058 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:17.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:17.061 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:17.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:17.254 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:17.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24311/189261 objects degraded (12.845%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:17.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 67843 67827 75.6415 56.8125 0.0157497 0.0132147 2023-09-23T15:22:17.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24311/189261 objects degraded (12.845%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:17.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24311/189261 objects degraded (12.845%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:18.371 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:18] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-23T15:22:18.373 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:18 smithi017 ceph-mon[215646]: pgmap v867: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 24583/194823 objects degraded (12.618%); 10494/194823 objects misplaced (5.386%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:18.373 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:18 smithi017 ceph-mon[222878]: pgmap v867: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 24583/194823 objects degraded (12.618%); 10494/194823 objects misplaced (5.386%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:18.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:22:18.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 68625 68609 75.1722 48.875 0.0174787 0.0132982 2023-09-23T15:22:18.560 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:18 smithi027 ceph-mon[195233]: pgmap v867: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 55 MiB/s wr, 885 op/s; 24583/194823 objects degraded (12.618%); 10494/194823 objects misplaced (5.386%); 3.3 MiB/s, 53 objects/s recovering 2023-09-23T15:22:18.724 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:18.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:18.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:18.828 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:0 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:1 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:2 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:config diff 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:config diff get 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:config get 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:config help [] 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:config set ... 2023-09-23T15:22:18.829 INFO:teuthology.orchestra.run.smithi027.stderr:admin_socket: invalid command 2023-09-23T15:22:18.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:18.913 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:19.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:22:19.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:19.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:19.065 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:19.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:19.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:19.341 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:19.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:19.358 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:19.404 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 69438 69422 74.7524 50.8125 0.0191674 0.0133716 2023-09-23T15:22:19.459 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:19.496 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:19.564 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:19.634 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:19.641 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T15:22:19.642 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T15:22:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:19.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:19.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:19.861 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:20.406 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 15 70291 70276 74.3876 53.375 0.0189268 0.0134377 2023-09-23T15:22:20.450 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:20.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:20.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:20.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:20 smithi027 ceph-mon[195233]: pgmap v868: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 25080/197790 objects degraded (12.680%); 10494/197790 objects misplaced (5.306%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:20.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:20 smithi017 ceph-mon[222878]: pgmap v868: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 25080/197790 objects degraded (12.680%); 10494/197790 objects misplaced (5.306%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:20.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:20 smithi017 ceph-mon[215646]: pgmap v868: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 843 op/s; 25080/197790 objects degraded (12.680%); 10494/197790 objects misplaced (5.306%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:20.640 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:20.685 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:20.814 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:20.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:20.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:20.922 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:21.003 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:21.235 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:22:21.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2003855552' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:21 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2003855552' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.263 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:21 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:22:21.407498+0000 min lat: 0.00125293 max lat: 2.15996 avg lat: 0.0135321 2023-09-23T15:22:21.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:22:21.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 70988 70972 73.8731 43.5 0.0129756 0.0135321 2023-09-23T15:22:21.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2003855552' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:21 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:22:21.674 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:21.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:21.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:21.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:21.817 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:21.882 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:21.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:21.888 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:21.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:22.075 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:22.383 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: pgmap v869: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 25350/207327 objects degraded (12.227%); 10749/207327 objects misplaced (5.185%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: osdmap e302: 8 total, 7 up, 8 in 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25350/207327 objects degraded (12.227%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:22.384 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: pgmap v869: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 25350/207327 objects degraded (12.227%); 10749/207327 objects misplaced (5.185%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: osdmap e302: 8 total, 7 up, 8 in 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25350/207327 objects degraded (12.227%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:22.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 71766 71750 73.4594 48.625 0.0102497 0.0136075 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: pgmap v869: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 25350/207327 objects degraded (12.227%); 10749/207327 objects misplaced (5.185%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: osdmap e302: 8 total, 7 up, 8 in 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: failed to encode map e302 with expected crc 2023-09-23T15:22:22.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25350/207327 objects degraded (12.227%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:22.764 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:22.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:22.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:22.933 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:23.400 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:23.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 72601 72585 73.1164 52.1875 0.0143813 0.0136714 2023-09-23T15:22:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:23.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:23.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:23.584 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:24.045 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:24.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:24.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:24.099 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:23 smithi027 conmon[311800]: 2023-09-23T15:22:23.597+0000 7f98ceb40540 -1 osd.6 280 log_to_monitors true 2023-09-23T15:22:24.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:24.271 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:24.391 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:22:24.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 73399 73383 72.7477 49.875 0.00749406 0.0137411 2023-09-23T15:22:24.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:22:24.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[215646]: pgmap v871: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 25840/210426 objects degraded (12.280%); 10749/210426 objects misplaced (5.108%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:22:24.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[215646]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[215646]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[222878]: pgmap v871: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 25840/210426 objects degraded (12.280%); 10749/210426 objects misplaced (5.108%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:22:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[222878]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:24 smithi017 ceph-mon[222878]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:22:24.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:22:24.545 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:24.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:24.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:24.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:22:24 smithi027 conmon[311800]: 2023-09-23T15:22:24.278+0000 7f98c0922700 -1 osd.6 280 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:22:24.602 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:22:24.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:24 smithi027 ceph-mon[195233]: pgmap v871: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovery_wait+undersized+remapped, 1 active+recovering+undersized+remapped, 10 active+undersized, 65 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 53 MiB/s wr, 845 op/s; 25840/210426 objects degraded (12.280%); 10749/210426 objects misplaced (5.108%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:22:24.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:24 smithi027 ceph-mon[195233]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:24 smithi027 ceph-mon[195233]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T15:22:24.643 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:22:24.644 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:22:24.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.684 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T15:22:24.684 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:22:24.732 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:24.901 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:24.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:24.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:24.960 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:22:24.960 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:22:25.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:25.094 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:25.149 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:22:25.193 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:25.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:25.299 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:25.346 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:25.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 74178 74162 72.3719 48.6875 0.00766815 0.0137945 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: osdmap e303: 8 total, 7 up, 8 in 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.597 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3748572339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: osdmap e303: 8 total, 7 up, 8 in 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3748572339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: osdmap e303: 8 total, 7 up, 8 in 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: from='osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: failed to encode map e303 with expected crc 2023-09-23T15:22:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3748572339' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:25.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:22:26.352 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:26.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:26.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 74178 74162 71.2591 0 - 0.0137945 2023-09-23T15:22:26.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:26.501 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:26.549 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:26.551 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: pgmap v873: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 25840/214017 objects degraded (12.074%); 10939/214017 objects misplaced (5.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:22:26.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:22:26.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: failed to encode map e304 with expected crc 2023-09-23T15:22:26.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963] boot 2023-09-23T15:22:26.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T15:22:26.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:22:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:26.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: pgmap v873: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 25840/214017 objects degraded (12.074%); 10939/214017 objects misplaced (5.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: failed to encode map e304 with expected crc 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963] boot 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: pgmap v873: 109 pgs: 21 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 10 active+undersized, 65 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 799 op/s; 25840/214017 objects degraded (12.074%); 10939/214017 objects misplaced (5.111%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:22:26.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:22:26.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: failed to encode map e304 with expected crc 2023-09-23T15:22:26.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: osd.6 [v2:172.21.15.27:6816/3452389963,v1:172.21.15.27:6817/3452389963] boot 2023-09-23T15:22:26.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T15:22:26.578 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T15:22:26.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:26.681 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:27.080 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:27.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:27.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:27.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:27.211 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e281 with expected crc 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e304 with expected crc 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T15:22:27.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25840/214017 objects degraded (12.074%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e281 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e304 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.357 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25840/214017 objects degraded (12.074%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:27.373 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:22:27.374 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:22:27.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 74178 74162 70.1802 0 - 0.0137945 2023-09-23T15:22:27.455 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:27.460 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:27.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:27.473 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:22:27.506 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:27.535 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:22:27.536 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:22:27.536 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:22:27.536 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:22:27.561 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:27.567 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e281 with expected crc 2023-09-23T15:22:27.567 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e304 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: failed to encode map e305 with expected crc 2023-09-23T15:22:27.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25840/214017 objects degraded (12.074%), 11 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T15:22:27.615 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:27.799 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:27.809 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:27.909 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:27.957 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:28.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:28.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:28.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:28.056 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:28] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-23T15:22:28.154 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:28.187 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: pgmap v876: 109 pgs: 4 peering, 2 remapped+peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 7 active+undersized, 65 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 26103/219237 objects degraded (11.906%); 10939/219237 objects misplaced (4.990%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:28.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:28.407 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 74758 74742 69.674 12.0833 0.0252579 0.0143459 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: pgmap v876: 109 pgs: 4 peering, 2 remapped+peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 7 active+undersized, 65 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 26103/219237 objects degraded (11.906%); 10939/219237 objects misplaced (4.990%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:28.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: pgmap v876: 109 pgs: 4 peering, 2 remapped+peering, 19 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+remapped, 7 active+undersized, 65 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 46 MiB/s wr, 729 op/s; 26103/219237 objects degraded (11.906%); 10939/219237 objects misplaced (4.990%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: failed to encode map e306 with expected crc 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:28.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:28.538 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:28.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:28.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:28.566 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:22:28.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:28.713 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:28.800 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:28.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:28.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:28.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:28.992 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:29.341 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:29.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:22:29.411 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 75561 75545 69.3852 50.1875 0.0271913 0.0144081 2023-09-23T15:22:29.420 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:29.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:29.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:29.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:29.598 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:29.620 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:29.864 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:29.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:29.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:29.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:30.037 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:30.141 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:30.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:30.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:30.317 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:30 smithi027 ceph-mon[195233]: pgmap v878: 109 pgs: 6 peering, 4 remapped+peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 67 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 17190/221319 objects degraded (7.767%); 10930/221319 objects misplaced (4.939%) 2023-09-23T15:22:30.368 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:30.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 76229 76213 68.9849 41.75 0.0301423 0.0144898 2023-09-23T15:22:30.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:30 smithi017 ceph-mon[215646]: pgmap v878: 109 pgs: 6 peering, 4 remapped+peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 67 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 17190/221319 objects degraded (7.767%); 10930/221319 objects misplaced (4.939%) 2023-09-23T15:22:30.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:30 smithi017 ceph-mon[222878]: pgmap v878: 109 pgs: 6 peering, 4 remapped+peering, 18 active+clean+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 5 active+undersized, 67 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 38 MiB/s wr, 606 op/s; 17190/221319 objects degraded (7.767%); 10930/221319 objects misplaced (4.939%) 2023-09-23T15:22:31.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 76999 76983 68.6871 48.125 0.00630975 0.0145367 2023-09-23T15:22:31.422 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:22:31.561 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:31.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:31.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:31.566 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:31.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:31.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:31.741 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:32.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 77776 77760 68.4037 48.5625 0.0311025 0.0146141 2023-09-23T15:22:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: pgmap v879: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 663 op/s; 26398/225999 objects degraded (11.681%); 1267/225999 objects misplaced (0.561%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T15:22:32.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T15:22:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26398/225999 objects degraded (11.681%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:32.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: pgmap v879: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 663 op/s; 26398/225999 objects degraded (11.681%); 1267/225999 objects misplaced (0.561%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T15:22:32.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26398/225999 objects degraded (11.681%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: pgmap v879: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 2 peering, 2 remapped+peering, 6 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 663 op/s; 26398/225999 objects degraded (11.681%); 1267/225999 objects misplaced (0.561%); 6.1 MiB/s, 97 objects/s recovering 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2060161102' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T15:22:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: failed to encode map e307 with expected crc 2023-09-23T15:22:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26398/225999 objects degraded (11.681%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:33.208 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:33.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:33.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:33.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 78559 78543 68.1334 48.9375 0.0175949 0.014672 2023-09-23T15:22:33.431 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:33.710 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:33.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:33.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:33.820 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:33.886 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:33.955 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:33.971 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:33.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:34.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:34.119 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:34.178 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:34.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:34.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:34.280 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:22:34.280 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:22:34.289 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:34.334 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:34.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 79285 79269 67.8218 45.375 0.0111768 0.0147384 2023-09-23T15:22:34.421 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:34.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:34.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:34.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:34 smithi017 ceph-mon[215646]: pgmap v881: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s; 35681/228357 objects degraded (15.625%); 1267/228357 objects misplaced (0.555%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T15:22:34.571 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:34 smithi017 ceph-mon[222878]: pgmap v881: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s; 35681/228357 objects degraded (15.625%); 1267/228357 objects misplaced (0.555%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T15:22:34.588 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:34 smithi027 ceph-mon[195233]: pgmap v881: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 30 MiB/s wr, 473 op/s; 35681/228357 objects degraded (15.625%); 1267/228357 objects misplaced (0.555%); 5.7 MiB/s, 91 objects/s recovering 2023-09-23T15:22:34.594 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:34.708 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:34.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:34.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:34.828 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:34.895 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:35.042 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:35.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:35.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:35.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:35.217 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:35.380 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:22:35.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:22:35.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 80088 80072 67.5836 50.1875 0.00468405 0.0147863 2023-09-23T15:22:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3013466853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:35.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:35 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3013466853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:35.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:35 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:35.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3013466853' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:35.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:35 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:22:36.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[215646]: pgmap v882: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 35681/232479 objects degraded (15.348%); 920/232479 objects misplaced (0.396%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:36.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[215646]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:22:36.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[215646]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T15:22:36.352 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[222878]: pgmap v882: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 35681/232479 objects degraded (15.348%); 920/232479 objects misplaced (0.396%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:36.352 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[222878]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.352 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:22:36.352 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[222878]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T15:22:36.352 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[222878]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.410 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 80732 80716 67.2193 40.25 0.0197784 0.0148708 2023-09-23T15:22:36.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:36 smithi017 ceph-mon[215646]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.713 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:36.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:36.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:36.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:36.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:22:36.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:36 smithi027 ceph-mon[195233]: pgmap v882: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 35681/232479 objects degraded (15.348%); 920/232479 objects misplaced (0.396%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:36.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:36 smithi027 ceph-mon[195233]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:36 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:22:36.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:36 smithi027 ceph-mon[195233]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T15:22:36.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:36 smithi027 ceph-mon[195233]: failed to encode map e308 with expected crc 2023-09-23T15:22:36.871 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:36.895 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:22:36.952 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:36.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:37.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:37.100 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:37.147 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:37.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:37.253 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:37.311 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:37.377 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 35681/232479 objects degraded (15.348%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:37.378 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:37.380 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 35681/232479 objects degraded (15.348%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:37.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:37.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:37.411 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 81460 81444 66.9326 45.5 0.0118398 0.0149352 2023-09-23T15:22:37.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:37.541 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 35681/232479 objects degraded (15.348%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:37.559 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:37.651 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:37.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:37.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:37.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:37.816 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:37.830 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:37.880 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:37.885 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:37.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:37.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:37.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:37.998 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:38.027 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:38] "GET /metrics HTTP/1.1" 200 36236 "" "Prometheus/2.43.0" 2023-09-23T15:22:38.071 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:38.381 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:38 smithi017 ceph-mon[215646]: pgmap v884: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 35681/236796 objects degraded (15.068%); 920/236796 objects misplaced (0.389%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:38.411 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 82184 82168 66.651 45.25 0.0190952 0.0149973 2023-09-23T15:22:38.426 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:38.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:38.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:38.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:38.579 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:38.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:22:38.630 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:22:38.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:38 smithi017 ceph-mon[222878]: pgmap v884: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 35681/236796 objects degraded (15.068%); 920/236796 objects misplaced (0.389%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:38.647 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:38.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:38.655 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:38 smithi027 ceph-mon[195233]: pgmap v884: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 40 MiB/s wr, 642 op/s; 35681/236796 objects degraded (15.068%); 920/236796 objects misplaced (0.389%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:22:38.674 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:22:38.727 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:22:38.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:38.771 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:22:38.812 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:38.821 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:22:38.857 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:22:38.898 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:38.909 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:22:38.912 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.912 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.912 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.913 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:22:38.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:38.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:39.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:39.081 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:39.169 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:39.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:39.179 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:39.280 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:39.341 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:39.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 82917 82901 66.384 45.8125 0.025471 0.0150573 2023-09-23T15:22:39.444 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:39.456 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1255213756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:39.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:39.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:39.624 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:39.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1255213756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:39.728 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:39.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:39.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:39.839 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:39.921 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:40.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1255213756' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:40.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 83658 83642 66.13 46.3125 0.0173642 0.015117 2023-09-23T15:22:40.549 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:40.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:40.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:40.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:40.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:40 smithi017 ceph-mon[215646]: pgmap v885: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 35681/239652 objects degraded (14.889%); 920/239652 objects misplaced (0.384%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:40.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:40 smithi017 ceph-mon[222878]: pgmap v885: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 35681/239652 objects degraded (14.889%); 920/239652 objects misplaced (0.384%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:40.753 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:40.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:40 smithi027 ceph-mon[195233]: pgmap v885: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 36 MiB/s wr, 568 op/s; 35681/239652 objects degraded (14.889%); 920/239652 objects misplaced (0.384%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:41.276 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:41.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:41.381 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:41.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:22:41.412825+0000 min lat: 0.00125293 max lat: 2.3363 avg lat: 0.0151573 2023-09-23T15:22:41.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:22:41.412 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 84465 84449 65.9339 50.4375 0.00517839 0.0151573 2023-09-23T15:22:41.421 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:41.505 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:41.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:41.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:41.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:41.638 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:42.376 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:42.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:42.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:42.413 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 85015 84999 65.5432 34.375 0.0122272 0.0152523 2023-09-23T15:22:42.483 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:42.564 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:42.673 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:42.681 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:42 smithi027 ceph-mon[195233]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 35681/247839 objects degraded (14.397%); 531/247839 objects misplaced (0.214%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T15:22:42.681 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 35681/247839 objects degraded (14.397%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:42.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:42.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:42.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:42 smithi017 ceph-mon[215646]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 35681/247839 objects degraded (14.397%); 531/247839 objects misplaced (0.214%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T15:22:42.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 35681/247839 objects degraded (14.397%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:42 smithi017 ceph-mon[222878]: pgmap v886: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 777 op/s; 35681/247839 objects degraded (14.397%); 531/247839 objects misplaced (0.214%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T15:22:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 35681/247839 objects degraded (14.397%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:42.783 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:42.831 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:42.901 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:42.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:42.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:43.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:43.052 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:43.149 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:43.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:43.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:43.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:43.375 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:43.414 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 85688 85672 65.2569 42.0625 0.0418256 0.0153168 2023-09-23T15:22:43.574 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:43.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:43.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:43.587 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:43.588 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:43.686 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:85845 8.1 e308/284 8:8ef04b0b:::benchmark_data_smithi017_485823_object85844:head v 308'5513, mlcod=308'5509)", 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:43.683492+0000", 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0069538009999999999, 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069867810000000001, 2023-09-23T15:22:43.693 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683492+0000", 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683492+0000", 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:43.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683491+0000", 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683519+0000", 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7610000000000002e-05 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683535+0000", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5964000000000002e-05 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683544+0000", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3370000000000006e-06 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683624+0000", 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9530999999999995e-05 2023-09-23T15:22:43.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.683647+0000", 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3255999999999999e-05 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:85841 8.e e308/297 8:750a4881:::benchmark_data_smithi017_485823_object85840:head v 308'5284, mlcod=0'0)", 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:43.689971+0000", 2023-09-23T15:22:43.697 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.00047488599999999999, 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000586328, 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.689971+0000", 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:22:43.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.689971+0000", 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.689970+0000", 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.689998+0000", 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7764999999999999e-05 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.690021+0000", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2699000000000001e-05 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.690030+0000", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8049999999999996e-06 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.690136+0000", 2023-09-23T15:22:43.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010606400000000001 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:43.690168+0000", 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2620000000000003e-05 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 2 2023-09-23T15:22:43.701 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:22:43.779 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:43.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:43.998 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:44.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:44.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:44.109 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:44.196 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:44.414 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 86244 86228 64.8895 34.75 0.176158 0.0153969 2023-09-23T15:22:44.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:22:44.499 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:22:44.500 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: pgmap v887: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 35681/250605 objects degraded (14.238%); 531/250605 objects misplaced (0.212%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4024850251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: pgmap v887: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 35681/250605 objects degraded (14.238%); 531/250605 objects misplaced (0.212%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4024850251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: pgmap v887: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 46 MiB/s wr, 740 op/s; 35681/250605 objects degraded (14.238%); 531/250605 objects misplaced (0.212%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4024850251' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.414 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 86780 86764 64.516 33.5 0.0139253 0.0154884 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.768 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: pgmap v889: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 35681/254424 objects degraded (14.024%); 126/254424 objects misplaced (0.050%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:22:45.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.769 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: pgmap v889: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 35681/254424 objects degraded (14.024%); 126/254424 objects misplaced (0.050%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:22:45.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.772 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:45 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.794 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:22:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:22:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3965719554' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:22:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:22:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: failed to encode map e309 with expected crc 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: pgmap v889: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 35681/254424 objects degraded (14.024%); 126/254424 objects misplaced (0.050%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:45 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:46.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:22:46.124 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:46.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:46.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:46.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:46.339 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:46.415 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 87414 87398 64.2224 39.625 0.0693514 0.0155627 2023-09-23T15:22:46.549 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:46.575 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:46.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:46.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:46.694 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:46.718 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:46.775 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:46 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:22:46.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:46 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:22:46.776 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:46.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:46 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:22:46.950 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:46.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:46.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:47.056 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:47.104 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:47.319 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:47.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:47.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:47.415 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 87704 87688 63.6867 18.125 0.00455453 0.0155788 2023-09-23T15:22:47.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:47.456 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:47.602 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:47.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:47.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: pgmap v891: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 35681/257820 objects degraded (13.840%); 126/257820 objects misplaced (0.049%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 35681/254424 objects degraded (14.024%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:47.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T15:22:47.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: pgmap v891: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 35681/257820 objects degraded (13.840%); 126/257820 objects misplaced (0.049%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:22:47.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 35681/254424 objects degraded (14.024%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:47.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:47.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:47.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T15:22:47.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: failed to encode map e310 with expected crc 2023-09-23T15:22:47.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: pgmap v891: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.2 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s; 35681/257820 objects degraded (13.840%); 126/257820 objects misplaced (0.049%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:22:47.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 35681/254424 objects degraded (14.024%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:48.029 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:48] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-23T15:22:48.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:22:48.415 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 88310 88294 63.3901 37.875 0.00994485 0.015765 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:48.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:48 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:48.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T15:22:48.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: failed to encode map e311 with expected crc 2023-09-23T15:22:48.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:48.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:48 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:49.265 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:49.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:49.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:49.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:49.397 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:49.415 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 88939 88923 63.1166 39.3125 0.00917986 0.0158161 2023-09-23T15:22:49.569 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:49.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:49 smithi017 ceph-mon[215646]: pgmap v893: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s; 35681/260130 objects degraded (13.717%); 126/260130 objects misplaced (0.048%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:49.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:49 smithi017 ceph-mon[222878]: pgmap v893: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s; 35681/260130 objects degraded (13.717%); 126/260130 objects misplaced (0.048%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:49.734 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:49.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:49.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:49.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:49 smithi027 ceph-mon[195233]: pgmap v893: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 33 MiB/s wr, 528 op/s; 35681/260130 objects degraded (13.717%); 126/260130 objects misplaced (0.048%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T15:22:49.888 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:49.974 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:49.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:50.084 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:50.218 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:50.427 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 89600 89584 62.8674 41.3125 0.020063 0.015899 2023-09-23T15:22:50.491 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:50.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:50.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:50.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:22:50.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:50.628 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:22:50.638 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:50.647 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:50.668 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:22:50.720 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:50.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:50.729 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:22:50.769 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:22:50.809 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:22:50.826 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:50 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:22:50.851 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:22:50.867 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:50.895 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.900 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:22:50.928 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:50 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:22:50.929 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:50 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:22:50.963 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:50.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:50.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:51.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:51.096 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:51.139 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:51.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:51.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:51.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:51.335 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:51.421 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0146 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 90134 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 62.5829 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 34.0796 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 192.688 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1001 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 545.29 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3083 2023-09-23T15:22:51.422 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2023-09-23T15:22:51.423 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0159741 2023-09-23T15:22:51.423 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0462835 2023-09-23T15:22:51.423 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 2.3363 2023-09-23T15:22:51.423 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00125293 2023-09-23T15:22:51.426 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:22:51.489 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:22:51.489 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:22:51.538 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:51 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:22:51.538 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/504212304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:51.538 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:51 smithi027 ceph-mon[195233]: pgmap v894: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 32572/266385 objects degraded (12.227%); 2889/266385 objects misplaced (1.085%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:51.570 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:51.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:22:51.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/504212304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:51.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[215646]: pgmap v894: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 32572/266385 objects degraded (12.227%); 2889/266385 objects misplaced (1.085%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:51.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:22:51.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/504212304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:22:51.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:51 smithi017 ceph-mon[222878]: pgmap v894: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 40 MiB/s wr, 640 op/s; 32572/266385 objects degraded (12.227%); 2889/266385 objects misplaced (1.085%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:51.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:51.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:51.727 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:51.840 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:51.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:51.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:51.947 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:51.994 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:52.114 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:52.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:52.224 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:52.292 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:52.436 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:52.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:52.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:52.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:52.600 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:52.685 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:52.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:52.693 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:52.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:52.842 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:52 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:22:52.842 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 32572/266385 objects degraded (12.227%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:52.857 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:52.869 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:52 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:22:52.870 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 32572/266385 objects degraded (12.227%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:52.870 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:52 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:22:52.870 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 32572/266385 objects degraded (12.227%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:52.937 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:52.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:52.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:53.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:53.100 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:53.164 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:53.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:53.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:53.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:53.340 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:53.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:53 smithi017 ceph-mon[222878]: pgmap v895: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 586 op/s; 32572/267996 objects degraded (12.154%); 2889/267996 objects misplaced (1.078%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:53.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:53 smithi017 ceph-mon[215646]: pgmap v895: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 586 op/s; 32572/267996 objects degraded (12.154%); 2889/267996 objects misplaced (1.078%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:53.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:53.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:53.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:53.801 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:22:53.856 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:53 smithi027 ceph-mon[195233]: pgmap v895: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 36 MiB/s wr, 586 op/s; 32572/267996 objects degraded (12.154%); 2889/267996 objects misplaced (1.078%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T15:22:53.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:53.964 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:54.329 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:54.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:54.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:54.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:54.505 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:54.582 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:54.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:54.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:54.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:54 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:22:54.702 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:54.752 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:54.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:54 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:22:54.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:54 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:22:55.328 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:55.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:55.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:22:55.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:22:55.535 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:55.699 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:22:55.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:55.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:22:55.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:22:55.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:22:55.866 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:55.931 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:22:55.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:55.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:22:56.009 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:55 smithi017 ceph-mon[222878]: pgmap v896: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 577 op/s; 32352/267390 objects degraded (12.099%); 2889/267390 objects misplaced (1.080%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:56.009 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:55 smithi017 ceph-mon[215646]: pgmap v896: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 577 op/s; 32352/267390 objects degraded (12.099%); 2889/267390 objects misplaced (1.080%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:56.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:56.094 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:55 smithi027 ceph-mon[195233]: pgmap v896: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 30 MiB/s wr, 577 op/s; 32352/267390 objects degraded (12.099%); 2889/267390 objects misplaced (1.080%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:22:56.097 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:56.213 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:56.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:56.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:56.320 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:56.403 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:56.505 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:56.516 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:56.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:22:56.658 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:56.772 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:22:56.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:56.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:22:56.804 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:22:56.892 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:56.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2096754305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.940 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:56 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2096754305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.940 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:56 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.963 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2096754305' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.963 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:56 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:22:56.966 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:57.058 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:57.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:57.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:57.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:22:57.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:57.344 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:22:57.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:57.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:22:57.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:22:57.518 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:57.779 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:22:57.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: pgmap v897: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 583 op/s; 32352/265656 objects degraded (12.178%); 2678/265656 objects misplaced (1.008%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 32352/265656 objects degraded (12.178%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: pgmap v897: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 583 op/s; 32352/265656 objects degraded (12.178%); 2678/265656 objects misplaced (1.008%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:22:57.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: failed to encode map e312 with expected crc 2023-09-23T15:22:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 32352/265656 objects degraded (12.178%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:58.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: pgmap v897: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 583 op/s; 32352/265656 objects degraded (12.178%); 2678/265656 objects misplaced (1.008%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:22:58.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:22:58.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: failed to encode map e312 with expected crc 2023-09-23T15:22:58.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:22:58.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T15:22:58.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: failed to encode map e312 with expected crc 2023-09-23T15:22:58.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: failed to encode map e312 with expected crc 2023-09-23T15:22:58.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 32352/265656 objects degraded (12.178%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:22:58.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:22:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:22:58] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.43.0" 2023-09-23T15:22:58.528 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:22:58.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:58.533 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:22:58.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:22:58.724 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:58.824 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:22:58.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:58.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:58.915 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: failed to encode map e313 with expected crc 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:58.916 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:22:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:58.932 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:58.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:22:59.002 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:22:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: failed to encode map e313 with expected crc 2023-09-23T15:22:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: failed to encode map e313 with expected crc 2023-09-23T15:22:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2156086380' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:22:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T15:22:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: failed to encode map e313 with expected crc 2023-09-23T15:22:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:22:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:22:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:22:59.123 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:22:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:59.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:22:59.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:22:59.300 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:22:59.795 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:22:59.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:22:59.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:22:59.912 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:22:59.950 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:00.054 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:00.196 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:00 smithi017 ceph-mon[215646]: pgmap v900: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 469 op/s; 32352/263310 objects degraded (12.287%); 2678/263310 objects misplaced (1.017%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T15:23:00.321 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:00.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:00.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:00.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:00.473 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:00 smithi017 ceph-mon[222878]: pgmap v900: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 469 op/s; 32352/263310 objects degraded (12.287%); 2678/263310 objects misplaced (1.017%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T15:23:00.481 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:00.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:00 smithi027 ceph-mon[195233]: pgmap v900: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 11 MiB/s wr, 469 op/s; 32352/263310 objects degraded (12.287%); 2678/263310 objects misplaced (1.017%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T15:23:00.773 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:00.773 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:01.255 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:01.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:01.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:01.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:01.411 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:01.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:23:02.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:02 smithi017 ceph-mon[215646]: pgmap v901: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 664 op/s; 31982/256527 objects degraded (12.467%); 2432/256527 objects misplaced (0.948%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T15:23:02.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 31982/256527 objects degraded (12.467%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:02.447 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:02 smithi017 ceph-mon[222878]: pgmap v901: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 664 op/s; 31982/256527 objects degraded (12.467%); 2432/256527 objects misplaced (0.948%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T15:23:02.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 31982/256527 objects degraded (12.467%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:02.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:02 smithi027 ceph-mon[195233]: pgmap v901: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 5.8 MiB/s wr, 664 op/s; 31982/256527 objects degraded (12.467%); 2432/256527 objects misplaced (0.948%); 5.9 MiB/s, 93 objects/s recovering 2023-09-23T15:23:02.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 31982/256527 objects degraded (12.467%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:02.775 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:02.840 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:23:02.868 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:02.880 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:02.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:03.034 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:03.096 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:03.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:03.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:03.200 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:03.257 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:03.322 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:03.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:03.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:03.337 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:03 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:03.436 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:03.497 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:03 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:03.498 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:03 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:03.509 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:03.603 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:03.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:03.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:03.665 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:03.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:03.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:04.303 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:04.330 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:04.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:04.504 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:04.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:04.566 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:04.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:04 smithi027 ceph-mon[195233]: pgmap v902: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 617 op/s; 31982/254118 objects degraded (12.585%); 2432/254118 objects misplaced (0.957%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T15:23:04.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.607 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:04.653 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:04.664 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.664 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.665 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[215646]: pgmap v902: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 617 op/s; 31982/254118 objects degraded (12.585%); 2432/254118 objects misplaced (0.957%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[222878]: pgmap v902: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 2.0 MiB/s wr, 617 op/s; 31982/254118 objects degraded (12.585%); 2432/254118 objects misplaced (0.957%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:04.937 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:04.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:04.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:05.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:05.109 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:05.199 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:05.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:05.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:05.311 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:05.387 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2792861276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:05 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:05 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:23:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:05 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:23:05.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:05 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:05.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2792861276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:05.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:05.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2792861276' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:23:05.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:05 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:06.224 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:06.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:06.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:06.337 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:06.369 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:06 smithi017 ceph-mon[215646]: pgmap v903: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 31614/250821 objects degraded (12.604%); 2432/250821 objects misplaced (0.970%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:23:06.369 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:06 smithi017 ceph-mon[222878]: pgmap v903: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 31614/250821 objects degraded (12.604%); 2432/250821 objects misplaced (0.970%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:23:06.372 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:06.453 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:06.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:06.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:06.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:06 smithi027 ceph-mon[195233]: pgmap v903: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.1 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 617 op/s; 31614/250821 objects degraded (12.604%); 2432/250821 objects misplaced (0.970%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:23:06.614 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:06.665 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:06.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:06.772 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:06.848 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:07.240 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:07.249 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:07.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:07.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:07.354 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:07.420 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:07 smithi017 ceph-mon[215646]: 1.6 deep-scrub starts 2023-09-23T15:23:07.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 31614/250821 objects degraded (12.604%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:07 smithi017 ceph-mon[222878]: 1.6 deep-scrub starts 2023-09-23T15:23:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 31614/250821 objects degraded (12.604%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:07.587 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:07.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:07.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:07 smithi027 ceph-mon[195233]: 1.6 deep-scrub starts 2023-09-23T15:23:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 31614/250821 objects degraded (12.604%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:07.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:07.755 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:07.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:07.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:07.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:07.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:08.038 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:08.077 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:23:08.123 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:08.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:08.152 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:08] "GET /metrics HTTP/1.1" 200 36407 "" "Prometheus/2.43.0" 2023-09-23T15:23:08.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:08.268 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:08.337 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:08.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:08.344 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:23:08.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[222878]: pgmap v904: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 31614/247356 objects degraded (12.781%); 2202/247356 objects misplaced (0.890%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:23:08.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1356299527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[215646]: pgmap v904: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 31614/247356 objects degraded (12.781%); 2202/247356 objects misplaced (0.890%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:23:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1356299527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:08 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:08.498 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:08.595 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:08 smithi027 ceph-mon[195233]: pgmap v904: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 613 op/s; 31614/247356 objects degraded (12.781%); 2202/247356 objects misplaced (0.890%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:23:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1356299527' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:08 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:23:08.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:08.601 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:08.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:08.777 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:08.834 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:08.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:08.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:08.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:08.992 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:09.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:23:09.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T15:23:09.324 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.325 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:09.338 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:23:09.462 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:09.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:09.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:09.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.607 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:23:09.607 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T15:23:09.607 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.607 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:09.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.614 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:23:09.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T15:23:09.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: failed to encode map e314 with expected crc 2023-09-23T15:23:09.615 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:09.620 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:09.815 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:09.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:09.863 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:23:09.934 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:09.977 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:10.069 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:10.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:10.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:10.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:10.236 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: pgmap v906: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 31614/245076 objects degraded (12.900%); 2202/245076 objects misplaced (0.898%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T15:23:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:10 smithi027 ceph-mon[195233]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.659 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:10.660 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: pgmap v906: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 31614/245076 objects degraded (12.900%); 2202/245076 objects misplaced (0.898%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[215646]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:10.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:10.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: pgmap v906: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 5.0 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 607 op/s; 31614/245076 objects degraded (12.900%); 2202/245076 objects misplaced (0.898%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:10.663 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:10.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3500283900' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:10.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T15:23:10.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:10 smithi017 ceph-mon[222878]: failed to encode map e315 with expected crc 2023-09-23T15:23:10.762 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:10.816 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:10.910 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:10.925 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:10.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:11.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:11.087 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:11.186 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:11.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:11.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:102290 8.b 8:d578408c:::benchmark_data_smithi017_485823_object12153:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:11.179752+0000", 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.018821431999999999, 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.018861155000000001, 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:11.204 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 102290 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179752+0000", 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179752+0000", 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179751+0000", 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179754+0000", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.145e-06 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179762+0000", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2690000000000002e-06 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.179767+0000", 2023-09-23T15:23:11.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4790000000000004e-06 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.181901+0000", 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021335540000000002 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.182050+0000", 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000149188 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.182200+0000", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015066100000000001 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.190519+0000", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083189830000000003 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192265+0000", 2023-09-23T15:23:11.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001745658 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192309+0000", 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4218000000000003e-05 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192366+0000", 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6681000000000002e-05 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.210 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45809.0:102290 8.b e315/297 ondisk, result = 0)", 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:11.186455+0000", 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012118088000000001, 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012264628, 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:11.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186455+0000", 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186455+0000", 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186455+0000", 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186457+0000", 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5380000000000001e-06 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186463+0000", 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4419999999999997e-06 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:11.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.186467+0000", 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2130000000000002e-06 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192293+0000", 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058264980000000003 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192302+0000", 2023-09-23T15:23:11.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9460000000000004e-06 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:102297 8.0 8:0d04857e:::benchmark_data_smithi017_485823_object12160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e315)", 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:11.192418+0000", 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.006154601, 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063666030000000002, 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:23:11.215 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 102297 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192418+0000", 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192418+0000", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192417+0000", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192424+0000", 2023-09-23T15:23:11.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.542e-06 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192439+0000", 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4628e-05 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192448+0000", 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5550000000000008e-06 2023-09-23T15:23:11.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192495+0000", 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7080000000000003e-05 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192609+0000", 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113618 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.192711+0000", 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102315 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.196767+0000", 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040551149999999998 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "MOSDPGPushReply(8.b 315/297 [PushReplyOp(8:d10b9998:::benchmark_data_smithi017_485823_object19919:head)])", 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:11.193698+0000", 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0048748660000000003, 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051675280000000002, 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193698+0000", 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193698+0000", 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193697+0000", 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193705+0000", 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193717+0000", 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2459999999999999e-05 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:11.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:11.193727+0000", 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4809999999999995e-06 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 4 2023-09-23T15:23:11.224 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:23:11.225 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:11.300 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:11.415 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:11.507 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:11.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:11.515 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:11.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:11.679 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:12.206 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:12.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:12.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:12.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:12.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:23:12.408 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:12.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:12 smithi017 ceph-mon[215646]: pgmap v908: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 31210/238008 objects degraded (13.113%); 1928/238008 objects misplaced (0.810%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T15:23:12.569 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 31210/238008 objects degraded (13.113%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:12.569 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:12 smithi017 ceph-mon[222878]: pgmap v908: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 31210/238008 objects degraded (13.113%); 1928/238008 objects misplaced (0.810%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T15:23:12.569 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 31210/238008 objects degraded (13.113%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:12.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:12 smithi027 ceph-mon[195233]: pgmap v908: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 671 op/s; 31210/238008 objects degraded (13.113%); 1928/238008 objects misplaced (0.810%); 7.0 MiB/s, 111 objects/s recovering 2023-09-23T15:23:12.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 31210/238008 objects degraded (13.113%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:12.666 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:12.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:12.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:12.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:12.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:23:12.821 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:12.960 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:12.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:12.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:13.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:13.171 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:13.440 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:13.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:13.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:13.554 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:13.639 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:13.697 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:13.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:13.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:13.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:13.849 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:13.977 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:13.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:13.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:14.092 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:14.160 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:14.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:14.270 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:14.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:14.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:14.280 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:14 smithi027 ceph-mon[195233]: pgmap v909: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 31210/235890 objects degraded (13.231%); 1928/235890 objects misplaced (0.817%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T15:23:14.281 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:14.281 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:14.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:14.410 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[222878]: pgmap v909: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 31210/235890 objects degraded (13.231%); 1928/235890 objects misplaced (0.817%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T15:23:14.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:14.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:14.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[215646]: pgmap v909: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 31210/235890 objects degraded (13.231%); 1928/235890 objects misplaced (0.817%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T15:23:14.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:14.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:14.486 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:14.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:14.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:14.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:14.682 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:14.736 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:14.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:14.932 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:14.976 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:14.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:14.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:14.998 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:15.045 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:15.091 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:15.098 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:15.127 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:15.168 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:15.200 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:15.213 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:15.254 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.255 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:23:15.327 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:15.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:15.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:15.433 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:15.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/467141996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:15.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/467141996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:15.506 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/467141996' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:15.600 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:15.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:15.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:15.705 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:15.737 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:16.261 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:23:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:16 smithi027 ceph-mon[195233]: pgmap v910: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 30822/232395 objects degraded (13.263%); 1928/232395 objects misplaced (0.830%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4195058272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:16.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[215646]: pgmap v910: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 30822/232395 objects degraded (13.263%); 1928/232395 objects misplaced (0.830%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:16.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4195058272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:16.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:16.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[222878]: pgmap v910: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 30822/232395 objects degraded (13.263%); 1928/232395 objects misplaced (0.830%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:16.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4195058272' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:16.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:23:17.040 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:17.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:17.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:17.155 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:17.200 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[215646]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[215646]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[215646]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 30822/232395 objects degraded (13.263%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:17.401 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[222878]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:23:17.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[222878]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T15:23:17.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[222878]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.402 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 30822/232395 objects degraded (13.263%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:17 smithi027 ceph-mon[195233]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:23:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:17 smithi027 ceph-mon[195233]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T15:23:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:17 smithi027 ceph-mon[195233]: failed to encode map e316 with expected crc 2023-09-23T15:23:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 30822/232395 objects degraded (13.263%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:17.785 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:17.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:17.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:17.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:17.952 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:18.045 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:18.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:18.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:18.070 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:18] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-09-23T15:23:18.071 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:23:18.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:18.261 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:18 smithi027 ceph-mon[195233]: pgmap v912: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 30822/228801 objects degraded (13.471%); 1665/228801 objects misplaced (0.728%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:18.282 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:18.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:18 smithi017 ceph-mon[215646]: pgmap v912: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 30822/228801 objects degraded (13.471%); 1665/228801 objects misplaced (0.728%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:18.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:18 smithi017 ceph-mon[222878]: pgmap v912: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 30822/228801 objects degraded (13.471%); 1665/228801 objects misplaced (0.728%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:19.079 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:19.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:19.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:19.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:19.243 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:19.319 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:19.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:19.432 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:19.490 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:19.493 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:19.575 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:19.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:19.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:19.653 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:19.653 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:19.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:19.785 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:20.185 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:20.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:20.272 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:23:20.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:20.323 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:20.372 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:20.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:20.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:20 smithi017 ceph-mon[215646]: pgmap v913: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 30822/226470 objects degraded (13.610%); 1665/226470 objects misplaced (0.735%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:23:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:20 smithi017 ceph-mon[222878]: pgmap v913: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 30822/226470 objects degraded (13.610%); 1665/226470 objects misplaced (0.735%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:23:20.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:20.545 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:20.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:20 smithi027 ceph-mon[195233]: pgmap v913: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 579 op/s; 30822/226470 objects degraded (13.610%); 1665/226470 objects misplaced (0.735%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T15:23:20.896 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:20.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:20.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:21.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:21.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:23:21.099 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:21.371 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:23:21.411 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:21.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2169911239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:21.413 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:21 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:21.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2169911239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:21.415 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:21 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:21.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:21.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:21.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:21.592 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2169911239' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:21.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:21 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:22.464 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:22.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:22.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:22.569 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:22.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: pgmap v914: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 30467/219645 objects degraded (13.871%); 1352/219645 objects misplaced (0.616%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:22.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:22.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.595 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:22.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T15:23:22.596 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 30467/219645 objects degraded (13.871%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:22.598 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:22.641 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:22.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:22.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:22.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: pgmap v914: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 30467/219645 objects degraded (13.871%); 1352/219645 objects misplaced (0.616%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:22.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 30467/219645 objects degraded (13.871%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: pgmap v914: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 611 op/s; 30467/219645 objects degraded (13.871%); 1352/219645 objects misplaced (0.616%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: failed to encode map e317 with expected crc 2023-09-23T15:23:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T15:23:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 30467/219645 objects degraded (13.871%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:22.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:22.814 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:23.143 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:23.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:23.154 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:23.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:23.255 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:23.338 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:23.646 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:23.701 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:23.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:23.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:23.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:23.912 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:24.114 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:24.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:24.220 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:24.287 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:24.376 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[222878]: pgmap v916: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 30467/217380 objects degraded (14.016%); 1352/217380 objects misplaced (0.622%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2657355023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[215646]: pgmap v916: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 30467/217380 objects degraded (14.016%); 1352/217380 objects misplaced (0.622%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2657355023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:24.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:24 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:24 smithi027 ceph-mon[195233]: pgmap v916: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 625 op/s; 30467/217380 objects degraded (14.016%); 1352/217380 objects misplaced (0.622%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:23:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2657355023' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:24 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:23:25.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:23:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[215646]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[215646]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:23:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[215646]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T15:23:25.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[215646]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[222878]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[222878]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:23:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[222878]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T15:23:25.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:25 smithi017 ceph-mon[222878]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:25 smithi027 ceph-mon[195233]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:25 smithi027 ceph-mon[195233]: failed to encode map e318 with expected crc 2023-09-23T15:23:25.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:23:25.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:25 smithi027 ceph-mon[195233]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T15:23:25.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:25 smithi027 ceph-mon[195233]: failed to encode map e318 with expected crc 2023-09-23T15:23:26.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:23:26.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:26 smithi017 ceph-mon[215646]: pgmap v918: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 30132/213912 objects degraded (14.086%); 1352/213912 objects misplaced (0.632%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T15:23:26.350 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:26 smithi017 ceph-mon[222878]: pgmap v918: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 30132/213912 objects degraded (14.086%); 1352/213912 objects misplaced (0.632%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T15:23:26.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:26 smithi027 ceph-mon[195233]: pgmap v918: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 620 op/s; 30132/213912 objects degraded (14.086%); 1352/213912 objects misplaced (0.632%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T15:23:26.798 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:26.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:26.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:26.904 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:26.959 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:27.076 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:27.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:27.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:27.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:27.238 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:27.242 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:27.304 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:27.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:27.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:27.402 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:27.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:27.453 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:27.495 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:27.534 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 30132/213912 objects degraded (14.086%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:27.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 30132/213912 objects degraded (14.086%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:27.535 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:27.537 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:27.567 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:27.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 30132/213912 objects degraded (14.086%), 9 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:27.604 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:27.637 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:27.699 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:27.743 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.746 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:23:27.747 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165850 7.8 e271/261 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head v 271'10159, mlcod=271'10156)", 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.28348047899999, 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35471572400000001, 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.749 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460486+0000", 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460491+0000", 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4039999999999997e-06 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460497+0000", 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460502+0000", 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4759999999999997e-06 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460547+0000", 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4233999999999997e-05 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460568+0000", 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1189999999999999e-05 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468865+0000", 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082969110000000006 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815202+0000", 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34633774499999997 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165851 7.1 7:8fdd4296:::benchmark_data_smithi017_422824_object71981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.282635941, 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36053184399999999, 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.754 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165851 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461330+0000", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461336+0000", 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5899999999999996e-06 2023-09-23T15:23:27.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461348+0000", 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2065000000000001e-05 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461357+0000", 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8189999999999998e-06 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.757 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9935999999999997e-05 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461554+0000", 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013704900000000001 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121172 2023-09-23T15:23:27.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815161+0000", 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35348590600000002 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815321+0000", 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160006 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821782+0000", 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064606120000000001 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821827+0000", 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5030000000000001e-05 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821863+0000", 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6208999999999998e-05 2023-09-23T15:23:27.760 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165853 7.3 7:c0488a25:::benchmark_data_smithi017_422824_object71983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.28259503800001, 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35803834699999998, 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.761 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165853 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:23:27.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461375+0000", 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2899999999999998e-06 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461393+0000", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8119000000000001e-05 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461404+0000", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0916e-05 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464554+0000", 2023-09-23T15:23:27.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003149799 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464660+0000", 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106097 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464785+0000", 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012551000000000001 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818430+0000", 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35364495699999998 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818460+0000", 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9578999999999999e-05 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.766 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819308+0000", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084795300000000003 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819358+0000", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9982999999999999e-05 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819411+0000", 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2698000000000003e-05 2023-09-23T15:23:27.767 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165854 7.a 7:57b0f785:::benchmark_data_smithi017_422824_object71984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.282549147, 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35469288599999999, 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.768 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165854 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.769 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461421+0000", 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1570000000000002e-06 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.770 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461429+0000", 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461435+0000", 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6699999999999999e-06 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.771 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461497+0000", 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1896999999999994e-05 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461605+0000", 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108937 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9722999999999996e-05 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.772 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815166+0000", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35349039599999998 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4358999999999996e-05 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816030+0000", 2023-09-23T15:23:27.773 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077006299999999998 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816073+0000", 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3195999999999998e-05 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816111+0000", 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7719999999999998e-05 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.774 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165856 7.2 7:45443312:::benchmark_data_smithi017_422824_object71986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.28252135599999, 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36007705000000001, 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.775 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165856 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.776 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461445+0000", 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461448+0000", 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1530000000000001e-06 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.777 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461456+0000", 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1609999999999997e-06 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461461+0000", 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4960000000000004e-06 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464405+0000", 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029436470000000002 2023-09-23T15:23:27.778 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464508+0000", 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010341899999999999 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464624+0000", 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115374 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.779 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815184+0000", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35056013400000002 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815339+0000", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015575800000000001 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821434+0000", 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060949569999999998 2023-09-23T15:23:27.780 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821484+0000", 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9515999999999998e-05 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821523+0000", 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8986999999999998e-05 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.781 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165858 7.3 7:c518666c:::benchmark_data_smithi017_422824_object71988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.28250125700001, 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36584072299999998, 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.782 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165858 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461468+0000", 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4530000000000001e-06 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461474+0000", 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7010000000000001e-06 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461486+0000", 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.201e-05 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467017+0000", 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055312130000000001 2023-09-23T15:23:27.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34830823100000002 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815477+0000", 2023-09-23T15:23:27.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015180900000000001 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819449+0000", 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039723479999999997 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.823533+0000", 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040831690000000002 2023-09-23T15:23:27.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827223+0000", 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036904910000000002 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827268+0000", 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5204000000000003e-05 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827307+0000", 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8433999999999998e-05 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165860 7.6 e271/223 7:667b3ce6:::benchmark_data_smithi017_422824_object71990:head v 271'10449, mlcod=271'10447)", 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.27747105200001, 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34876393300000003, 2023-09-23T15:23:27.789 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466495+0000", 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466503+0000", 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7440000000000007e-06 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466515+0000", 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2150999999999999e-05 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466523+0000", 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466566+0000", 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2422000000000001e-05 2023-09-23T15:23:27.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466587+0000", 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1475e-05 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815220+0000", 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34863227000000002 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0509000000000001e-05 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.276597501, 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34789368300000001, 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.794 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467369+0000", 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467375+0000", 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6760000000000004e-06 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815177+0000", 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.347802218 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2822e-05 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815244+0000", 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4230000000000001e-05 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815253+0000", 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1579999999999994e-06 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815263+0000", 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0268999999999999e-05 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.798 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.27489905499999, 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34704774999999999, 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:23:27.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469067+0000", 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469074+0000", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2230000000000004e-06 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815176+0000", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.346102403 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:23:27.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3959e-05 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815563+0000", 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037292800000000002 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816024+0000", 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046142900000000002 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816116+0000", 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1516999999999998e-05 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165851 7.1 e271/263 ondisk, result = 0)", 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 186.27239341800001, 2023-09-23T15:23:27.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34375118199999999, 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:23:27.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471573+0000", 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471579+0000", 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1940000000000003e-06 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815160+0000", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.343580518 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815175+0000", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5639000000000001e-05 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815306+0000", 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130894 2023-09-23T15:23:27.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815315+0000", 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5769999999999994e-06 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0227999999999999e-05 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.807 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179348 7.1 7:83a21fe1:::benchmark_data_smithi017_422824_object85478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 166.06214869999999, 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32463872399999999, 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179348 2023-09-23T15:23:27.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681817+0000", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681825+0000", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0039999999999999e-06 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681838+0000", 2023-09-23T15:23:27.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3648e-05 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681846+0000", 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0199999999999994e-06 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685481+0000", 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00363489 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685642+0000", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016116599999999999 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685776+0000", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133442 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688382+0000", 2023-09-23T15:23:27.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026061270000000002 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688535+0000", 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015359299999999999 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006352+0000", 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31781674399999998 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006412+0000", 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0061e-05 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006457+0000", 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5016999999999999e-05 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179350 7.1 7:87665378:::benchmark_data_smithi017_422824_object85480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 166.055244921, 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31832527799999999, 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179350 2023-09-23T15:23:27.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688721+0000", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688727+0000", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.806e-06 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688738+0000", 2023-09-23T15:23:27.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1192000000000001e-05 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688745+0000", 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2049999999999996e-06 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688796+0000", 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0779000000000003e-05 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688930+0000", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013444299999999999 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689051+0000", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120451 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.691835+0000", 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027841459999999999 2023-09-23T15:23:27.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694698+0000", 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028631170000000001 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006962+0000", 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31226363000000001 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007010+0000", 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8396999999999998e-05 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007048+0000", 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7428999999999997e-05 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179357 7.2 7:4242eeb1:::benchmark_data_smithi017_422824_object85487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 166.055198445, 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.317811762, 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179357 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.822 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688772+0000", 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.569e-06 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688782+0000", 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8120000000000002e-06 2023-09-23T15:23:27.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688788+0000", 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6980000000000001e-06 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688840+0000", 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1610000000000002e-05 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688932+0000", 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2485999999999995e-05 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048049699999999998 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006235+0000", 2023-09-23T15:23:27.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31682244500000001 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006406+0000", 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017080200000000001 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006494+0000", 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8374000000000005e-05 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006540+0000", 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5130000000000003e-05 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006581+0000", 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0924999999999999e-05 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179355 7.6 e276/223 7:615b3fe2:::benchmark_data_smithi017_422824_object85485:head v 276'11306, mlcod=276'11301)", 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 166.05488687100001, 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31722602799999999, 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689087+0000", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5159999999999997e-06 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689096+0000", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689103+0000", 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6159999999999998e-06 2023-09-23T15:23:27.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692603+0000", 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035004910000000001 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692623+0000", 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0027999999999999e-05 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006273+0000", 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31364994099999999 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006306+0000", 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3408000000000003e-05 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.833 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179351 7.c e276/223 7:387cb04d:::benchmark_data_smithi017_422824_object85481:head v 276'11330, mlcod=276'11328)", 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 166.05460711500001, 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31712001899999998, 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689359+0000", 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689369+0000", 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2050000000000008e-06 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689378+0000", 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3030000000000005e-06 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689397+0000", 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8868999999999998e-05 2023-09-23T15:23:27.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689491+0000", 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3949999999999993e-05 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689511+0000", 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0625999999999999e-05 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694710+0000", 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051988340000000003 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006480+0000", 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31176990500000001 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.838 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50576 8.b 8:d9711445:::benchmark_data_smithi017_485823_object50575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 91.551711463999993, 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1259012579999998, 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50576 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192254+0000", 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:23:27.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192281+0000", 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7236000000000001e-05 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192304+0000", 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3314000000000001e-05 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192314+0000", 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6269999999999995e-06 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192529+0000", 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000214637 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192538+0000", 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:23:27.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119735+0000", 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92719635 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119749+0000", 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4219999999999999e-05 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307742+0000", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.187992758 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307887+0000", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014497199999999999 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.308028+0000", 2023-09-23T15:23:27.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014144799999999999 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314256+0000", 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062275179999999996 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314333+0000", 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6796000000000001e-05 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315870+0000", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001537524 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315924+0000", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3733999999999997e-05 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.318157+0000", 2023-09-23T15:23:27.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022332469999999998 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50579 8.b 8:dd05f5d9:::benchmark_data_smithi017_485823_object50578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 91.551597674999996, 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.129065862, 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.847 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50579 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192369+0000", 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192402+0000", 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2693e-05 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192410+0000", 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192415+0000", 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8710000000000002e-06 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192564+0000", 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014868300000000001 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192571+0000", 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1160000000000001e-06 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119777+0000", 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92720541499999998 2023-09-23T15:23:27.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119787+0000", 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9520000000000006e-06 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312151+0000", 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1923646489999999 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312321+0000", 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017003 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312518+0000", 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196746 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314638+0000", 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021200590000000001 2023-09-23T15:23:27.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315028+0000", 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038974000000000002 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319363+0000", 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043355499999999996 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319441+0000", 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7822000000000001e-05 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.321435+0000", 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019941450000000001 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.855 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:84492 8.b 8:da4f3388:::benchmark_data_smithi017_485823_object84491:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:41.451140+0000", 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 46.292826779999999, 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.249512386, 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 84492 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451140+0000", 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451140+0000", 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451139+0000", 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451162+0000", 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2198999999999999e-05 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451173+0000", 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.137e-05 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.451180+0000", 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1590000000000004e-06 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.512198+0000", 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.061017898000000001 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.512358+0000", 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000159593 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.512519+0000", 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016165999999999999 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.524450+0000", 2023-09-23T15:23:27.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011930221 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.524457+0000", 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4309999999999997e-06 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.698585+0000", 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.17412825200000001 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.698658+0000", 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2976000000000001e-05 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:41.700653+0000", 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019945919999999999 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:86748 8.b 8:d4ce9a84:::benchmark_data_smithi017_485823_object86747:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:45.134361+0000", 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.609606272999997, 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24969455500000001, 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 86748 2023-09-23T15:23:27.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134361+0000", 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134361+0000", 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134359+0000", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134414+0000", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5281000000000001e-05 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134430+0000", 2023-09-23T15:23:27.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5676999999999999e-05 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134438+0000", 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9549999999999992e-06 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134490+0000", 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2373000000000002e-05 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134663+0000", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017249400000000001 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134822+0000", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015945 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381094+0000", 2023-09-23T15:23:27.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24627185800000001 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381168+0000", 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3928000000000001e-05 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381461+0000", 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029318399999999999 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381521+0000", 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0188e-05 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.384055+0000", 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0025340950000000001 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:86749 8.b 8:dabf8292:::benchmark_data_smithi017_485823_object86748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.609514124999997, 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.252098983, 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 86749 2023-09-23T15:23:27.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134474+0000", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134481+0000", 2023-09-23T15:23:27.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5270000000000001e-06 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134486+0000", 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0710000000000002e-06 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137557+0000", 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030706990000000001 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137742+0000", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018558499999999999 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137914+0000", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017187300000000001 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381133+0000", 2023-09-23T15:23:27.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243218563 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381203+0000", 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9811e-05 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.384070+0000", 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028675089999999999 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.384134+0000", 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3756000000000004e-05 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.386552+0000", 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024180650000000001 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:23:27.876 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:23:27.877 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:23:27.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:27.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:27.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:28.025 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:28] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.43.0" 2023-09-23T15:23:28.042 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:28.141 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:28.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:28.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:28.257 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:28.338 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:28.465 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:28.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:28.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:28.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:28 smithi027 ceph-mon[195233]: pgmap v919: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 30132/210315 objects degraded (14.327%); 1013/210315 objects misplaced (0.482%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T15:23:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/576754746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:28.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:28.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[215646]: pgmap v919: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 30132/210315 objects degraded (14.327%); 1013/210315 objects misplaced (0.482%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T15:23:28.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/576754746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:28.625 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:28.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[222878]: pgmap v919: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 672 op/s; 30132/210315 objects degraded (14.327%); 1013/210315 objects misplaced (0.482%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T15:23:28.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/576754746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:28.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:28.630 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:30.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:23:30.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:30 smithi027 ceph-mon[195233]: pgmap v920: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 30132/207948 objects degraded (14.490%); 1013/207948 objects misplaced (0.487%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:23:30.694 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:30 smithi017 ceph-mon[222878]: pgmap v920: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 30132/207948 objects degraded (14.490%); 1013/207948 objects misplaced (0.487%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:23:30.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:30 smithi017 ceph-mon[215646]: pgmap v920: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 487 op/s; 30132/207948 objects degraded (14.490%); 1013/207948 objects misplaced (0.487%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:23:30.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:30.699 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:30.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:30.802 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:30.877 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:30.940 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:30.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:30.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:31.043 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:31.075 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:31.119 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:31.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:31.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:31.273 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:31.500 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:31.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:31.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:31.606 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:31.679 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:31.776 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:31.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:31.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:31.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:31.925 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:32.025 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:32.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:32.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:32.131 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:32.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:32.395 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:32.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:32.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:32.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:23:32.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:32.551 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:32.653 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:32.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:32.658 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:32.658 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:32 smithi027 ceph-mon[195233]: pgmap v921: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 29704/200799 objects degraded (14.793%); 668/200799 objects misplaced (0.333%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T15:23:32.659 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29704/200799 objects degraded (14.793%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:32.659 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3528932294' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.659 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:32 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[215646]: pgmap v921: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 29704/200799 objects degraded (14.793%); 668/200799 objects misplaced (0.333%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29704/200799 objects degraded (14.793%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3528932294' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[222878]: pgmap v921: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 29704/200799 objects degraded (14.793%); 668/200799 objects misplaced (0.333%); 7.1 MiB/s, 113 objects/s recovering 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29704/200799 objects degraded (14.793%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3528932294' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:32 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:23:32.759 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:32.818 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:32.929 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:32.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:32.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:33.038 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:33.105 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:33.150 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:33.181 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:23:33.184 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:23:33.306 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:33.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:33.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:33.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:33.506 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:23:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:33 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:33 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:23:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:23:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:33 smithi027 ceph-mon[195233]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T15:23:33.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:33.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:33.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:23:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:23:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:23:33.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[222878]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T15:23:33.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:23:33.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:23:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3621799584' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:23:33.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:33 smithi017 ceph-mon[215646]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T15:23:33.702 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:23:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:23:33.703 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:23:33.703 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:23:33.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:33.809 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:34.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:23:34.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[215646]: pgmap v922: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 29704/198459 objects degraded (14.967%); 668/198459 objects misplaced (0.337%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[215646]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[215646]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[215646]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[222878]: pgmap v922: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 29704/198459 objects degraded (14.967%); 668/198459 objects misplaced (0.337%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[222878]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[222878]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:34 smithi017 ceph-mon[222878]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:34 smithi027 ceph-mon[195233]: pgmap v922: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 630 op/s; 29704/198459 objects degraded (14.967%); 668/198459 objects misplaced (0.337%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:23:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:34 smithi027 ceph-mon[195233]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:34 smithi027 ceph-mon[195233]: failed to encode map e319 with expected crc 2023-09-23T15:23:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:34 smithi027 ceph-mon[195233]: failed to encode map e319 with expected crc 2023-09-23T15:23:35.280 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:35.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:35.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:35.299 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:35.322 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:23:35.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:35.465 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:35.591 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:35.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:35.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:35.701 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:35.737 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:35.990 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:35.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:36.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:36.153 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:36.362 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:36.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:36.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:36 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:36.441 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:36 smithi017 ceph-mon[215646]: pgmap v924: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 29354/194937 objects degraded (15.058%); 668/194937 objects misplaced (0.343%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T15:23:36.443 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:36.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:36 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:36.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:36 smithi017 ceph-mon[222878]: pgmap v924: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 29354/194937 objects degraded (15.058%); 668/194937 objects misplaced (0.343%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T15:23:36.474 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:36.567 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:36.785 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:36.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:36.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:36 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:36 smithi027 ceph-mon[195233]: pgmap v924: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 29354/194937 objects degraded (15.058%); 668/194937 objects misplaced (0.343%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T15:23:36.894 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:36.959 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:37.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:37.121 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:37.124 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:37.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:37.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:37.166 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:37.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:37.228 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:37.266 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:37.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:23:37.297 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:37.311 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:37.358 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:37.392 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.400 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:23:37.435 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:37.435 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:37.471 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:37.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:37.483 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:37.584 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:37.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2569560280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:37.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29354/194937 objects degraded (15.058%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:37.654 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:37.679 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2569560280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:37.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29354/194937 objects degraded (15.058%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:37.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2569560280' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:37.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29354/194937 objects degraded (15.058%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:37.781 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:37.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:37.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:37.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:37.938 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:38.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:38] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T15:23:38.326 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:23:38.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:38 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:38.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:38 smithi027 ceph-mon[195233]: pgmap v925: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 29354/191709 objects degraded (15.312%); 311/191709 objects misplaced (0.162%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:38.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:38 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:38.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:38.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[215646]: pgmap v925: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 29354/191709 objects degraded (15.312%); 311/191709 objects misplaced (0.162%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:38.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:38.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:38.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[222878]: pgmap v925: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 619 op/s; 29354/191709 objects degraded (15.312%); 311/191709 objects misplaced (0.162%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:38.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:38 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:38.917 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:39.048 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:39.056 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:39.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:39.237 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:39.372 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:39.407 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:39.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:39.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:39.603 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:39.731 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:39.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:39.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:39.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:39.912 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:40.084 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:40.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:40.201 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:40.308 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:40.467 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:40.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:40.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:40.511 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:23:40.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:40.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:40 smithi027 ceph-mon[195233]: pgmap v926: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 29354/189468 objects degraded (15.493%); 311/189468 objects misplaced (0.164%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:40.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:40 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:40.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1823251634' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.664 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:40.747 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:40.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:40.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:40.854 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:40.880 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[215646]: pgmap v926: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 29354/189468 objects degraded (15.493%); 311/189468 objects misplaced (0.164%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:40.880 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:40.880 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1823251634' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[222878]: pgmap v926: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 615 op/s; 29354/189468 objects degraded (15.493%); 311/189468 objects misplaced (0.164%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:23:40.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:40.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1823251634' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:23:40.883 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:40.982 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:40.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:41.043 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:41.044 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:41.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:23:41.086 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:23:41.118 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:41.275 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:23:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:41.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:23:41.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:41.406 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:41.483 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:41.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:41.552 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.553 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[222878]: pgmap v928: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 29010/182649 objects degraded (15.883%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.554 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:41 smithi017 ceph-mon[215646]: pgmap v928: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 29010/182649 objects degraded (15.883%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:23:41.555 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:23:41.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:23:41.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:41.653 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:41.734 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:41.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:41.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:23:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T15:23:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: failed to encode map e320 with expected crc 2023-09-23T15:23:41.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:41 smithi027 ceph-mon[195233]: pgmap v928: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 658 op/s; 29010/182649 objects degraded (15.883%); 8.0 MiB/s, 127 objects/s recovering 2023-09-23T15:23:41.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:41.915 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:41.945 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:42.121 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:42.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:42.189 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:23:42.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:42.303 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:42.447 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:42.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 29010/182649 objects degraded (15.883%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:42.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:42.560 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 29010/182649 objects degraded (15.883%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: failed to encode map e321 with expected crc 2023-09-23T15:23:42.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T15:23:42.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 29010/182649 objects degraded (15.883%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:42.576 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:42.639 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:42.661 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:42.782 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:42.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:42.796 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:42.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:42.932 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:42.975 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:43.199 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:23:43.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:43.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: pgmap v930: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 29010/180585 objects degraded (16.064%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T15:23:43.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:23:43.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:43.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: pgmap v930: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 29010/180585 objects degraded (16.064%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:43.518 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: pgmap v930: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 29010/180585 objects degraded (16.064%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2414954589' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:43.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:44.353 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:44.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:44.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:44.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:44.538 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:44.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[215646]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[215646]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[222878]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[222878]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:44 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:44.824 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:44.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:44.833 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:44 smithi027 ceph-mon[195233]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:44 smithi027 ceph-mon[195233]: failed to encode map e322 with expected crc 2023-09-23T15:23:44.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:44 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:44.933 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:44.978 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:45.343 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:45.343 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:45.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:45.364 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:45.401 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:45.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:45.473 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:45.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:45.534 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:45.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:45.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:45.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:45.606 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.637 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:23:45.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:45.713 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:45.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:45 smithi027 ceph-mon[195233]: pgmap v932: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 28734/178362 objects degraded (16.110%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T15:23:45.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3624818687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:45.857 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:45 smithi017 ceph-mon[215646]: pgmap v932: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 28734/178362 objects degraded (16.110%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T15:23:45.857 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3624818687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:45.858 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:45 smithi017 ceph-mon[222878]: pgmap v932: 109 pgs: 1 peering, 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 28734/178362 objects degraded (16.110%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T15:23:45.858 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3624818687' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:45.910 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:45.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:45.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:46.014 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:46.045 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:46.096 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:46.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:46.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:46.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:46.270 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:46.386 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:23:46.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:23:46.518 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:46.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:46.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:46.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:46.695 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:46 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:46.740 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:46.854 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:46.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:46.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:46.950 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:46 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:46.951 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:46 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:46.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:47.038 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:47.133 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:47.142 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:47.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:47.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:47.189 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:47.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:47.297 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:23:47.310 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:47.317 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:23:47.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:47 smithi027 ceph-mon[195233]: pgmap v933: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 28461/174858 objects degraded (16.277%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:47.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28734/178362 objects degraded (16.110%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:47.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:47 smithi027 ceph-mon[195233]: from='client.55921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:48.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[215646]: pgmap v933: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 28461/174858 objects degraded (16.277%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:48.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28734/178362 objects degraded (16.110%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:48.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[215646]: from='client.55921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:48.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:48] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T15:23:48.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[222878]: pgmap v933: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 28461/174858 objects degraded (16.277%); 7.3 MiB/s, 116 objects/s recovering 2023-09-23T15:23:48.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28734/178362 objects degraded (16.110%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:48.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:47 smithi017 ceph-mon[222878]: from='client.55921 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:48.361 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:48.367 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:48.468 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:48.497 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:48.542 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:48.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:48.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:48.561 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:23:48.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:48.680 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:48.844 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":933,"stamp":"2023-09-23T15:23:46.722597+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3822774641,"num_objects":58286,"num_object_clones":0,"num_object_copies":174858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28461,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58286,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":122941,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8678,"num_bytes_recovered":569412981,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108525,"ondisk_log_size":108525,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14637980,"kb_used_data":11574108,"kb_used_omap":44,"kb_used_meta":3063763,"kb_avail":735323236,"statfs":{"total":767960285184,"available":752970993664,"internally_reserved":0,"allocated":11851886592,"data_stored":11842355861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3137294267},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-208666624,"num_objects":-3184,"num_object_clones":0,"num_object_copies":-9552,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-549,"num_objects_misplaced":-311,"num_objects_unfound":0,"num_objects_dirty":-3184,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3184,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":722,"num_bytes_recovered":47316992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.223757"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505792+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:23:41.505792+0000","last_peered":"2023-09-23T15:23:41.505792+0000","last_clean":"2023-09-23T15:23:41.505792+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:23:41.505792+0000","last_undegraded":"2023-09-23T15:23:41.505792+0000","last_fullsized":"2023-09-23T15:23:41.505792+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":278,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.161541+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:23:43.161541+0000","last_peered":"2023-09-23T15:23:43.161541+0000","last_clean":"2023-09-23T15:23:43.161541+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:23:43.161541+0000","last_undegraded":"2023-09-23T15:23:43.161541+0000","last_fullsized":"2023-09-23T15:23:43.161541+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":225,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.501346+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:23:40.501346+0000","last_peered":"2023-09-23T15:23:40.501346+0000","last_clean":"2023-09-23T15:23:40.501346+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:23:40.501346+0000","last_undegraded":"2023-09-23T15:23:40.501346+0000","last_fullsized":"2023-09-23T15:23:40.501346+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":300,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.512602+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:23:41.512602+0000","last_peered":"2023-09-23T15:23:41.512602+0000","last_clean":"2023-09-23T15:23:41.512602+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:23:41.512602+0000","last_undegraded":"2023-09-23T15:23:41.512602+0000","last_fullsized":"2023-09-23T15:23:41.512602+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":323,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504511+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:23:41.504511+0000","last_peered":"2023-09-23T15:23:41.504511+0000","last_clean":"2023-09-23T15:23:41.504511+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:23:41.504511+0000","last_undegraded":"2023-09-23T15:23:41.504511+0000","last_fullsized":"2023-09-23T15:23:41.504511+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505352+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:23:41.505352+0000","last_peered":"2023-09-23T15:23:41.505352+0000","last_clean":"2023-09-23T15:23:41.505352+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:23:41.505352+0000","last_undegraded":"2023-09-23T15:23:41.505352+0000","last_fullsized":"2023-09-23T15:23:41.505352+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":269,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.510191+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:23:41.510191+0000","last_peered":"2023-09-23T15:23:41.510191+0000","last_clean":"2023-09-23T15:23:41.510191+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:23:41.510191+0000","last_undegraded":"2023-09-23T15:23:41.510191+0000","last_fullsized":"2023-09-23T15:23:41.510191+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":320,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504956+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:23:41.504956+0000","last_peered":"2023-09-23T15:23:41.504956+0000","last_clean":"2023-09-23T15:23:41.504956+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:23:41.504956+0000","last_undegraded":"2023-09-23T15:23:41.504956+0000","last_fullsized":"2023-09-23T15:23:41.504956+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":679,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508696+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:23:41.508696+0000","last_peered":"2023-09-23T15:23:41.508696+0000","last_clean":"2023-09-23T15:23:41.508696+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:23:41.508696+0000","last_undegraded":"2023-09-23T15:23:41.508696+0000","last_fullsized":"2023-09-23T15:23:41.508696+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":322,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505494+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:23:41.505494+0000","last_peered":"2023-09-23T15:23:41.505494+0000","last_clean":"2023-09-23T15:23:41.505494+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:23:41.505494+0000","last_undegraded":"2023-09-23T15:23:41.505494+0000","last_fullsized":"2023-09-23T15:23:41.505494+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":274,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504574+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:23:41.504574+0000","last_peered":"2023-09-23T15:23:41.504574+0000","last_clean":"2023-09-23T15:23:41.504574+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:23:41.504574+0000","last_undegraded":"2023-09-23T15:23:41.504574+0000","last_fullsized":"2023-09-23T15:23:41.504574+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":66,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509284+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:23:41.509284+0000","last_peered":"2023-09-23T15:23:41.509284+0000","last_clean":"2023-09-23T15:23:41.509284+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:23:41.509284+0000","last_undegraded":"2023-09-23T15:23:41.509284+0000","last_fullsized":"2023-09-23T15:23:41.509284+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":211,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.502218+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:23:40.502218+0000","last_peered":"2023-09-23T15:23:40.502218+0000","last_clean":"2023-09-23T15:23:40.502218+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:23:40.502218+0000","last_undegraded":"2023-09-23T15:23:40.502218+0000","last_fullsized":"2023-09-23T15:23:40.502218+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":56,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506960+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:23:41.506960+0000","last_peered":"2023-09-23T15:23:41.506960+0000","last_clean":"2023-09-23T15:23:41.506960+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:23:41.506960+0000","last_undegraded":"2023-09-23T15:23:41.506960+0000","last_fullsized":"2023-09-23T15:23:41.506960+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506468+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:23:41.506468+0000","last_peered":"2023-09-23T15:23:41.506468+0000","last_clean":"2023-09-23T15:23:41.506468+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:23:41.506468+0000","last_undegraded":"2023-09-23T15:23:41.506468+0000","last_fullsized":"2023-09-23T15:23:41.506468+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":631,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.510324+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:23:41.510324+0000","last_peered":"2023-09-23T15:23:41.510324+0000","last_clean":"2023-09-23T15:23:41.510324+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:23:41.510324+0000","last_undegraded":"2023-09-23T15:23:41.510324+0000","last_fullsized":"2023-09-23T15:23:41.510324+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":70,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507872+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:23:41.507872+0000","last_peered":"2023-09-23T15:23:41.507872+0000","last_clean":"2023-09-23T15:23:41.507872+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:23:41.507872+0000","last_undegraded":"2023-09-23T15:23:41.507872+0000","last_fullsized":"2023-09-23T15:23:41.507872+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":325,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505025+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:23:41.505025+0000","last_peered":"2023-09-23T15:23:41.505025+0000","last_clean":"2023-09-23T15:23:41.505025+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:23:41.505025+0000","last_undegraded":"2023-09-23T15:23:41.505025+0000","last_fullsized":"2023-09-23T15:23:41.505025+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":680,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504140+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:23:41.504140+0000","last_peered":"2023-09-23T15:23:41.504140+0000","last_clean":"2023-09-23T15:23:41.504140+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:23:41.504140+0000","last_undegraded":"2023-09-23T15:23:41.504140+0000","last_fullsized":"2023-09-23T15:23:41.504140+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509192+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:23:41.509192+0000","last_peered":"2023-09-23T15:23:41.509192+0000","last_clean":"2023-09-23T15:23:41.509192+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:23:41.509192+0000","last_undegraded":"2023-09-23T15:23:41.509192+0000","last_fullsized":"2023-09-23T15:23:41.509192+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504877+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:23:41.504877+0000","last_peered":"2023-09-23T15:23:41.504877+0000","last_clean":"2023-09-23T15:23:41.504877+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:23:41.504877+0000","last_undegraded":"2023-09-23T15:23:41.504877+0000","last_fullsized":"2023-09-23T15:23:41.504877+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":610,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.509269+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:23:40.509269+0000","last_peered":"2023-09-23T15:23:40.509269+0000","last_clean":"2023-09-23T15:23:40.509269+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:23:40.509269+0000","last_undegraded":"2023-09-23T15:23:40.509269+0000","last_fullsized":"2023-09-23T15:23:40.509269+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507585+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:23:41.507585+0000","last_peered":"2023-09-23T15:23:41.507585+0000","last_clean":"2023-09-23T15:23:41.507585+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:23:41.507585+0000","last_undegraded":"2023-09-23T15:23:41.507585+0000","last_fullsized":"2023-09-23T15:23:41.507585+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":287,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.512662+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:23:41.512662+0000","last_peered":"2023-09-23T15:23:41.512662+0000","last_clean":"2023-09-23T15:23:41.512662+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:23:41.512662+0000","last_undegraded":"2023-09-23T15:23:41.512662+0000","last_fullsized":"2023-09-23T15:23:41.512662+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":638,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504908+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:23:41.504908+0000","last_peered":"2023-09-23T15:23:41.504908+0000","last_clean":"2023-09-23T15:23:41.504908+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:23:41.504908+0000","last_undegraded":"2023-09-23T15:23:41.504908+0000","last_fullsized":"2023-09-23T15:23:41.504908+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":258,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504280+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:23:41.504280+0000","last_peered":"2023-09-23T15:23:41.504280+0000","last_clean":"2023-09-23T15:23:41.504280+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:23:41.504280+0000","last_undegraded":"2023-09-23T15:23:41.504280+0000","last_fullsized":"2023-09-23T15:23:41.504280+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":325,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505338+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:23:41.505338+0000","last_peered":"2023-09-23T15:23:41.505338+0000","last_clean":"2023-09-23T15:23:41.505338+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:23:41.505338+0000","last_undegraded":"2023-09-23T15:23:41.505338+0000","last_fullsized":"2023-09-23T15:23:41.505338+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"322'7790","reported_seq":8406,"reported_epoch":322,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:44.852326+0000","last_change":"2023-09-23T15:23:39.629532+0000","last_active":"2023-09-23T15:23:44.852326+0000","last_peered":"2023-09-23T15:23:44.852326+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:58.288439+0000","last_became_peered":"2023-09-23T15:21:58.288439+0000","last_unstale":"2023-09-23T15:23:44.852326+0000","last_undegraded":"2023-09-23T15:21:58.149099+0000","last_fullsized":"2023-09-23T15:21:58.128266+0000","mapping_epoch":297,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6890,"log_dups_size":0,"ondisk_log_size":6890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:22:01.838822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237502464,"num_objects":3624,"num_object_clones":0,"num_object_copies":10872,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2959,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3624,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7790,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":283,"num_bytes_recovered":18546688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":665}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":584,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509372+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:23:41.509372+0000","last_peered":"2023-09-23T15:23:41.509372+0000","last_clean":"2023-09-23T15:23:41.509372+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:23:41.509372+0000","last_undegraded":"2023-09-23T15:23:41.509372+0000","last_fullsized":"2023-09-23T15:23:41.509372+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":62,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508305+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:23:41.508305+0000","last_peered":"2023-09-23T15:23:41.508305+0000","last_clean":"2023-09-23T15:23:41.508305+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:23:41.508305+0000","last_undegraded":"2023-09-23T15:23:41.508305+0000","last_fullsized":"2023-09-23T15:23:41.508305+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":314,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.503977+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:23:41.503977+0000","last_peered":"2023-09-23T15:23:41.503977+0000","last_clean":"2023-09-23T15:23:41.503977+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:23:41.503977+0000","last_undegraded":"2023-09-23T15:23:41.503977+0000","last_fullsized":"2023-09-23T15:23:41.503977+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"322'7451","reported_seq":13260,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:44.357700+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:23:44.357700+0000","last_peered":"2023-09-23T15:23:44.357700+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:23:44.357700+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"291'600","ondisk_log_start":"291'600","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6851,"log_dups_size":0,"ondisk_log_size":6851,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237436928,"num_objects":3623,"num_object_clones":0,"num_object_copies":10869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3782,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7451,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1914,"num_bytes_recovered":125435904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":1149},{"shards":"0,4,7","objects":1484},{"shards":"4,7","objects":990}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":642,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504941+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:23:41.504941+0000","last_peered":"2023-09-23T15:23:41.504941+0000","last_clean":"2023-09-23T15:23:41.504941+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:23:41.504941+0000","last_undegraded":"2023-09-23T15:23:41.504941+0000","last_fullsized":"2023-09-23T15:23:41.504941+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":62,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508205+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:23:41.508205+0000","last_peered":"2023-09-23T15:23:41.508205+0000","last_clean":"2023-09-23T15:23:41.508205+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:23:41.508205+0000","last_undegraded":"2023-09-23T15:23:41.508205+0000","last_fullsized":"2023-09-23T15:23:41.508205+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505719+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:23:41.505719+0000","last_peered":"2023-09-23T15:23:41.505719+0000","last_clean":"2023-09-23T15:23:41.505719+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:23:41.505719+0000","last_undegraded":"2023-09-23T15:23:41.505719+0000","last_fullsized":"2023-09-23T15:23:41.505719+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"322'7619","reported_seq":7686,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:44.091459+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:23:44.091459+0000","last_peered":"2023-09-23T15:23:44.091459+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:23:44.091459+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"292'1000","ondisk_log_start":"292'1000","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235601920,"num_objects":3595,"num_object_clones":0,"num_object_copies":10785,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3595,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7619,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2617},{"shards":"0,7","objects":-1035}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":529,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505525+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:23:41.505525+0000","last_peered":"2023-09-23T15:23:41.505525+0000","last_clean":"2023-09-23T15:23:41.505525+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:23:41.505525+0000","last_undegraded":"2023-09-23T15:23:41.505525+0000","last_fullsized":"2023-09-23T15:23:41.505525+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":62,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509236+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:23:41.509236+0000","last_peered":"2023-09-23T15:23:41.509236+0000","last_clean":"2023-09-23T15:23:41.509236+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:23:41.509236+0000","last_undegraded":"2023-09-23T15:23:41.509236+0000","last_fullsized":"2023-09-23T15:23:41.509236+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":285,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505749+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:23:41.505749+0000","last_peered":"2023-09-23T15:23:41.505749+0000","last_clean":"2023-09-23T15:23:41.505749+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:23:41.505749+0000","last_undegraded":"2023-09-23T15:23:41.505749+0000","last_fullsized":"2023-09-23T15:23:41.505749+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"321'7601","reported_seq":7653,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:41.891431+0000","last_change":"2023-09-23T15:22:27.588207+0000","last_active":"2023-09-23T15:23:41.891431+0000","last_peered":"2023-09-23T15:23:41.891431+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:23:41.891431+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"292'1000","ondisk_log_start":"292'1000","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243204096,"num_objects":3711,"num_object_clones":0,"num_object_copies":11133,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3711,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7601,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-945}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":537,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507971+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:23:41.507971+0000","last_peered":"2023-09-23T15:23:41.507971+0000","last_clean":"2023-09-23T15:23:41.507971+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:23:41.507971+0000","last_undegraded":"2023-09-23T15:23:41.507971+0000","last_fullsized":"2023-09-23T15:23:41.507971+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506168+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:23:41.506168+0000","last_peered":"2023-09-23T15:23:41.506168+0000","last_clean":"2023-09-23T15:23:41.506168+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:23:41.506168+0000","last_undegraded":"2023-09-23T15:23:41.506168+0000","last_fullsized":"2023-09-23T15:23:41.506168+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":303,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509598+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:23:41.509598+0000","last_peered":"2023-09-23T15:23:41.509598+0000","last_clean":"2023-09-23T15:23:41.509598+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:23:41.509598+0000","last_undegraded":"2023-09-23T15:23:41.509598+0000","last_fullsized":"2023-09-23T15:23:41.509598+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"322'7656","reported_seq":7704,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:45.570961+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:23:45.570961+0000","last_peered":"2023-09-23T15:23:45.570961+0000","last_clean":"2023-09-23T15:23:45.570961+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:23:45.570961+0000","last_undegraded":"2023-09-23T15:23:45.570961+0000","last_fullsized":"2023-09-23T15:23:45.570961+0000","mapping_epoch":284,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":5756,"log_dups_size":0,"ondisk_log_size":5756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237109248,"num_objects":3618,"num_object_clones":0,"num_object_copies":10854,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3618,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7656,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":558,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507709+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:23:41.507709+0000","last_peered":"2023-09-23T15:23:41.507709+0000","last_clean":"2023-09-23T15:23:41.507709+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:23:41.507709+0000","last_undegraded":"2023-09-23T15:23:41.507709+0000","last_fullsized":"2023-09-23T15:23:41.507709+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":314,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504792+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:23:41.504792+0000","last_peered":"2023-09-23T15:23:41.504792+0000","last_clean":"2023-09-23T15:23:41.504792+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:23:41.504792+0000","last_undegraded":"2023-09-23T15:23:41.504792+0000","last_fullsized":"2023-09-23T15:23:41.504792+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506856+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:23:41.506856+0000","last_peered":"2023-09-23T15:23:41.506856+0000","last_clean":"2023-09-23T15:23:41.506856+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:23:41.506856+0000","last_undegraded":"2023-09-23T15:23:41.506856+0000","last_fullsized":"2023-09-23T15:23:41.506856+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"321'7721","reported_seq":7768,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.886587+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:23:41.886587+0000","last_peered":"2023-09-23T15:23:41.886587+0000","last_clean":"2023-09-23T15:23:41.886587+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:23:41.886587+0000","last_undegraded":"2023-09-23T15:23:41.886587+0000","last_fullsized":"2023-09-23T15:23:41.886587+0000","mapping_epoch":284,"log_start":"292'1100","ondisk_log_start":"292'1100","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7721,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":626,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504185+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:23:41.504185+0000","last_peered":"2023-09-23T15:23:41.504185+0000","last_clean":"2023-09-23T15:23:41.504185+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:23:41.504185+0000","last_undegraded":"2023-09-23T15:23:41.504185+0000","last_fullsized":"2023-09-23T15:23:41.504185+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":269,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505832+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:23:41.505832+0000","last_peered":"2023-09-23T15:23:41.505832+0000","last_clean":"2023-09-23T15:23:41.505832+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:23:41.505832+0000","last_undegraded":"2023-09-23T15:23:41.505832+0000","last_fullsized":"2023-09-23T15:23:41.505832+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507506+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:23:41.507506+0000","last_peered":"2023-09-23T15:23:41.507506+0000","last_clean":"2023-09-23T15:23:41.507506+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:23:41.507506+0000","last_undegraded":"2023-09-23T15:23:41.507506+0000","last_fullsized":"2023-09-23T15:23:41.507506+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"322'7878","reported_seq":14423,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:44.680075+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:23:44.680075+0000","last_peered":"2023-09-23T15:23:44.680075+0000","last_clean":"2023-09-23T15:23:44.680075+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:23:44.680075+0000","last_undegraded":"2023-09-23T15:23:44.680075+0000","last_fullsized":"2023-09-23T15:23:44.680075+0000","mapping_epoch":310,"log_start":"292'1300","ondisk_log_start":"292'1300","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233570304,"num_objects":3564,"num_object_clones":0,"num_object_copies":10692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7878,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":577,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.509186+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:23:40.509186+0000","last_peered":"2023-09-23T15:23:40.509186+0000","last_clean":"2023-09-23T15:23:40.509186+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:23:40.509186+0000","last_undegraded":"2023-09-23T15:23:40.509186+0000","last_fullsized":"2023-09-23T15:23:40.509186+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505809+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:23:41.505809+0000","last_peered":"2023-09-23T15:23:41.505809+0000","last_clean":"2023-09-23T15:23:41.505809+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:23:41.505809+0000","last_undegraded":"2023-09-23T15:23:41.505809+0000","last_fullsized":"2023-09-23T15:23:41.505809+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507891+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:23:41.507891+0000","last_peered":"2023-09-23T15:23:41.507891+0000","last_clean":"2023-09-23T15:23:41.507891+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:23:41.507891+0000","last_undegraded":"2023-09-23T15:23:41.507891+0000","last_fullsized":"2023-09-23T15:23:41.507891+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"321'7539","reported_seq":7591,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:41.866641+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:23:41.866641+0000","last_peered":"2023-09-23T15:23:41.866641+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:23:41.866641+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"292'1000","ondisk_log_start":"292'1000","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246349824,"num_objects":3759,"num_object_clones":0,"num_object_copies":11277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7539,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-859}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":638,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504778+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:23:41.504778+0000","last_peered":"2023-09-23T15:23:41.504778+0000","last_clean":"2023-09-23T15:23:41.504778+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:23:41.504778+0000","last_undegraded":"2023-09-23T15:23:41.504778+0000","last_fullsized":"2023-09-23T15:23:41.504778+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":336,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504492+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:23:41.504492+0000","last_peered":"2023-09-23T15:23:41.504492+0000","last_clean":"2023-09-23T15:23:41.504492+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:23:41.504492+0000","last_undegraded":"2023-09-23T15:23:41.504492+0000","last_fullsized":"2023-09-23T15:23:41.504492+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":228,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.505623+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:23:40.505623+0000","last_peered":"2023-09-23T15:23:40.505623+0000","last_clean":"2023-09-23T15:23:40.505623+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:23:40.505623+0000","last_undegraded":"2023-09-23T15:23:40.505623+0000","last_fullsized":"2023-09-23T15:23:40.505623+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"322'7469","reported_seq":7534,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:44.090443+0000","last_change":"2023-09-23T15:21:58.303371+0000","last_active":"2023-09-23T15:23:44.090443+0000","last_peered":"2023-09-23T15:23:44.090443+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:23:44.090443+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236257280,"num_objects":3605,"num_object_clones":0,"num_object_copies":10815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7469,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":512},{"shards":"0,2,7","objects":1161}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":519,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505439+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:23:41.505439+0000","last_peered":"2023-09-23T15:23:41.505439+0000","last_clean":"2023-09-23T15:23:41.505439+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:23:41.505439+0000","last_undegraded":"2023-09-23T15:23:41.505439+0000","last_fullsized":"2023-09-23T15:23:41.505439+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":520,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.501775+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:23:40.501775+0000","last_peered":"2023-09-23T15:23:40.501775+0000","last_clean":"2023-09-23T15:23:40.501775+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:23:40.501775+0000","last_undegraded":"2023-09-23T15:23:40.501775+0000","last_fullsized":"2023-09-23T15:23:40.501775+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":298,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504799+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:23:41.504799+0000","last_peered":"2023-09-23T15:23:41.504799+0000","last_clean":"2023-09-23T15:23:41.504799+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:23:41.504799+0000","last_undegraded":"2023-09-23T15:23:41.504799+0000","last_fullsized":"2023-09-23T15:23:41.504799+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":265,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508786+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:23:41.508786+0000","last_peered":"2023-09-23T15:23:41.508786+0000","last_clean":"2023-09-23T15:23:41.508786+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:23:41.508786+0000","last_undegraded":"2023-09-23T15:23:41.508786+0000","last_fullsized":"2023-09-23T15:23:41.508786+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"322'7640","reported_seq":7696,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:44.815471+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:23:44.815471+0000","last_peered":"2023-09-23T15:23:44.815471+0000","last_clean":"2023-09-23T15:23:44.815471+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:23:44.815471+0000","last_undegraded":"2023-09-23T15:23:44.815471+0000","last_fullsized":"2023-09-23T15:23:44.815471+0000","mapping_epoch":287,"log_start":"291'800","ondisk_log_start":"291'800","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6840,"log_dups_size":0,"ondisk_log_size":6840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234881024,"num_objects":3584,"num_object_clones":0,"num_object_copies":10752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7640,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":619,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504490+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:23:41.504490+0000","last_peered":"2023-09-23T15:23:41.504490+0000","last_clean":"2023-09-23T15:23:41.504490+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:23:41.504490+0000","last_undegraded":"2023-09-23T15:23:41.504490+0000","last_fullsized":"2023-09-23T15:23:41.504490+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":857,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505033+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:23:41.505033+0000","last_peered":"2023-09-23T15:23:41.505033+0000","last_clean":"2023-09-23T15:23:41.505033+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:23:41.505033+0000","last_undegraded":"2023-09-23T15:23:41.505033+0000","last_fullsized":"2023-09-23T15:23:41.505033+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506381+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:23:41.506381+0000","last_peered":"2023-09-23T15:23:41.506381+0000","last_clean":"2023-09-23T15:23:41.506381+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:23:41.506381+0000","last_undegraded":"2023-09-23T15:23:41.506381+0000","last_fullsized":"2023-09-23T15:23:41.506381+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":217,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.501452+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:23:40.501452+0000","last_peered":"2023-09-23T15:23:40.501452+0000","last_clean":"2023-09-23T15:23:40.501452+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:23:40.501452+0000","last_undegraded":"2023-09-23T15:23:40.501452+0000","last_fullsized":"2023-09-23T15:23:40.501452+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"322'7580","reported_seq":13864,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:45.536570+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:23:45.536570+0000","last_peered":"2023-09-23T15:23:45.536570+0000","last_clean":"2023-09-23T15:23:45.536570+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:23:45.536570+0000","last_undegraded":"2023-09-23T15:23:45.536570+0000","last_fullsized":"2023-09-23T15:23:45.536570+0000","mapping_epoch":320,"log_start":"293'1800","ondisk_log_start":"293'1800","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":5780,"log_dups_size":0,"ondisk_log_size":5780,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231342080,"num_objects":3530,"num_object_clones":0,"num_object_copies":10590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7580,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":677,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507447+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:23:41.507447+0000","last_peered":"2023-09-23T15:23:41.507447+0000","last_clean":"2023-09-23T15:23:41.507447+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:23:41.507447+0000","last_undegraded":"2023-09-23T15:23:41.507447+0000","last_fullsized":"2023-09-23T15:23:41.507447+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":938,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509160+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:23:41.509160+0000","last_peered":"2023-09-23T15:23:41.509160+0000","last_clean":"2023-09-23T15:23:41.509160+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:23:41.509160+0000","last_undegraded":"2023-09-23T15:23:41.509160+0000","last_fullsized":"2023-09-23T15:23:41.509160+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":212,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.503130+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:23:40.503130+0000","last_peered":"2023-09-23T15:23:40.503130+0000","last_clean":"2023-09-23T15:23:40.503130+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:23:40.503130+0000","last_undegraded":"2023-09-23T15:23:40.503130+0000","last_fullsized":"2023-09-23T15:23:40.503130+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":221,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.505554+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:23:40.505554+0000","last_peered":"2023-09-23T15:23:40.505554+0000","last_clean":"2023-09-23T15:23:40.505554+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:23:40.505554+0000","last_undegraded":"2023-09-23T15:23:40.505554+0000","last_fullsized":"2023-09-23T15:23:40.505554+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"322'7691","reported_seq":7739,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:44.034204+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:23:44.034204+0000","last_peered":"2023-09-23T15:23:44.034204+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:23:44.034204+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"292'1100","ondisk_log_start":"292'1100","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240975872,"num_objects":3677,"num_object_clones":0,"num_object_copies":11031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7691,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1029}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":651,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505357+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:23:41.505357+0000","last_peered":"2023-09-23T15:23:41.505357+0000","last_clean":"2023-09-23T15:23:41.505357+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:23:41.505357+0000","last_undegraded":"2023-09-23T15:23:41.505357+0000","last_fullsized":"2023-09-23T15:23:41.505357+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":641,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507625+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:23:41.507625+0000","last_peered":"2023-09-23T15:23:41.507625+0000","last_clean":"2023-09-23T15:23:41.507625+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:23:41.507625+0000","last_undegraded":"2023-09-23T15:23:41.507625+0000","last_fullsized":"2023-09-23T15:23:41.507625+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":255,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504547+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:23:41.504547+0000","last_peered":"2023-09-23T15:23:41.504547+0000","last_clean":"2023-09-23T15:23:41.504547+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:23:41.504547+0000","last_undegraded":"2023-09-23T15:23:41.504547+0000","last_fullsized":"2023-09-23T15:23:41.504547+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":273,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504615+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:23:41.504615+0000","last_peered":"2023-09-23T15:23:41.504615+0000","last_clean":"2023-09-23T15:23:41.504615+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:23:41.504615+0000","last_undegraded":"2023-09-23T15:23:41.504615+0000","last_fullsized":"2023-09-23T15:23:41.504615+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"321'7461","reported_seq":7508,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.903719+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:23:41.903719+0000","last_peered":"2023-09-23T15:23:41.903719+0000","last_clean":"2023-09-23T15:23:41.903719+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:23:41.903719+0000","last_undegraded":"2023-09-23T15:23:41.903719+0000","last_fullsized":"2023-09-23T15:23:41.903719+0000","mapping_epoch":284,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6561,"log_dups_size":0,"ondisk_log_size":6561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241762304,"num_objects":3689,"num_object_clones":0,"num_object_copies":11067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7461,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":627,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.509103+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:23:40.509103+0000","last_peered":"2023-09-23T15:23:40.509103+0000","last_clean":"2023-09-23T15:23:40.509103+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:23:40.509103+0000","last_undegraded":"2023-09-23T15:23:40.509103+0000","last_fullsized":"2023-09-23T15:23:40.509103+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":66,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507557+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:23:41.507557+0000","last_peered":"2023-09-23T15:23:41.507557+0000","last_clean":"2023-09-23T15:23:41.507557+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:23:41.507557+0000","last_undegraded":"2023-09-23T15:23:41.507557+0000","last_fullsized":"2023-09-23T15:23:41.507557+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":289,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.510233+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:23:41.510233+0000","last_peered":"2023-09-23T15:23:41.510233+0000","last_clean":"2023-09-23T15:23:41.510233+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:23:41.510233+0000","last_undegraded":"2023-09-23T15:23:41.510233+0000","last_fullsized":"2023-09-23T15:23:41.510233+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":300,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.512454+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:23:41.512454+0000","last_peered":"2023-09-23T15:23:41.512454+0000","last_clean":"2023-09-23T15:23:41.512454+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:23:41.512454+0000","last_undegraded":"2023-09-23T15:23:41.512454+0000","last_fullsized":"2023-09-23T15:23:41.512454+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"322'7829","reported_seq":7877,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:45.609909+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:23:45.609909+0000","last_peered":"2023-09-23T15:23:45.609909+0000","last_clean":"2023-09-23T15:23:45.609909+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:23:45.609909+0000","last_undegraded":"2023-09-23T15:23:45.609909+0000","last_fullsized":"2023-09-23T15:23:45.609909+0000","mapping_epoch":284,"log_start":"293'2000","ondisk_log_start":"293'2000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":5829,"log_dups_size":0,"ondisk_log_size":5829,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243859456,"num_objects":3721,"num_object_clones":0,"num_object_copies":11163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7829,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":564,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.631706+0000","last_change":"2023-09-23T15:23:43.631706+0000","last_active":"2023-09-23T15:23:43.631706+0000","last_peered":"2023-09-23T15:23:43.631706+0000","last_clean":"2023-09-23T15:23:43.631706+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:23:43.631706+0000","last_undegraded":"2023-09-23T15:23:43.631706+0000","last_fullsized":"2023-09-23T15:23:43.631706+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1015,"reported_epoch":321,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:41.507253+0000","last_change":"2023-09-23T15:22:27.435515+0000","last_active":"2023-09-23T15:23:41.507253+0000","last_peered":"2023-09-23T15:23:41.507253+0000","last_clean":"2023-09-23T15:21:46.756689+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:23:41.507253+0000","last_undegraded":"2023-09-23T15:22:27.290223+0000","last_fullsized":"2023-09-23T15:23:41.507253+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":197,"reported_epoch":320,"state":"active+clean","last_fresh":"2023-09-23T15:23:40.501640+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:23:40.501640+0000","last_peered":"2023-09-23T15:23:40.501640+0000","last_clean":"2023-09-23T15:23:40.501640+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:23:40.501640+0000","last_undegraded":"2023-09-23T15:23:40.501640+0000","last_fullsized":"2023-09-23T15:23:40.501640+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":285,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509840+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:23:41.509840+0000","last_peered":"2023-09-23T15:23:41.509840+0000","last_clean":"2023-09-23T15:23:41.509840+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:23:41.509840+0000","last_undegraded":"2023-09-23T15:23:41.509840+0000","last_fullsized":"2023-09-23T15:23:41.509840+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"322'7473","reported_seq":7530,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.979726+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:23:43.979726+0000","last_peered":"2023-09-23T15:23:43.979726+0000","last_clean":"2023-09-23T15:23:43.979726+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:23:43.979726+0000","last_undegraded":"2023-09-23T15:23:43.979726+0000","last_fullsized":"2023-09-23T15:23:43.979726+0000","mapping_epoch":287,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6573,"log_dups_size":0,"ondisk_log_size":6573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239009792,"num_objects":3647,"num_object_clones":0,"num_object_copies":10941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7473,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":630,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507428+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:23:41.507428+0000","last_peered":"2023-09-23T15:23:41.507428+0000","last_clean":"2023-09-23T15:23:41.507428+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:23:41.507428+0000","last_undegraded":"2023-09-23T15:23:41.507428+0000","last_fullsized":"2023-09-23T15:23:41.507428+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":77,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508610+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:23:41.508610+0000","last_peered":"2023-09-23T15:23:41.508610+0000","last_clean":"2023-09-23T15:23:41.508610+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:23:41.508610+0000","last_undegraded":"2023-09-23T15:23:41.508610+0000","last_fullsized":"2023-09-23T15:23:41.508610+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":852,"reported_epoch":320,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:40.502081+0000","last_change":"2023-09-23T15:22:27.316872+0000","last_active":"2023-09-23T15:23:40.502081+0000","last_peered":"2023-09-23T15:23:40.502081+0000","last_clean":"2023-09-23T15:21:04.172426+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:23:40.502081+0000","last_undegraded":"2023-09-23T15:22:27.289016+0000","last_fullsized":"2023-09-23T15:23:40.502081+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504612+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:23:41.504612+0000","last_peered":"2023-09-23T15:23:41.504612+0000","last_clean":"2023-09-23T15:23:41.504612+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:23:41.504612+0000","last_undegraded":"2023-09-23T15:23:41.504612+0000","last_fullsized":"2023-09-23T15:23:41.504612+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"322'7614","reported_seq":7668,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:44.864242+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:23:44.864242+0000","last_peered":"2023-09-23T15:23:44.864242+0000","last_clean":"2023-09-23T15:23:44.864242+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:23:44.864242+0000","last_undegraded":"2023-09-23T15:23:44.864242+0000","last_fullsized":"2023-09-23T15:23:44.864242+0000","mapping_epoch":287,"log_start":"291'700","ondisk_log_start":"291'700","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6914,"log_dups_size":0,"ondisk_log_size":6914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":228655128,"num_objects":3490,"num_object_clones":0,"num_object_copies":10470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3490,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7614,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":646,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504802+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:23:41.504802+0000","last_peered":"2023-09-23T15:23:41.504802+0000","last_clean":"2023-09-23T15:23:41.504802+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:23:41.504802+0000","last_undegraded":"2023-09-23T15:23:41.504802+0000","last_fullsized":"2023-09-23T15:23:41.504802+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":758,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505241+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:23:41.505241+0000","last_peered":"2023-09-23T15:23:41.505241+0000","last_clean":"2023-09-23T15:23:41.505241+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:23:41.505241+0000","last_undegraded":"2023-09-23T15:23:41.505241+0000","last_fullsized":"2023-09-23T15:23:41.505241+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":327,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.512538+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:23:41.512538+0000","last_peered":"2023-09-23T15:23:41.512538+0000","last_clean":"2023-09-23T15:23:41.512538+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:23:41.512538+0000","last_undegraded":"2023-09-23T15:23:41.512538+0000","last_fullsized":"2023-09-23T15:23:41.512538+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505631+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:23:41.505631+0000","last_peered":"2023-09-23T15:23:41.505631+0000","last_clean":"2023-09-23T15:23:41.505631+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:23:41.505631+0000","last_undegraded":"2023-09-23T15:23:41.505631+0000","last_fullsized":"2023-09-23T15:23:41.505631+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":304,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.509758+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:23:41.509758+0000","last_peered":"2023-09-23T15:23:41.509758+0000","last_clean":"2023-09-23T15:23:41.509758+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:23:41.509758+0000","last_undegraded":"2023-09-23T15:23:41.509758+0000","last_fullsized":"2023-09-23T15:23:41.509758+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":325,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.508838+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:23:41.508838+0000","last_peered":"2023-09-23T15:23:41.508838+0000","last_clean":"2023-09-23T15:23:41.508838+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:23:41.508838+0000","last_undegraded":"2023-09-23T15:23:41.508838+0000","last_fullsized":"2023-09-23T15:23:41.508838+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":335,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506252+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:23:41.506252+0000","last_peered":"2023-09-23T15:23:41.506252+0000","last_clean":"2023-09-23T15:23:41.506252+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:23:41.506252+0000","last_undegraded":"2023-09-23T15:23:41.506252+0000","last_fullsized":"2023-09-23T15:23:41.506252+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506543+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:23:41.506543+0000","last_peered":"2023-09-23T15:23:41.506543+0000","last_clean":"2023-09-23T15:23:41.506543+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:23:41.506543+0000","last_undegraded":"2023-09-23T15:23:41.506543+0000","last_fullsized":"2023-09-23T15:23:41.506543+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504711+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:23:41.504711+0000","last_peered":"2023-09-23T15:23:41.504711+0000","last_clean":"2023-09-23T15:23:41.504711+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:23:41.504711+0000","last_undegraded":"2023-09-23T15:23:41.504711+0000","last_fullsized":"2023-09-23T15:23:41.504711+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505852+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:23:41.505852+0000","last_peered":"2023-09-23T15:23:41.505852+0000","last_clean":"2023-09-23T15:23:41.505852+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:23:41.505852+0000","last_undegraded":"2023-09-23T15:23:41.505852+0000","last_fullsized":"2023-09-23T15:23:41.505852+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":312,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504954+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:23:41.504954+0000","last_peered":"2023-09-23T15:23:41.504954+0000","last_clean":"2023-09-23T15:23:41.504954+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:23:41.504954+0000","last_undegraded":"2023-09-23T15:23:41.504954+0000","last_fullsized":"2023-09-23T15:23:41.504954+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":311,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504544+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:23:41.504544+0000","last_peered":"2023-09-23T15:23:41.504544+0000","last_clean":"2023-09-23T15:23:41.504544+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:23:41.504544+0000","last_undegraded":"2023-09-23T15:23:41.504544+0000","last_fullsized":"2023-09-23T15:23:41.504544+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":274,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.510325+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:23:41.510325+0000","last_peered":"2023-09-23T15:23:41.510325+0000","last_clean":"2023-09-23T15:23:41.510325+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:23:41.510325+0000","last_undegraded":"2023-09-23T15:23:41.510325+0000","last_fullsized":"2023-09-23T15:23:41.510325+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505325+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:23:41.505325+0000","last_peered":"2023-09-23T15:23:41.505325+0000","last_clean":"2023-09-23T15:23:41.505325+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:23:41.505325+0000","last_undegraded":"2023-09-23T15:23:41.505325+0000","last_fullsized":"2023-09-23T15:23:41.505325+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3817930776,"num_objects":58258,"num_object_clones":0,"num_object_copies":174774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":28444,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58258,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122012,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8550,"num_bytes_recovered":560332800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11751075840,"data_stored":11751063624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104112,"ondisk_log_size":104112,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201628,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1489824,"kb_used_data":1358744,"kb_used_omap":0,"kb_used_meta":131072,"kb_avail":92255328,"statfs":{"total":95995035648,"available":94469455872,"internally_reserved":0,"allocated":1391353856,"data_stored":1390371595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":134217728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":7.2160000000000002,"15min":3.8849999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":7.7629999999999999,"5min":226.226,"15min":226.226},"last":7.3239999999999998},{"interface":"front","average":{"1min":2.9399999999999999,"5min":8.5329999999999995,"15min":4.5190000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":9.4749999999999996,"5min":226.619,"15min":226.619},"last":10.073}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.3680000000000001,"15min":2.1480000000000001},"min":{"1min":0.32600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":3.6850000000000001},{"interface":"front","average":{"1min":1.768,"5min":1.585,"15min":2.569},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1840000000000002,"5min":7.2510000000000003,"15min":16.065999999999999},"last":3.0310000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.4630000000000001,"15min":3.9340000000000002},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":6.6799999999999997,"5min":90.227999999999994,"15min":170.63800000000001},"last":3.5819999999999999},{"interface":"front","average":{"1min":1.889,"5min":2.778,"15min":4.5090000000000003},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2059999999999995,"5min":90.197999999999993,"15min":170.47999999999999},"last":10.127000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:22:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":4.8659999999999997,"15min":3.7090000000000001},"min":{"1min":0.36399999999999999,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":12.379,"5min":146.166,"15min":146.166},"last":3.6480000000000001},{"interface":"front","average":{"1min":2.677,"5min":5.0460000000000003,"15min":3.7250000000000001},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":12.362,"5min":146.18299999999999,"15min":146.18299999999999},"last":3.6139999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.74099999999999999,"15min":2.4809999999999999},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":16.771999999999998,"5min":16.771999999999998,"15min":150.119},"last":1.8560000000000001},{"interface":"front","average":{"1min":0.99099999999999999,"5min":0.56799999999999995,"15min":2.6930000000000001},"min":{"1min":0.28699999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.4710000000000001,"5min":4.4710000000000001,"15min":150.596},"last":2.8359999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:22:43 2023","interfaces":[{"interface":"back","average":{"1min":10.605,"5min":2.5329999999999999,"15min":2.7799999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":224.441,"5min":224.441,"15min":224.441},"last":0.41099999999999998},{"interface":"front","average":{"1min":10.952,"5min":2.6339999999999999,"15min":2.8090000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":224.28700000000001,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.46000000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.2320000000000002,"15min":2.2320000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.2629999999999999,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":2.879},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.336,"5min":10.336,"15min":10.336},"last":0.65600000000000003}]}]},{"osd":6,"up_from":304,"seq":1305670058001,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":387092,"kb_used_data":322188,"kb_used_omap":13,"kb_used_meta":64882,"kb_avail":93358060,"statfs":{"total":95995035648,"available":95598653440,"internally_reserved":0,"allocated":329920512,"data_stored":328950388,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":66439523},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":5.7649999999999997},"last":2.464},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.8780000000000001,"15min":2.8780000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.044,"5min":10.044,"15min":10.044},"last":0.48099999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.742,"15min":2.742},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.064,"5min":11.064,"15min":11.064},"last":4.1239999999999997},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":17.001999999999999,"5min":17.001999999999999,"15min":17.001999999999999},"last":2.5369999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.526,"15min":1.526},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":2.0990000000000002},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":4.2190000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":2.2799999999999998,"15min":2.2799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.0449999999999999,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":15.926},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5049999999999999,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.1550000000000002,"5min":5.1550000000000002,"15min":5.1550000000000002},"last":2.5059999999999998},{"interface":"front","average":{"1min":1.821,"5min":1.821,"15min":1.821},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":9.1929999999999996},"last":0.40799999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":12.711,"5min":12.711,"15min":12.711},"last":4.1779999999999999},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.1950000000000003,"5min":5.1950000000000003,"15min":5.1950000000000003},"last":15.868}]},{"osd":7,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.0609999999999999,"5min":5.0609999999999999,"15min":5.0609999999999999},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.823,"5min":1.823,"15min":1.823},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":8.5340000000000007}]}]},{"osd":1,"up_from":183,"seq":785979015301,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2329876,"kb_used_data":1891404,"kb_used_omap":6,"kb_used_meta":438457,"kb_avail":91415276,"statfs":{"total":95995035648,"available":93609242624,"internally_reserved":0,"allocated":1936797696,"data_stored":1935302733,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":448980043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":3.9390000000000001,"15min":3.5049999999999999},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":9.5739999999999998,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.96299999999999997},{"interface":"front","average":{"1min":4.0019999999999998,"5min":4.0300000000000002,"15min":4.2030000000000003},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.537000000000001,"5min":63.594999999999999,"15min":63.594999999999999},"last":2.2890000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":4.2759999999999998,"15min":4.1550000000000002},"min":{"1min":0.26900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.1600000000000001,"5min":109.375,"15min":109.375},"last":4.4420000000000002},{"interface":"front","average":{"1min":2.7269999999999999,"5min":3.7309999999999999,"15min":3.722},"min":{"1min":0.32600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":9.6259999999999994,"5min":105.16,"15min":105.16},"last":0.92900000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":6.399,"15min":8.2629999999999999},"min":{"1min":0.33100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":64.540999999999997,"5min":110.285,"15min":211.405},"last":2.024},{"interface":"front","average":{"1min":6.4770000000000003,"5min":6.6900000000000004,"15min":9.0289999999999999},"min":{"1min":0.31900000000000001,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":67.465000000000003,"5min":110.095,"15min":210.988},"last":2.0089999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":3.3580000000000001,"15min":3.294},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3289999999999997,"5min":80.570999999999998,"15min":80.570999999999998},"last":3.2890000000000001},{"interface":"front","average":{"1min":2.7040000000000002,"5min":3.4550000000000001,"15min":3.5249999999999999},"min":{"1min":0.38100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":9.6600000000000001,"5min":80.75,"15min":80.75},"last":3.456}]},{"osd":5,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":2.5110000000000001,"15min":3.0649999999999999},"min":{"1min":0.39500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.3659999999999997,"5min":34.512999999999998,"15min":34.512999999999998},"last":2.339},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.5870000000000002,"15min":2.8639999999999999},"min":{"1min":0.37,"5min":0.311,"15min":0.311},"max":{"1min":8.9559999999999995,"5min":37.911000000000001,"15min":37.911000000000001},"last":3.23}]},{"osd":6,"last update":"Sat Sep 23 15:23:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.0710000000000002,"15min":3.0710000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":12.26,"5min":12.26,"15min":12.26},"last":3.3199999999999998},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.3230000000000004,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":0.71999999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":2.2549999999999999,"15min":2.8050000000000002},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":6.0339999999999998,"5min":12.861000000000001,"15min":12.861000000000001},"last":3.351},{"interface":"front","average":{"1min":2.6120000000000001,"5min":2.0510000000000002,"15min":2.6400000000000001},"min":{"1min":0.442,"5min":0.377,"15min":0.377},"max":{"1min":9.6929999999999996,"5min":13.282999999999999,"15min":16.841000000000001},"last":0.89700000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893351,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2273712,"kb_used_data":1701032,"kb_used_omap":8,"kb_used_meta":572663,"kb_avail":91471440,"statfs":{"total":95995035648,"available":93666754560,"internally_reserved":0,"allocated":1741856768,"data_stored":1740469180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":586407508},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8069999999999999,"5min":3.5179999999999998,"15min":3.5710000000000002},"min":{"1min":0.23899999999999999,"5min":0.19500000000000001,"15min":0.18099999999999999},"max":{"1min":11.718,"5min":64.412999999999997,"15min":64.412999999999997},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.5419999999999998,"15min":3.4790000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":11.015000000000001,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.35499999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":2.7759999999999998,"15min":4.5229999999999997},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":10.949,"5min":30.510999999999999,"15min":122.20099999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.3069999999999999,"15min":4.3010000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.18099999999999999},"max":{"1min":10.861000000000001,"5min":64.466999999999999,"15min":122.071},"last":0.371}]},{"osd":3,"last update":"Sat Sep 23 15:22:45 2023","interfaces":[{"interface":"back","average":{"1min":2.9889999999999999,"5min":3.4289999999999998,"15min":5.0250000000000004},"min":{"1min":0.32100000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":10.989000000000001,"5min":65.018000000000001,"15min":146.315},"last":0.39100000000000001},{"interface":"front","average":{"1min":2.1200000000000001,"5min":4.2679999999999998,"15min":5.46},"min":{"1min":0.30099999999999999,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":6.7839999999999998,"5min":64.441000000000003,"15min":145.517},"last":0.41399999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":1.8899999999999999,"15min":2.1419999999999999},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.358000000000001,"5min":15.358000000000001,"15min":31.341999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.964,"15min":2.246},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":99.168999999999997},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.492,"15min":2.411},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.768000000000001,"5min":64.349999999999994,"15min":109.369},"last":0.436},{"interface":"front","average":{"1min":3.6789999999999998,"5min":1.9990000000000001,"15min":2.355},"min":{"1min":0.497,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":15.436999999999999,"5min":15.436999999999999,"15min":109.387},"last":0.48199999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":3.3610000000000002,"5min":3.3610000000000002,"15min":3.3610000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":17.385000000000002,"5min":17.385000000000002,"15min":17.385000000000002},"last":0.61099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.5550000000000002,"15min":2.488},"min":{"1min":0.42499999999999999,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":11.734,"5min":64.304000000000002,"15min":99.549999999999997},"last":0.49399999999999999},{"interface":"front","average":{"1min":4.9279999999999999,"5min":2.3039999999999998,"15min":2.1720000000000002},"min":{"1min":0.45900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":31.225999999999999},"last":0.44400000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566298,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2507676,"kb_used_data":1910868,"kb_used_omap":11,"kb_used_meta":596788,"kb_avail":91237476,"statfs":{"total":95995035648,"available":93427175424,"internally_reserved":0,"allocated":1956728832,"data_stored":1955354515,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":611111696},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":6.9870000000000001,"5min":5.4580000000000002,"15min":4.1790000000000003},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":64.022000000000006,"5min":81.218999999999994,"15min":87.132999999999996},"last":88.622},{"interface":"front","average":{"1min":7.8109999999999999,"5min":5.6849999999999996,"15min":3.863},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":64.091999999999999,"5min":81.352000000000004,"15min":84.820999999999998},"last":88.649000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.8399999999999999,"15min":4.5869999999999997},"min":{"1min":0.22800000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":18.094000000000001,"5min":44.508000000000003,"15min":84.641000000000005},"last":0.95599999999999996},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4319999999999999,"15min":3.3090000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":18.628,"5min":45.765999999999998,"15min":84.751999999999995},"last":0.77900000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:22:46 2023","interfaces":[{"interface":"back","average":{"1min":6.1959999999999997,"5min":5.5940000000000003,"15min":4.9859999999999998},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":70.445999999999998,"5min":70.445999999999998,"15min":87.108000000000004},"last":0.34200000000000003},{"interface":"front","average":{"1min":6.1689999999999996,"5min":6.1820000000000004,"15min":5.0529999999999999},"min":{"1min":0.30599999999999999,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":70.317999999999998,"5min":70.317999999999998,"15min":86.158000000000001},"last":0.97799999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.716,"15min":2.343},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":5.798,"5min":11.452999999999999,"15min":84.608000000000004},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.595,"5min":1.7410000000000001,"15min":2.968},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":5.8380000000000001,"5min":13.106,"15min":87.019999999999996},"last":0.376}]},{"osd":5,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.9019999999999999,"15min":2.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.2400000000000002,"5min":8.9909999999999997,"15min":84.498000000000005},"last":0.90700000000000003},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9810000000000001,"15min":2.5129999999999999},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":9.8780000000000001,"5min":15.891,"15min":84.664000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.2839999999999998,"15min":2.2839999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4329999999999998,"5min":7.4329999999999998,"15min":7.4329999999999998},"last":0.93300000000000005},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.42499999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.0470000000000002,"15min":2.5649999999999999},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":10.227,"5min":10.227,"15min":84.596000000000004},"last":1},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.0840000000000001,"15min":2.4249999999999998},"min":{"1min":0.504,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":7.0099999999999998,"5min":15.933999999999999,"15min":84.783000000000001},"last":0.83399999999999996}]}]},{"osd":3,"up_from":110,"seq":472446402772,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2166176,"kb_used_data":1587928,"kb_used_omap":3,"kb_used_meta":578236,"kb_avail":91578976,"statfs":{"total":95995035648,"available":93776871424,"internally_reserved":0,"allocated":1626038272,"data_stored":1624660667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":592113833},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":3.113,"15min":3.1880000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":7.2939999999999996,"5min":33.220999999999997,"15min":79.611000000000004},"last":2.169},{"interface":"front","average":{"1min":2.8300000000000001,"5min":3.512,"15min":3.4329999999999998},"min":{"1min":0.21099999999999999,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":36.465000000000003,"5min":36.465000000000003,"15min":79.656000000000006},"last":2.1539999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:23:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.0830000000000002,"15min":4.9660000000000002},"min":{"1min":0.23799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":126.875,"5min":126.875,"15min":126.875},"last":7.5739999999999998},{"interface":"front","average":{"1min":11.324999999999999,"5min":5.8259999999999996,"15min":5.2320000000000002},"min":{"1min":0.33600000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":128.249,"5min":128.249,"15min":128.249},"last":8.4169999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.3109999999999999,"15min":4.6440000000000001},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":7.3449999999999998,"5min":17.199999999999999,"15min":109.077},"last":2.2109999999999999},{"interface":"front","average":{"1min":3.569,"5min":4.4249999999999998,"15min":5.6589999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":36.582000000000001,"5min":119.449,"15min":119.72799999999999},"last":2.8460000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.1339999999999999,"15min":2.1869999999999998},"min":{"1min":0.33400000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":36.551000000000002,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.589,"15min":2.0960000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":9.1839999999999993,"5min":12.500999999999999,"15min":52.228999999999999},"last":8.0050000000000008}]},{"osd":5,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":3.5390000000000001,"15min":2.7829999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":36.369,"5min":148.619,"15min":148.619},"last":9.5139999999999993},{"interface":"front","average":{"1min":1.8660000000000001,"5min":2.9740000000000002,"15min":2.7349999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":9.1430000000000007,"5min":148.56999999999999,"15min":148.56999999999999},"last":6.1710000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.2200000000000006,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":2.496},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":36.517000000000003,"5min":36.517000000000003,"15min":36.517000000000003},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":3.0419999999999998,"15min":2.6880000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":9.2569999999999997,"5min":145.25,"15min":145.25},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.7370000000000001,"5min":3.1080000000000001,"15min":2.4420000000000002},"min":{"1min":0.40400000000000003,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":7.4509999999999996,"5min":145.00899999999999,"15min":145.00899999999999},"last":2.3010000000000002}]}]},{"osd":4,"up_from":31,"seq":133143986537,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1257944,"kb_used_data":941776,"kb_used_omap":0,"kb_used_meta":316160,"kb_avail":92487208,"statfs":{"total":95995035648,"available":94706900992,"internally_reserved":0,"allocated":964378624,"data_stored":963404461,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":323747840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":5.7130000000000001,"15min":4.0220000000000002},"min":{"1min":0.23599999999999999,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":5.3170000000000002,"5min":162.75999999999999,"15min":162.75999999999999},"last":0.92500000000000004},{"interface":"front","average":{"1min":2.5699999999999998,"5min":5.6079999999999997,"15min":4.1219999999999999},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":8.7289999999999992,"5min":162.34800000000001,"15min":162.34800000000001},"last":0.94799999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":3.3130000000000002,"15min":3.419},"min":{"1min":0.249,"5min":0.245,"15min":0.245},"max":{"1min":9.5350000000000001,"5min":113.11199999999999,"15min":113.11199999999999},"last":1.26},{"interface":"front","average":{"1min":2.746,"5min":3.4660000000000002,"15min":3.851},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":12.726000000000001,"5min":113.02,"15min":113.02},"last":2.9750000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":3.734,"15min":4.5199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":7.016,"5min":61.646000000000001,"15min":148.08000000000001},"last":3.496},{"interface":"front","average":{"1min":3.488,"5min":3.8290000000000002,"15min":4.758},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":6.9980000000000002,"5min":64.840000000000003,"15min":147.88900000000001},"last":3.145}]},{"osd":3,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":6.306,"15min":5.9610000000000003},"min":{"1min":0.39400000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":5.3860000000000001,"5min":156.46000000000001,"15min":163.97499999999999},"last":1.323},{"interface":"front","average":{"1min":1.8600000000000001,"5min":6.641,"15min":5.8760000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27800000000000002},"max":{"1min":6.2510000000000003,"5min":156.74299999999999,"15min":164.447},"last":0.32700000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":0.91000000000000003,"15min":1.633},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":6.4409999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":0.81299999999999994},{"interface":"front","average":{"1min":3.1680000000000001,"5min":1.131,"15min":1.573},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":14.449999999999999,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.36699999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:23:28 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.446,"15min":1.446},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6470000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.39600000000000002},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.1369999999999996,"5min":6.1369999999999996,"15min":6.1369999999999996},"last":0.89400000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:22:44 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":0.68200000000000005,"15min":1.135},"min":{"1min":0.31,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":6.0730000000000004,"5min":6.0730000000000004,"15min":18.23},"last":0.98099999999999998},{"interface":"front","average":{"1min":1.5129999999999999,"5min":0.80200000000000005,"15min":1.232},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":5.3520000000000003,"5min":5.3520000000000003,"15min":14.653},"last":0.85699999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724900,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2225680,"kb_used_data":1860168,"kb_used_omap":0,"kb_used_meta":365504,"kb_avail":91519472,"statfs":{"total":95995035648,"available":93715939328,"internally_reserved":0,"allocated":1904812032,"data_stored":1903842322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":374276096},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.8639999999999999,"15min":3.2309999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":4.1059999999999999,"5min":124.31,"15min":124.31},"last":6.1639999999999997},{"interface":"front","average":{"1min":2.6349999999999998,"5min":4.8070000000000004,"15min":3.2799999999999998},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":9.3889999999999993,"5min":124.374,"15min":124.374},"last":1.667}]},{"osd":1,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":3.8199999999999998,"15min":2.5880000000000001},"min":{"1min":0.36199999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3149999999999995,"5min":124.38800000000001,"15min":124.38800000000001},"last":6.2169999999999996},{"interface":"front","average":{"1min":2.165,"5min":4.0679999999999996,"15min":2.8919999999999999},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":5.8769999999999998,"5min":124.29000000000001,"15min":124.29000000000001},"last":1.6339999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":3.77,"15min":3.7210000000000001},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":8.3030000000000008,"5min":193.00999999999999,"15min":193.00999999999999},"last":8.4979999999999993},{"interface":"front","average":{"1min":2.4550000000000001,"5min":4.0960000000000001,"15min":3.8340000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":9.3420000000000005,"5min":193.15700000000001,"15min":193.15700000000001},"last":1.7250000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:22:43 2023","interfaces":[{"interface":"back","average":{"1min":1.8540000000000001,"5min":6.1600000000000001,"15min":5.0030000000000001},"min":{"1min":0.36299999999999999,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":5.6799999999999997,"5min":186.56999999999999,"15min":186.56999999999999},"last":8.5449999999999999},{"interface":"front","average":{"1min":1.4410000000000001,"5min":6.1079999999999997,"15min":4.7679999999999998},"min":{"1min":0.34899999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.548,"5min":186.74600000000001,"15min":186.74600000000001},"last":6.2480000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":0.73399999999999999,"15min":1.8380000000000001},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":7.3440000000000003,"5min":7.3440000000000003,"15min":138.68600000000001},"last":6.282},{"interface":"front","average":{"1min":1.641,"5min":0.754,"15min":1.9019999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":138.49600000000001},"last":1.748}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.3969999999999998,"15min":2.3969999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.6050000000000004,"5min":9.6050000000000004,"15min":9.6050000000000004},"last":1.694},{"interface":"front","average":{"1min":2.4100000000000001,"5min":2.4100000000000001,"15min":2.4100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":13.257999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:22:43 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53800000000000003,"15min":1.2589999999999999},"min":{"1min":0.30399999999999999,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":1.972,"5min":2.113,"15min":15.301},"last":12.065},{"interface":"front","average":{"1min":0.60499999999999998,"5min":0.59899999999999998,"15min":1.2410000000000001},"min":{"1min":0.374,"5min":0.34100000000000003,"15min":0.311},"max":{"1min":2.173,"5min":3.0369999999999999,"15min":15.762},"last":6.3079999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1731723264,"data_stored":1731723264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1921187840,"data_stored":1921187840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1937575936,"data_stored":1937571864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1614217216,"data_stored":1614217216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":958005248,"data_stored":958005248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1883901952,"data_stored":1883897880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326107136,"data_stored":326107136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1378357248,"data_stored":1378353176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:23:48.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:23:48.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:48 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:48.956 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:48.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:48.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:49.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:48 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:49.028 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:48 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:49.075 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:49.179 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:49.298 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:49.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:49.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:49.385 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:49.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:49.479 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:49.630 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:23:49.631 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:23:49.631 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:23:49.631 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:23:49.631 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:23:49.752 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:49.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:49.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:49.857 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:23:49.914 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:49 smithi027 ceph-mon[195233]: pgmap v934: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 28461/172578 objects degraded (16.492%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:23:49.914 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:49 smithi027 ceph-mon[195233]: 1.6 deep-scrub starts 2023-09-23T15:23:49.927 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:49.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:49 smithi017 ceph-mon[215646]: pgmap v934: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 28461/172578 objects degraded (16.492%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:23:49.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:49 smithi017 ceph-mon[215646]: 1.6 deep-scrub starts 2023-09-23T15:23:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:49 smithi017 ceph-mon[222878]: pgmap v934: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 28461/172578 objects degraded (16.492%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:23:49.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:49 smithi017 ceph-mon[222878]: 1.6 deep-scrub starts 2023-09-23T15:23:50.049 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:23:50.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:50.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:23:50.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:50.210 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:50.266 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:50.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:50.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:50.374 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:50.447 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:50.540 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:50.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:50.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:50.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:50.670 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:23:50.755 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:50.768 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:23:50.870 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:50.892 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:50 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:23:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:50 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:23:50.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:51.057 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:51.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:23:51.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:50 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:23:51.343 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:51.467 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:51.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:51.472 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:51.488 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:51.573 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:51.661 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:51.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: pgmap v935: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 27892/165897 objects degraded (16.813%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T15:23:51.744 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:51.761 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:51.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:51.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: pgmap v935: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 27892/165897 objects degraded (16.813%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: failed to encode map e323 with expected crc 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: failed to encode map e323 with expected crc 2023-09-23T15:23:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: failed to encode map e323 with expected crc 2023-09-23T15:23:51.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:51 smithi027 ceph-mon[195233]: failed to encode map e323 with expected crc 2023-09-23T15:23:51.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:51.930 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:52.055 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: pgmap v935: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 27892/165897 objects degraded (16.813%); 5.4 MiB/s, 87 objects/s recovering 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[222878]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.056 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1892939290' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.057 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:51 smithi017 ceph-mon[215646]: failed to encode map e323 with expected crc 2023-09-23T15:23:52.088 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:52.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:52.194 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:52.268 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:52.402 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:52.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:52.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:52.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:52.544 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:52.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:52.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:52.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:52.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:52.822 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:23:52.845 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27892/165897 objects degraded (16.813%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:52.847 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/733246038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:52.847 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27892/165897 objects degraded (16.813%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:52.848 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/733246038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:52.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:52.892 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:23:52.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:23:52.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:52.955 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.956 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:23:52.957 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:23:53.016 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:53.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27892/165897 objects degraded (16.813%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:53.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/733246038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:23:53.705 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:53.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:53.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:53.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:53.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:23:53.875 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:53.942 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:53.942 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:54.057 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:54.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:54.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:54.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:23:54.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:54.340 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:54 smithi017 ceph-mon[215646]: pgmap v937: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 27892/163440 objects degraded (17.066%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:54.340 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:54 smithi017 ceph-mon[222878]: pgmap v937: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 27892/163440 objects degraded (17.066%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:54.355 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:54 smithi027 ceph-mon[195233]: pgmap v937: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 597 op/s; 27892/163440 objects degraded (17.066%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:23:54.419 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:23:54.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:54.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:23:54.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:23:54.573 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:54.642 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:23:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:54.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:23:54.646 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:54.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:23:54.808 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:23:54.890 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:23:54.923 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:23:54.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:54.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:23:55.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:55.101 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:55.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:23:55.959 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:23:55.997 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:56.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:56.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:56.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:23:56.102 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:56.133 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:56.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:56 smithi017 ceph-mon[215646]: pgmap v938: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 27538/160779 objects degraded (17.128%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:56.221 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:56 smithi017 ceph-mon[215646]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:56.329 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:56.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:56.392 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:56 smithi027 ceph-mon[195233]: pgmap v938: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 27538/160779 objects degraded (17.128%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:56.392 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:56 smithi027 ceph-mon[195233]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:56.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:56.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:56 smithi017 ceph-mon[222878]: pgmap v938: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 27538/160779 objects degraded (17.128%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T15:23:56.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:56 smithi017 ceph-mon[222878]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:23:56.518 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:56.601 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":938,"stamp":"2023-09-23T15:23:54.725521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3515214193,"num_objects":53593,"num_object_clones":0,"num_object_copies":160779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27538,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53593,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":127634,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9247,"num_bytes_recovered":606702965,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109018,"ondisk_log_size":109018,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":13849152,"kb_used_data":10772864,"kb_used_omap":44,"kb_used_meta":3075667,"kb_avail":736112064,"statfs":{"total":767960285184,"available":753778753536,"internally_reserved":0,"allocated":11031412736,"data_stored":11021840348,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3149483963},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,8,2,3,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384106496,"num_objects":-5861,"num_object_clones":0,"num_object_copies":-17583,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1196,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-5861,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5861,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":842,"num_bytes_recovered":55181312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003746"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505792+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:23:41.505792+0000","last_peered":"2023-09-23T15:23:41.505792+0000","last_clean":"2023-09-23T15:23:41.505792+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:23:41.505792+0000","last_undegraded":"2023-09-23T15:23:41.505792+0000","last_fullsized":"2023-09-23T15:23:41.505792+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":278,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.161541+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:23:43.161541+0000","last_peered":"2023-09-23T15:23:43.161541+0000","last_clean":"2023-09-23T15:23:43.161541+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:23:43.161541+0000","last_undegraded":"2023-09-23T15:23:43.161541+0000","last_fullsized":"2023-09-23T15:23:43.161541+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":227,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.168008+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:23:43.168008+0000","last_peered":"2023-09-23T15:23:43.168008+0000","last_clean":"2023-09-23T15:23:43.168008+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:23:43.168008+0000","last_undegraded":"2023-09-23T15:23:43.168008+0000","last_fullsized":"2023-09-23T15:23:43.168008+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":301,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.191992+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:23:43.191992+0000","last_peered":"2023-09-23T15:23:43.191992+0000","last_clean":"2023-09-23T15:23:43.191992+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:23:43.191992+0000","last_undegraded":"2023-09-23T15:23:43.191992+0000","last_fullsized":"2023-09-23T15:23:43.191992+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":323,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504511+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:23:41.504511+0000","last_peered":"2023-09-23T15:23:41.504511+0000","last_clean":"2023-09-23T15:23:41.504511+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:23:41.504511+0000","last_undegraded":"2023-09-23T15:23:41.504511+0000","last_fullsized":"2023-09-23T15:23:41.504511+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":273,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167183+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:23:43.167183+0000","last_peered":"2023-09-23T15:23:43.167183+0000","last_clean":"2023-09-23T15:23:43.167183+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:23:43.167183+0000","last_undegraded":"2023-09-23T15:23:43.167183+0000","last_fullsized":"2023-09-23T15:23:43.167183+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":270,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.187990+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:23:43.187990+0000","last_peered":"2023-09-23T15:23:43.187990+0000","last_clean":"2023-09-23T15:23:43.187990+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:23:43.187990+0000","last_undegraded":"2023-09-23T15:23:43.187990+0000","last_fullsized":"2023-09-23T15:23:43.187990+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":320,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504956+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:23:41.504956+0000","last_peered":"2023-09-23T15:23:41.504956+0000","last_clean":"2023-09-23T15:23:41.504956+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:23:41.504956+0000","last_undegraded":"2023-09-23T15:23:41.504956+0000","last_fullsized":"2023-09-23T15:23:41.504956+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":681,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:49.214208+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:23:49.214208+0000","last_peered":"2023-09-23T15:23:49.214208+0000","last_clean":"2023-09-23T15:23:49.214208+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:23:49.214208+0000","last_undegraded":"2023-09-23T15:23:49.214208+0000","last_fullsized":"2023-09-23T15:23:49.214208+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":322,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505494+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:23:41.505494+0000","last_peered":"2023-09-23T15:23:41.505494+0000","last_clean":"2023-09-23T15:23:41.505494+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:23:41.505494+0000","last_undegraded":"2023-09-23T15:23:41.505494+0000","last_fullsized":"2023-09-23T15:23:41.505494+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":274,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504574+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:23:41.504574+0000","last_peered":"2023-09-23T15:23:41.504574+0000","last_clean":"2023-09-23T15:23:41.504574+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:23:41.504574+0000","last_undegraded":"2023-09-23T15:23:41.504574+0000","last_fullsized":"2023-09-23T15:23:41.504574+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":67,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.175413+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:23:43.175413+0000","last_peered":"2023-09-23T15:23:43.175413+0000","last_clean":"2023-09-23T15:23:43.175413+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:23:43.175413+0000","last_undegraded":"2023-09-23T15:23:43.175413+0000","last_fullsized":"2023-09-23T15:23:43.175413+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":213,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.176232+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:23:43.176232+0000","last_peered":"2023-09-23T15:23:43.176232+0000","last_clean":"2023-09-23T15:23:43.176232+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:23:43.176232+0000","last_undegraded":"2023-09-23T15:23:43.176232+0000","last_fullsized":"2023-09-23T15:23:43.176232+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":56,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506960+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:23:41.506960+0000","last_peered":"2023-09-23T15:23:41.506960+0000","last_clean":"2023-09-23T15:23:41.506960+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:23:41.506960+0000","last_undegraded":"2023-09-23T15:23:41.506960+0000","last_fullsized":"2023-09-23T15:23:41.506960+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506468+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:23:41.506468+0000","last_peered":"2023-09-23T15:23:41.506468+0000","last_clean":"2023-09-23T15:23:41.506468+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:23:41.506468+0000","last_undegraded":"2023-09-23T15:23:41.506468+0000","last_fullsized":"2023-09-23T15:23:41.506468+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":632,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.183769+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:23:43.183769+0000","last_peered":"2023-09-23T15:23:43.183769+0000","last_clean":"2023-09-23T15:23:43.183769+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:23:43.183769+0000","last_undegraded":"2023-09-23T15:23:43.183769+0000","last_fullsized":"2023-09-23T15:23:43.183769+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":71,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.164800+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:23:43.164800+0000","last_peered":"2023-09-23T15:23:43.164800+0000","last_clean":"2023-09-23T15:23:43.164800+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:23:43.164800+0000","last_undegraded":"2023-09-23T15:23:43.164800+0000","last_fullsized":"2023-09-23T15:23:43.164800+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":325,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505025+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:23:41.505025+0000","last_peered":"2023-09-23T15:23:41.505025+0000","last_clean":"2023-09-23T15:23:41.505025+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:23:41.505025+0000","last_undegraded":"2023-09-23T15:23:41.505025+0000","last_fullsized":"2023-09-23T15:23:41.505025+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":680,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504140+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:23:41.504140+0000","last_peered":"2023-09-23T15:23:41.504140+0000","last_clean":"2023-09-23T15:23:41.504140+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:23:41.504140+0000","last_undegraded":"2023-09-23T15:23:41.504140+0000","last_fullsized":"2023-09-23T15:23:41.504140+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":300,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.183110+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:23:43.183110+0000","last_peered":"2023-09-23T15:23:43.183110+0000","last_clean":"2023-09-23T15:23:43.183110+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:23:43.183110+0000","last_undegraded":"2023-09-23T15:23:43.183110+0000","last_fullsized":"2023-09-23T15:23:43.183110+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":307,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504877+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:23:41.504877+0000","last_peered":"2023-09-23T15:23:41.504877+0000","last_clean":"2023-09-23T15:23:41.504877+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:23:41.504877+0000","last_undegraded":"2023-09-23T15:23:41.504877+0000","last_fullsized":"2023-09-23T15:23:41.504877+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":613,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:46.926031+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:23:46.926031+0000","last_peered":"2023-09-23T15:23:46.926031+0000","last_clean":"2023-09-23T15:23:46.926031+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:23:46.926031+0000","last_undegraded":"2023-09-23T15:23:46.926031+0000","last_fullsized":"2023-09-23T15:23:46.926031+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":71,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.164321+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:23:43.164321+0000","last_peered":"2023-09-23T15:23:43.164321+0000","last_clean":"2023-09-23T15:23:43.164321+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:23:43.164321+0000","last_undegraded":"2023-09-23T15:23:43.164321+0000","last_fullsized":"2023-09-23T15:23:43.164321+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":288,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.192072+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:23:43.192072+0000","last_peered":"2023-09-23T15:23:43.192072+0000","last_clean":"2023-09-23T15:23:43.192072+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:23:43.192072+0000","last_undegraded":"2023-09-23T15:23:43.192072+0000","last_fullsized":"2023-09-23T15:23:43.192072+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":638,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504908+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:23:41.504908+0000","last_peered":"2023-09-23T15:23:41.504908+0000","last_clean":"2023-09-23T15:23:41.504908+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:23:41.504908+0000","last_undegraded":"2023-09-23T15:23:41.504908+0000","last_fullsized":"2023-09-23T15:23:41.504908+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":259,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.166371+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:23:43.166371+0000","last_peered":"2023-09-23T15:23:43.166371+0000","last_clean":"2023-09-23T15:23:43.166371+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:23:43.166371+0000","last_undegraded":"2023-09-23T15:23:43.166371+0000","last_fullsized":"2023-09-23T15:23:43.166371+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":325,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505338+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:23:41.505338+0000","last_peered":"2023-09-23T15:23:41.505338+0000","last_clean":"2023-09-23T15:23:41.505338+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:23:41.505338+0000","last_undegraded":"2023-09-23T15:23:41.505338+0000","last_fullsized":"2023-09-23T15:23:41.505338+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"322'7984","reported_seq":9034,"reported_epoch":322,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:49.866302+0000","last_change":"2023-09-23T15:23:39.629532+0000","last_active":"2023-09-23T15:23:49.866302+0000","last_peered":"2023-09-23T15:23:49.866302+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:58.288439+0000","last_became_peered":"2023-09-23T15:21:58.288439+0000","last_unstale":"2023-09-23T15:23:49.866302+0000","last_undegraded":"2023-09-23T15:21:58.149099+0000","last_fullsized":"2023-09-23T15:21:58.128266+0000","mapping_epoch":297,"log_start":"292'1100","ondisk_log_start":"292'1100","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:22:01.838822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224788480,"num_objects":3430,"num_object_clones":0,"num_object_copies":10290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2744,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7984,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":498,"num_bytes_recovered":32636928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":686}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":585,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.175732+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:23:43.175732+0000","last_peered":"2023-09-23T15:23:43.175732+0000","last_clean":"2023-09-23T15:23:43.175732+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:23:43.175732+0000","last_undegraded":"2023-09-23T15:23:43.175732+0000","last_fullsized":"2023-09-23T15:23:43.175732+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":63,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.166754+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:23:43.166754+0000","last_peered":"2023-09-23T15:23:43.166754+0000","last_clean":"2023-09-23T15:23:43.166754+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:23:43.166754+0000","last_undegraded":"2023-09-23T15:23:43.166754+0000","last_fullsized":"2023-09-23T15:23:43.166754+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":314,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.503977+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:23:41.503977+0000","last_peered":"2023-09-23T15:23:41.503977+0000","last_clean":"2023-09-23T15:23:41.503977+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:23:41.503977+0000","last_undegraded":"2023-09-23T15:23:41.503977+0000","last_fullsized":"2023-09-23T15:23:41.503977+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"323'7804","reported_seq":14674,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:54.433318+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:23:54.433318+0000","last_peered":"2023-09-23T15:23:54.433318+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:23:54.433318+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6904,"log_dups_size":0,"ondisk_log_size":6904,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214302720,"num_objects":3270,"num_object_clones":0,"num_object_copies":9810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3074,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7804,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2268,"num_bytes_recovered":148635648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":795},{"shards":"0,4,7","objects":1484},{"shards":"4,7","objects":991}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":642,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504941+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:23:41.504941+0000","last_peered":"2023-09-23T15:23:41.504941+0000","last_clean":"2023-09-23T15:23:41.504941+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:23:41.504941+0000","last_undegraded":"2023-09-23T15:23:41.504941+0000","last_fullsized":"2023-09-23T15:23:41.504941+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":63,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.165092+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:23:43.165092+0000","last_peered":"2023-09-23T15:23:43.165092+0000","last_clean":"2023-09-23T15:23:43.165092+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:23:43.165092+0000","last_undegraded":"2023-09-23T15:23:43.165092+0000","last_fullsized":"2023-09-23T15:23:43.165092+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505719+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:23:41.505719+0000","last_peered":"2023-09-23T15:23:41.505719+0000","last_clean":"2023-09-23T15:23:41.505719+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:23:41.505719+0000","last_undegraded":"2023-09-23T15:23:41.505719+0000","last_fullsized":"2023-09-23T15:23:41.505719+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"323'7994","reported_seq":8062,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:54.037672+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:23:54.037672+0000","last_peered":"2023-09-23T15:23:54.037672+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:23:54.037672+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"292'1400","ondisk_log_start":"292'1400","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6594,"log_dups_size":0,"ondisk_log_size":6594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211025920,"num_objects":3220,"num_object_clones":0,"num_object_copies":9660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7994,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":2242},{"shards":"0,7","objects":-1410}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":530,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167356+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:23:43.167356+0000","last_peered":"2023-09-23T15:23:43.167356+0000","last_clean":"2023-09-23T15:23:43.167356+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:23:43.167356+0000","last_undegraded":"2023-09-23T15:23:43.167356+0000","last_fullsized":"2023-09-23T15:23:43.167356+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":63,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.175312+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:23:43.175312+0000","last_peered":"2023-09-23T15:23:43.175312+0000","last_clean":"2023-09-23T15:23:43.175312+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:23:43.175312+0000","last_undegraded":"2023-09-23T15:23:43.175312+0000","last_fullsized":"2023-09-23T15:23:43.175312+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":286,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.166493+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:23:43.166493+0000","last_peered":"2023-09-23T15:23:43.166493+0000","last_clean":"2023-09-23T15:23:43.166493+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:23:43.166493+0000","last_undegraded":"2023-09-23T15:23:43.166493+0000","last_fullsized":"2023-09-23T15:23:43.166493+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"323'8006","reported_seq":8059,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:51.913287+0000","last_change":"2023-09-23T15:22:27.588207+0000","last_active":"2023-09-23T15:23:51.913287+0000","last_peered":"2023-09-23T15:23:51.913287+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:23:51.913287+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"292'1400","ondisk_log_start":"292'1400","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6606,"log_dups_size":0,"ondisk_log_size":6606,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8006,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-1350}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":537,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507971+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:23:41.507971+0000","last_peered":"2023-09-23T15:23:41.507971+0000","last_clean":"2023-09-23T15:23:41.507971+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:23:41.507971+0000","last_undegraded":"2023-09-23T15:23:41.507971+0000","last_fullsized":"2023-09-23T15:23:41.507971+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506168+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:23:41.506168+0000","last_peered":"2023-09-23T15:23:41.506168+0000","last_clean":"2023-09-23T15:23:41.506168+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:23:41.506168+0000","last_undegraded":"2023-09-23T15:23:41.506168+0000","last_fullsized":"2023-09-23T15:23:41.506168+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":304,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.190417+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:23:43.190417+0000","last_peered":"2023-09-23T15:23:43.190417+0000","last_clean":"2023-09-23T15:23:43.190417+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:23:43.190417+0000","last_undegraded":"2023-09-23T15:23:43.190417+0000","last_fullsized":"2023-09-23T15:23:43.190417+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"322'7864","reported_seq":7912,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.608070+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:23:50.608070+0000","last_peered":"2023-09-23T15:23:50.608070+0000","last_clean":"2023-09-23T15:23:50.608070+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:23:50.608070+0000","last_undegraded":"2023-09-23T15:23:50.608070+0000","last_fullsized":"2023-09-23T15:23:50.608070+0000","mapping_epoch":284,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":5964,"log_dups_size":0,"ondisk_log_size":5964,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223477760,"num_objects":3410,"num_object_clones":0,"num_object_copies":10230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7864,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":559,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.164424+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:23:43.164424+0000","last_peered":"2023-09-23T15:23:43.164424+0000","last_clean":"2023-09-23T15:23:43.164424+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:23:43.164424+0000","last_undegraded":"2023-09-23T15:23:43.164424+0000","last_fullsized":"2023-09-23T15:23:43.164424+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":314,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504792+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:23:41.504792+0000","last_peered":"2023-09-23T15:23:41.504792+0000","last_clean":"2023-09-23T15:23:41.504792+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:23:41.504792+0000","last_undegraded":"2023-09-23T15:23:41.504792+0000","last_fullsized":"2023-09-23T15:23:41.504792+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506856+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:23:41.506856+0000","last_peered":"2023-09-23T15:23:41.506856+0000","last_clean":"2023-09-23T15:23:41.506856+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:23:41.506856+0000","last_undegraded":"2023-09-23T15:23:41.506856+0000","last_fullsized":"2023-09-23T15:23:41.506856+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"323'8117","reported_seq":8166,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:51.913311+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:23:51.913311+0000","last_peered":"2023-09-23T15:23:51.913311+0000","last_clean":"2023-09-23T15:23:51.913311+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:23:51.913311+0000","last_undegraded":"2023-09-23T15:23:51.913311+0000","last_fullsized":"2023-09-23T15:23:51.913311+0000","mapping_epoch":284,"log_start":"292'1500","ondisk_log_start":"292'1500","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224460800,"num_objects":3425,"num_object_clones":0,"num_object_copies":10275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8117,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":626,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504185+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:23:41.504185+0000","last_peered":"2023-09-23T15:23:41.504185+0000","last_clean":"2023-09-23T15:23:41.504185+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:23:41.504185+0000","last_undegraded":"2023-09-23T15:23:41.504185+0000","last_fullsized":"2023-09-23T15:23:41.504185+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":270,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.166576+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:23:43.166576+0000","last_peered":"2023-09-23T15:23:43.166576+0000","last_clean":"2023-09-23T15:23:43.166576+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:23:43.166576+0000","last_undegraded":"2023-09-23T15:23:43.166576+0000","last_fullsized":"2023-09-23T15:23:43.166576+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507506+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:23:41.507506+0000","last_peered":"2023-09-23T15:23:41.507506+0000","last_clean":"2023-09-23T15:23:41.507506+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:23:41.507506+0000","last_undegraded":"2023-09-23T15:23:41.507506+0000","last_fullsized":"2023-09-23T15:23:41.507506+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"322'8045","reported_seq":14590,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:49.630917+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:23:49.630917+0000","last_peered":"2023-09-23T15:23:49.630917+0000","last_clean":"2023-09-23T15:23:49.630917+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:23:49.630917+0000","last_undegraded":"2023-09-23T15:23:49.630917+0000","last_fullsized":"2023-09-23T15:23:49.630917+0000","mapping_epoch":310,"log_start":"292'1500","ondisk_log_start":"292'1500","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222625792,"num_objects":3397,"num_object_clones":0,"num_object_copies":10191,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3397,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8045,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":579,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.169806+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:23:43.169806+0000","last_peered":"2023-09-23T15:23:43.169806+0000","last_clean":"2023-09-23T15:23:43.169806+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:23:43.169806+0000","last_undegraded":"2023-09-23T15:23:43.169806+0000","last_fullsized":"2023-09-23T15:23:43.169806+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505809+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:23:41.505809+0000","last_peered":"2023-09-23T15:23:41.505809+0000","last_clean":"2023-09-23T15:23:41.505809+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:23:41.505809+0000","last_undegraded":"2023-09-23T15:23:41.505809+0000","last_fullsized":"2023-09-23T15:23:41.505809+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507891+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:23:41.507891+0000","last_peered":"2023-09-23T15:23:41.507891+0000","last_clean":"2023-09-23T15:23:41.507891+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:23:41.507891+0000","last_undegraded":"2023-09-23T15:23:41.507891+0000","last_fullsized":"2023-09-23T15:23:41.507891+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"323'7943","reported_seq":7996,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:51.937407+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:23:51.937407+0000","last_peered":"2023-09-23T15:23:51.937407+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:23:51.937407+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"292'1400","ondisk_log_start":"292'1400","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6543,"log_dups_size":0,"ondisk_log_size":6543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219873280,"num_objects":3355,"num_object_clones":0,"num_object_copies":10065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7943,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-1263}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":638,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504778+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:23:41.504778+0000","last_peered":"2023-09-23T15:23:41.504778+0000","last_clean":"2023-09-23T15:23:41.504778+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:23:41.504778+0000","last_undegraded":"2023-09-23T15:23:41.504778+0000","last_fullsized":"2023-09-23T15:23:41.504778+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":336,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504492+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:23:41.504492+0000","last_peered":"2023-09-23T15:23:41.504492+0000","last_clean":"2023-09-23T15:23:41.504492+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:23:41.504492+0000","last_undegraded":"2023-09-23T15:23:41.504492+0000","last_fullsized":"2023-09-23T15:23:41.504492+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":230,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.176424+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:23:43.176424+0000","last_peered":"2023-09-23T15:23:43.176424+0000","last_clean":"2023-09-23T15:23:43.176424+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:23:43.176424+0000","last_undegraded":"2023-09-23T15:23:43.176424+0000","last_fullsized":"2023-09-23T15:23:43.176424+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"323'7833","reported_seq":7899,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:54.088543+0000","last_change":"2023-09-23T15:21:58.303371+0000","last_active":"2023-09-23T15:23:54.088543+0000","last_peered":"2023-09-23T15:23:54.088543+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:23:54.088543+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"292'1300","ondisk_log_start":"292'1300","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212402176,"num_objects":3241,"num_object_clones":0,"num_object_copies":9723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7833,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":148},{"shards":"0,2,7","objects":797}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":520,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167269+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:23:43.167269+0000","last_peered":"2023-09-23T15:23:43.167269+0000","last_clean":"2023-09-23T15:23:43.167269+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:23:43.167269+0000","last_undegraded":"2023-09-23T15:23:43.167269+0000","last_fullsized":"2023-09-23T15:23:43.167269+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":522,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.168627+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:23:43.168627+0000","last_peered":"2023-09-23T15:23:43.168627+0000","last_clean":"2023-09-23T15:23:43.168627+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:23:43.168627+0000","last_undegraded":"2023-09-23T15:23:43.168627+0000","last_fullsized":"2023-09-23T15:23:43.168627+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":298,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504799+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:23:41.504799+0000","last_peered":"2023-09-23T15:23:41.504799+0000","last_clean":"2023-09-23T15:23:41.504799+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:23:41.504799+0000","last_undegraded":"2023-09-23T15:23:41.504799+0000","last_fullsized":"2023-09-23T15:23:41.504799+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":266,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.173781+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:23:43.173781+0000","last_peered":"2023-09-23T15:23:43.173781+0000","last_clean":"2023-09-23T15:23:43.173781+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:23:43.173781+0000","last_undegraded":"2023-09-23T15:23:43.173781+0000","last_fullsized":"2023-09-23T15:23:43.173781+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"322'7811","reported_seq":7867,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:49.848125+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:23:49.848125+0000","last_peered":"2023-09-23T15:23:49.848125+0000","last_clean":"2023-09-23T15:23:49.848125+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:23:49.848125+0000","last_undegraded":"2023-09-23T15:23:49.848125+0000","last_fullsized":"2023-09-23T15:23:49.848125+0000","mapping_epoch":287,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6911,"log_dups_size":0,"ondisk_log_size":6911,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223674368,"num_objects":3413,"num_object_clones":0,"num_object_copies":10239,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3413,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7811,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":619,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504490+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:23:41.504490+0000","last_peered":"2023-09-23T15:23:41.504490+0000","last_clean":"2023-09-23T15:23:41.504490+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:23:41.504490+0000","last_undegraded":"2023-09-23T15:23:41.504490+0000","last_fullsized":"2023-09-23T15:23:41.504490+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":857,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505033+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:23:41.505033+0000","last_peered":"2023-09-23T15:23:41.505033+0000","last_clean":"2023-09-23T15:23:41.505033+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:23:41.505033+0000","last_undegraded":"2023-09-23T15:23:41.505033+0000","last_fullsized":"2023-09-23T15:23:41.505033+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506381+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:23:41.506381+0000","last_peered":"2023-09-23T15:23:41.506381+0000","last_clean":"2023-09-23T15:23:41.506381+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:23:41.506381+0000","last_undegraded":"2023-09-23T15:23:41.506381+0000","last_fullsized":"2023-09-23T15:23:41.506381+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":219,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167880+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:23:43.167880+0000","last_peered":"2023-09-23T15:23:43.167880+0000","last_clean":"2023-09-23T15:23:43.167880+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:23:43.167880+0000","last_undegraded":"2023-09-23T15:23:43.167880+0000","last_fullsized":"2023-09-23T15:23:43.167880+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"322'7780","reported_seq":14064,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.595436+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:23:50.595436+0000","last_peered":"2023-09-23T15:23:50.595436+0000","last_clean":"2023-09-23T15:23:50.595436+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:23:50.595436+0000","last_undegraded":"2023-09-23T15:23:50.595436+0000","last_fullsized":"2023-09-23T15:23:50.595436+0000","mapping_epoch":320,"log_start":"293'1800","ondisk_log_start":"293'1800","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":5980,"log_dups_size":0,"ondisk_log_size":5980,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218234880,"num_objects":3330,"num_object_clones":0,"num_object_copies":9990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7780,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":678,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.165433+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:23:43.165433+0000","last_peered":"2023-09-23T15:23:43.165433+0000","last_clean":"2023-09-23T15:23:43.165433+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:23:43.165433+0000","last_undegraded":"2023-09-23T15:23:43.165433+0000","last_fullsized":"2023-09-23T15:23:43.165433+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":939,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.172980+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:23:43.172980+0000","last_peered":"2023-09-23T15:23:43.172980+0000","last_clean":"2023-09-23T15:23:43.172980+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:23:43.172980+0000","last_undegraded":"2023-09-23T15:23:43.172980+0000","last_fullsized":"2023-09-23T15:23:43.172980+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":214,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.169413+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:23:43.169413+0000","last_peered":"2023-09-23T15:23:43.169413+0000","last_clean":"2023-09-23T15:23:43.169413+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:23:43.169413+0000","last_undegraded":"2023-09-23T15:23:43.169413+0000","last_fullsized":"2023-09-23T15:23:43.169413+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":223,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.176338+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:23:43.176338+0000","last_peered":"2023-09-23T15:23:43.176338+0000","last_clean":"2023-09-23T15:23:43.176338+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:23:43.176338+0000","last_undegraded":"2023-09-23T15:23:43.176338+0000","last_fullsized":"2023-09-23T15:23:43.176338+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"323'8045","reported_seq":8094,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:23:54.004139+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:23:54.004139+0000","last_peered":"2023-09-23T15:23:54.004139+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:23:54.004139+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"292'1500","ondisk_log_start":"292'1500","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6545,"log_dups_size":0,"ondisk_log_size":6545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217776128,"num_objects":3323,"num_object_clones":0,"num_object_copies":9969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8045,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1383}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":651,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505357+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:23:41.505357+0000","last_peered":"2023-09-23T15:23:41.505357+0000","last_clean":"2023-09-23T15:23:41.505357+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:23:41.505357+0000","last_undegraded":"2023-09-23T15:23:41.505357+0000","last_fullsized":"2023-09-23T15:23:41.505357+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":642,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.164639+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:23:43.164639+0000","last_peered":"2023-09-23T15:23:43.164639+0000","last_clean":"2023-09-23T15:23:43.164639+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:23:43.164639+0000","last_undegraded":"2023-09-23T15:23:43.164639+0000","last_fullsized":"2023-09-23T15:23:43.164639+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":255,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504547+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:23:41.504547+0000","last_peered":"2023-09-23T15:23:41.504547+0000","last_clean":"2023-09-23T15:23:41.504547+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:23:41.504547+0000","last_undegraded":"2023-09-23T15:23:41.504547+0000","last_fullsized":"2023-09-23T15:23:41.504547+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":274,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167222+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:23:43.167222+0000","last_peered":"2023-09-23T15:23:43.167222+0000","last_clean":"2023-09-23T15:23:43.167222+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:23:43.167222+0000","last_undegraded":"2023-09-23T15:23:43.167222+0000","last_fullsized":"2023-09-23T15:23:43.167222+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"323'7835","reported_seq":7884,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:51.924458+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:23:51.924458+0000","last_peered":"2023-09-23T15:23:51.924458+0000","last_clean":"2023-09-23T15:23:51.924458+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:23:51.924458+0000","last_undegraded":"2023-09-23T15:23:51.924458+0000","last_fullsized":"2023-09-23T15:23:51.924458+0000","mapping_epoch":284,"log_start":"292'1300","ondisk_log_start":"292'1300","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6535,"log_dups_size":0,"ondisk_log_size":6535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217251840,"num_objects":3315,"num_object_clones":0,"num_object_copies":9945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7835,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":629,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.169730+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:23:43.169730+0000","last_peered":"2023-09-23T15:23:43.169730+0000","last_clean":"2023-09-23T15:23:43.169730+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:23:43.169730+0000","last_undegraded":"2023-09-23T15:23:43.169730+0000","last_fullsized":"2023-09-23T15:23:43.169730+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":67,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.164591+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:23:43.164591+0000","last_peered":"2023-09-23T15:23:43.164591+0000","last_clean":"2023-09-23T15:23:43.164591+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:23:43.164591+0000","last_undegraded":"2023-09-23T15:23:43.164591+0000","last_fullsized":"2023-09-23T15:23:43.164591+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":290,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.183684+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:23:43.183684+0000","last_peered":"2023-09-23T15:23:43.183684+0000","last_clean":"2023-09-23T15:23:43.183684+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:23:43.183684+0000","last_undegraded":"2023-09-23T15:23:43.183684+0000","last_fullsized":"2023-09-23T15:23:43.183684+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":301,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.191249+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:23:43.191249+0000","last_peered":"2023-09-23T15:23:43.191249+0000","last_clean":"2023-09-23T15:23:43.191249+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:23:43.191249+0000","last_undegraded":"2023-09-23T15:23:43.191249+0000","last_fullsized":"2023-09-23T15:23:43.191249+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"322'8027","reported_seq":8075,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.589496+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:23:50.589496+0000","last_peered":"2023-09-23T15:23:50.589496+0000","last_clean":"2023-09-23T15:23:50.589496+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:23:50.589496+0000","last_undegraded":"2023-09-23T15:23:50.589496+0000","last_fullsized":"2023-09-23T15:23:50.589496+0000","mapping_epoch":284,"log_start":"293'2000","ondisk_log_start":"293'2000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6027,"log_dups_size":0,"ondisk_log_size":6027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230883328,"num_objects":3523,"num_object_clones":0,"num_object_copies":10569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8027,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":577,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.575768+0000","last_change":"2023-09-23T15:23:50.575734+0000","last_active":"2023-09-23T15:23:50.575768+0000","last_peered":"2023-09-23T15:23:50.575768+0000","last_clean":"2023-09-23T15:23:50.575768+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:23:50.575768+0000","last_undegraded":"2023-09-23T15:23:50.575768+0000","last_fullsized":"2023-09-23T15:23:50.575768+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1016,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:43.175584+0000","last_change":"2023-09-23T15:22:27.435515+0000","last_active":"2023-09-23T15:23:43.175584+0000","last_peered":"2023-09-23T15:23:43.175584+0000","last_clean":"2023-09-23T15:21:46.756689+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:23:43.175584+0000","last_undegraded":"2023-09-23T15:22:27.290223+0000","last_fullsized":"2023-09-23T15:23:43.175584+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":199,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.168500+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:23:43.168500+0000","last_peered":"2023-09-23T15:23:43.168500+0000","last_clean":"2023-09-23T15:23:43.168500+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:23:43.168500+0000","last_undegraded":"2023-09-23T15:23:43.168500+0000","last_fullsized":"2023-09-23T15:23:43.168500+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":286,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.190657+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:23:43.190657+0000","last_peered":"2023-09-23T15:23:43.190657+0000","last_clean":"2023-09-23T15:23:43.190657+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:23:43.190657+0000","last_undegraded":"2023-09-23T15:23:43.190657+0000","last_fullsized":"2023-09-23T15:23:43.190657+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"323'7839","reported_seq":7897,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:54.011742+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:23:54.011742+0000","last_peered":"2023-09-23T15:23:54.011742+0000","last_clean":"2023-09-23T15:23:54.011742+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:23:54.011742+0000","last_undegraded":"2023-09-23T15:23:54.011742+0000","last_fullsized":"2023-09-23T15:23:54.011742+0000","mapping_epoch":287,"log_start":"292'1300","ondisk_log_start":"292'1300","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6539,"log_dups_size":0,"ondisk_log_size":6539,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023616,"num_objects":3281,"num_object_clones":0,"num_object_copies":9843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7839,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":630,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.507428+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:23:41.507428+0000","last_peered":"2023-09-23T15:23:41.507428+0000","last_clean":"2023-09-23T15:23:41.507428+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:23:41.507428+0000","last_undegraded":"2023-09-23T15:23:41.507428+0000","last_fullsized":"2023-09-23T15:23:41.507428+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":78,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.173122+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:23:43.173122+0000","last_peered":"2023-09-23T15:23:43.173122+0000","last_clean":"2023-09-23T15:23:43.173122+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:23:43.173122+0000","last_undegraded":"2023-09-23T15:23:43.173122+0000","last_fullsized":"2023-09-23T15:23:43.173122+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":854,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:43.168732+0000","last_change":"2023-09-23T15:22:27.316872+0000","last_active":"2023-09-23T15:23:43.168732+0000","last_peered":"2023-09-23T15:23:43.168732+0000","last_clean":"2023-09-23T15:21:04.172426+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:23:43.168732+0000","last_undegraded":"2023-09-23T15:22:27.289016+0000","last_fullsized":"2023-09-23T15:23:43.168732+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504612+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:23:41.504612+0000","last_peered":"2023-09-23T15:23:41.504612+0000","last_clean":"2023-09-23T15:23:41.504612+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:23:41.504612+0000","last_undegraded":"2023-09-23T15:23:41.504612+0000","last_fullsized":"2023-09-23T15:23:41.504612+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"322'7778","reported_seq":7832,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:49.832634+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:23:49.832634+0000","last_peered":"2023-09-23T15:23:49.832634+0000","last_clean":"2023-09-23T15:23:49.832634+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:23:49.832634+0000","last_undegraded":"2023-09-23T15:23:49.832634+0000","last_fullsized":"2023-09-23T15:23:49.832634+0000","mapping_epoch":287,"log_start":"291'900","ondisk_log_start":"291'900","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6878,"log_dups_size":0,"ondisk_log_size":6878,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217907224,"num_objects":3326,"num_object_clones":0,"num_object_copies":9978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3326,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7778,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":646,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504802+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:23:41.504802+0000","last_peered":"2023-09-23T15:23:41.504802+0000","last_clean":"2023-09-23T15:23:41.504802+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:23:41.504802+0000","last_undegraded":"2023-09-23T15:23:41.504802+0000","last_fullsized":"2023-09-23T15:23:41.504802+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":759,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.167088+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:23:43.167088+0000","last_peered":"2023-09-23T15:23:43.167088+0000","last_clean":"2023-09-23T15:23:43.167088+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:23:43.167088+0000","last_undegraded":"2023-09-23T15:23:43.167088+0000","last_fullsized":"2023-09-23T15:23:43.167088+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":328,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.191324+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:23:43.191324+0000","last_peered":"2023-09-23T15:23:43.191324+0000","last_clean":"2023-09-23T15:23:43.191324+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:23:43.191324+0000","last_undegraded":"2023-09-23T15:23:43.191324+0000","last_fullsized":"2023-09-23T15:23:43.191324+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505631+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:23:41.505631+0000","last_peered":"2023-09-23T15:23:41.505631+0000","last_clean":"2023-09-23T15:23:41.505631+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:23:41.505631+0000","last_undegraded":"2023-09-23T15:23:41.505631+0000","last_fullsized":"2023-09-23T15:23:41.505631+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":305,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.190580+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:23:43.190580+0000","last_peered":"2023-09-23T15:23:43.190580+0000","last_clean":"2023-09-23T15:23:43.190580+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:23:43.190580+0000","last_undegraded":"2023-09-23T15:23:43.190580+0000","last_fullsized":"2023-09-23T15:23:43.190580+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":326,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.173589+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:23:43.173589+0000","last_peered":"2023-09-23T15:23:43.173589+0000","last_clean":"2023-09-23T15:23:43.173589+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:23:43.173589+0000","last_undegraded":"2023-09-23T15:23:43.173589+0000","last_fullsized":"2023-09-23T15:23:43.173589+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":335,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506252+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:23:41.506252+0000","last_peered":"2023-09-23T15:23:41.506252+0000","last_clean":"2023-09-23T15:23:41.506252+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:23:41.506252+0000","last_undegraded":"2023-09-23T15:23:41.506252+0000","last_fullsized":"2023-09-23T15:23:41.506252+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.506543+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:23:41.506543+0000","last_peered":"2023-09-23T15:23:41.506543+0000","last_clean":"2023-09-23T15:23:41.506543+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:23:41.506543+0000","last_undegraded":"2023-09-23T15:23:41.506543+0000","last_fullsized":"2023-09-23T15:23:41.506543+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":315,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504711+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:23:41.504711+0000","last_peered":"2023-09-23T15:23:41.504711+0000","last_clean":"2023-09-23T15:23:41.504711+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:23:41.504711+0000","last_undegraded":"2023-09-23T15:23:41.504711+0000","last_fullsized":"2023-09-23T15:23:41.504711+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505852+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:23:41.505852+0000","last_peered":"2023-09-23T15:23:41.505852+0000","last_clean":"2023-09-23T15:23:41.505852+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:23:41.505852+0000","last_undegraded":"2023-09-23T15:23:41.505852+0000","last_fullsized":"2023-09-23T15:23:41.505852+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":312,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504954+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:23:41.504954+0000","last_peered":"2023-09-23T15:23:41.504954+0000","last_clean":"2023-09-23T15:23:41.504954+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:23:41.504954+0000","last_undegraded":"2023-09-23T15:23:41.504954+0000","last_fullsized":"2023-09-23T15:23:41.504954+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":311,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.504544+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:23:41.504544+0000","last_peered":"2023-09-23T15:23:41.504544+0000","last_clean":"2023-09-23T15:23:41.504544+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:23:41.504544+0000","last_undegraded":"2023-09-23T15:23:41.504544+0000","last_fullsized":"2023-09-23T15:23:41.504544+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":275,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T15:23:43.188252+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:23:43.188252+0000","last_peered":"2023-09-23T15:23:43.188252+0000","last_clean":"2023-09-23T15:23:43.188252+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:23:43.188252+0000","last_undegraded":"2023-09-23T15:23:43.188252+0000","last_fullsized":"2023-09-23T15:23:43.188252+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":49,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T15:23:41.505325+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:23:41.505325+0000","last_peered":"2023-09-23T15:23:41.505325+0000","last_clean":"2023-09-23T15:23:41.505325+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:23:41.505325+0000","last_undegraded":"2023-09-23T15:23:41.505325+0000","last_fullsized":"2023-09-23T15:23:41.505325+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":3510370328,"num_objects":53565,"num_object_clones":0,"num_object_copies":160695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":27521,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":53565,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126705,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9119,"num_bytes_recovered":597622784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10934693888,"data_stored":10934681672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":104605,"ondisk_log_size":104605,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201630,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1383020,"kb_used_data":1241828,"kb_used_omap":0,"kb_used_meta":140992,"kb_avail":92362132,"statfs":{"total":95995035648,"available":94578823168,"internally_reserved":0,"allocated":1271631872,"data_stored":1270645744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":144375808},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":7.2160000000000002,"15min":3.8849999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":7.7629999999999999,"5min":226.226,"15min":226.226},"last":2.8889999999999998},{"interface":"front","average":{"1min":2.9399999999999999,"5min":8.5329999999999995,"15min":4.5190000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":9.4749999999999996,"5min":226.619,"15min":226.619},"last":8.9359999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.3680000000000001,"15min":2.1480000000000001},"min":{"1min":0.32600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.768,"5min":1.585,"15min":2.569},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1840000000000002,"5min":7.2510000000000003,"15min":16.065999999999999},"last":1.143}]},{"osd":2,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.4630000000000001,"15min":3.9340000000000002},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":6.6799999999999997,"5min":90.227999999999994,"15min":170.63800000000001},"last":2.9260000000000002},{"interface":"front","average":{"1min":1.889,"5min":2.778,"15min":4.5090000000000003},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2059999999999995,"5min":90.197999999999993,"15min":170.47999999999999},"last":8.8859999999999992}]},{"osd":3,"last update":"Sat Sep 23 15:22:58 2023","interfaces":[{"interface":"back","average":{"1min":2.1019999999999999,"5min":4.8659999999999997,"15min":3.7090000000000001},"min":{"1min":0.36399999999999999,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":12.379,"5min":146.166,"15min":146.166},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.677,"5min":5.0460000000000003,"15min":3.7250000000000001},"min":{"1min":0.39200000000000002,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":12.362,"5min":146.18299999999999,"15min":146.18299999999999},"last":5.1909999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":1.181,"15min":2.5990000000000002},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6739999999999995,"5min":16.771999999999998,"15min":150.119},"last":2.9569999999999999},{"interface":"front","average":{"1min":2.302,"5min":0.92300000000000004,"15min":2.7919999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.298,"5min":9.298,"15min":150.596},"last":9}]},{"osd":5,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":2.8079999999999998,"15min":2.8479999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":10.153,"5min":224.441,"15min":224.441},"last":0.38300000000000001},{"interface":"front","average":{"1min":2.7400000000000002,"5min":3.056,"15min":2.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.6600000000000001,"5min":224.28700000000001,"15min":224.28700000000001},"last":4.1669999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.2320000000000002,"15min":2.2320000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.2629999999999999,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":8.9659999999999993},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.336,"5min":10.336,"15min":10.336},"last":10.135}]}]},{"osd":6,"up_from":304,"seq":1305670058003,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377824,"kb_used_data":322072,"kb_used_omap":13,"kb_used_meta":55730,"kb_avail":93367328,"statfs":{"total":95995035648,"available":95608143872,"internally_reserved":0,"allocated":329801728,"data_stored":328827737,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":57067875},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":5.7649999999999997},"last":0.44600000000000001},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.8780000000000001,"15min":2.8780000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.044,"5min":10.044,"15min":10.044},"last":0.67200000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.742,"15min":2.742},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.064,"5min":11.064,"15min":11.064},"last":0.89400000000000002},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":17.001999999999999,"5min":17.001999999999999,"15min":17.001999999999999},"last":0.46899999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.526,"15min":1.526},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":0.96499999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":2.2799999999999998,"15min":2.2799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.0449999999999999,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5049999999999999,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":0.65500000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.1550000000000002,"5min":5.1550000000000002,"15min":5.1550000000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.821,"5min":1.821,"15min":1.821},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":9.1929999999999996},"last":0.69999999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":12.711,"5min":12.711,"15min":12.711},"last":6.7039999999999997},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.1950000000000003,"5min":5.1950000000000003,"15min":5.1950000000000003},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.0609999999999999,"5min":5.0609999999999999,"15min":5.0609999999999999},"last":0.71699999999999997},{"interface":"front","average":{"1min":1.823,"5min":1.823,"15min":1.823},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":0.93000000000000005}]}]},{"osd":1,"up_from":183,"seq":785979015303,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2163908,"kb_used_data":1717052,"kb_used_omap":6,"kb_used_meta":446649,"kb_avail":91581244,"statfs":{"total":95995035648,"available":93779193856,"internally_reserved":0,"allocated":1758261248,"data_stored":1756743745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":457368651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":3.9390000000000001,"15min":3.5049999999999999},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":9.5739999999999998,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.755},{"interface":"front","average":{"1min":4.0019999999999998,"5min":4.0300000000000002,"15min":4.2030000000000003},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.537000000000001,"5min":63.594999999999999,"15min":63.594999999999999},"last":3.5249999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":4.2759999999999998,"15min":4.1550000000000002},"min":{"1min":0.26900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.1600000000000001,"5min":109.375,"15min":109.375},"last":6.6980000000000004},{"interface":"front","average":{"1min":2.7269999999999999,"5min":3.7309999999999999,"15min":3.722},"min":{"1min":0.32600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":9.6259999999999994,"5min":105.16,"15min":105.16},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":6.399,"15min":8.2629999999999999},"min":{"1min":0.33100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":64.540999999999997,"5min":110.285,"15min":211.405},"last":0.72499999999999998},{"interface":"front","average":{"1min":6.4770000000000003,"5min":6.6900000000000004,"15min":9.0289999999999999},"min":{"1min":0.31900000000000001,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":67.465000000000003,"5min":110.095,"15min":210.988},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":3.3580000000000001,"15min":3.294},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3289999999999997,"5min":80.570999999999998,"15min":80.570999999999998},"last":6.6609999999999996},{"interface":"front","average":{"1min":2.7040000000000002,"5min":3.4550000000000001,"15min":3.5249999999999999},"min":{"1min":0.38100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":9.6600000000000001,"5min":80.75,"15min":80.75},"last":0.69799999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":2.5110000000000001,"15min":3.0649999999999999},"min":{"1min":0.39500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.3659999999999997,"5min":34.512999999999998,"15min":34.512999999999998},"last":3.4350000000000001},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.5870000000000002,"15min":2.8639999999999999},"min":{"1min":0.37,"5min":0.311,"15min":0.311},"max":{"1min":8.9559999999999995,"5min":37.911000000000001,"15min":37.911000000000001},"last":6.9500000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:23:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.0710000000000002,"15min":3.0710000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":12.26,"5min":12.26,"15min":12.26},"last":15.611000000000001},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.3230000000000004,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":3.4750000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":2.2549999999999999,"15min":2.8050000000000002},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":6.0339999999999998,"5min":12.861000000000001,"15min":12.861000000000001},"last":15.398999999999999},{"interface":"front","average":{"1min":2.6120000000000001,"5min":2.0510000000000002,"15min":2.6400000000000001},"min":{"1min":0.442,"5min":0.377,"15min":0.377},"max":{"1min":9.6929999999999996,"5min":13.282999999999999,"15min":16.841000000000001},"last":0.67000000000000004}]}]},{"osd":0,"up_from":95,"seq":408021893353,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2122812,"kb_used_data":1558324,"kb_used_omap":8,"kb_used_meta":564471,"kb_avail":91622340,"statfs":{"total":95995035648,"available":93821276160,"internally_reserved":0,"allocated":1595723776,"data_stored":1594332321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":578018900},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8069999999999999,"5min":3.5179999999999998,"15min":3.5710000000000002},"min":{"1min":0.23899999999999999,"5min":0.19500000000000001,"15min":0.18099999999999999},"max":{"1min":11.718,"5min":64.412999999999997,"15min":64.412999999999997},"last":4.6349999999999998},{"interface":"front","average":{"1min":2.2989999999999999,"5min":2.5419999999999998,"15min":3.4790000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":11.015000000000001,"5min":51.304000000000002,"15min":51.304000000000002},"last":12.058999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":2.7759999999999998,"15min":4.5229999999999997},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":10.949,"5min":30.510999999999999,"15min":122.20099999999999},"last":7.8920000000000003},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.3069999999999999,"15min":4.3010000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.18099999999999999},"max":{"1min":10.861000000000001,"5min":64.466999999999999,"15min":122.071},"last":4.7690000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:23:48 2023","interfaces":[{"interface":"back","average":{"1min":12.212999999999999,"5min":5.2119999999999997,"15min":5.7519999999999998},"min":{"1min":0.39100000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":197.78899999999999,"5min":197.78899999999999,"15min":197.78899999999999},"last":8.0700000000000003},{"interface":"front","average":{"1min":12.901999999999999,"5min":5.6210000000000004,"15min":6.226},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":200.58799999999999,"5min":200.58799999999999,"15min":200.58799999999999},"last":4.8460000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":1.8899999999999999,"15min":2.1419999999999999},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.358000000000001,"5min":15.358000000000001,"15min":31.341999999999999},"last":11.869},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.964,"15min":2.246},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":99.168999999999997},"last":8.2629999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.492,"15min":2.411},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.768000000000001,"5min":64.349999999999994,"15min":109.369},"last":7.4080000000000004},{"interface":"front","average":{"1min":3.6789999999999998,"5min":1.9990000000000001,"15min":2.355},"min":{"1min":0.497,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":15.436999999999999,"5min":15.436999999999999,"15min":109.387},"last":12.023}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":11.989000000000001},{"interface":"front","average":{"1min":3.3610000000000002,"5min":3.3610000000000002,"15min":3.3610000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":17.385000000000002,"5min":17.385000000000002,"15min":17.385000000000002},"last":11.952}]},{"osd":7,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.5550000000000002,"15min":2.488},"min":{"1min":0.42499999999999999,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":11.734,"5min":64.304000000000002,"15min":99.549999999999997},"last":7.3739999999999997},{"interface":"front","average":{"1min":4.9279999999999999,"5min":2.3039999999999998,"15min":2.1720000000000002},"min":{"1min":0.45900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":31.225999999999999},"last":13.706}]}]},{"osd":2,"up_from":105,"seq":450971566300,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2350828,"kb_used_data":1745764,"kb_used_omap":11,"kb_used_meta":604980,"kb_avail":91394324,"statfs":{"total":95995035648,"available":93587787776,"internally_reserved":0,"allocated":1787662336,"data_stored":1786280503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":619500304},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":6.9870000000000001,"5min":5.4580000000000002,"15min":4.1790000000000003},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":64.022000000000006,"5min":81.218999999999994,"15min":87.132999999999996},"last":5.5179999999999998},{"interface":"front","average":{"1min":7.8109999999999999,"5min":5.6849999999999996,"15min":3.863},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":64.091999999999999,"5min":81.352000000000004,"15min":84.820999999999998},"last":3.3010000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.8399999999999999,"15min":4.5869999999999997},"min":{"1min":0.22800000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":18.094000000000001,"5min":44.508000000000003,"15min":84.641000000000005},"last":0.56399999999999995},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4319999999999999,"15min":3.3090000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":18.628,"5min":45.765999999999998,"15min":84.751999999999995},"last":0.39500000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":8.5559999999999992,"5min":6.1820000000000004,"15min":5.298},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":77.031999999999996,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.53400000000000003},{"interface":"front","average":{"1min":8.2710000000000008,"5min":6.5289999999999999,"15min":5.3239999999999998},"min":{"1min":0.313,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":76.742000000000004,"5min":76.742000000000004,"15min":86.158000000000001},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.716,"15min":2.343},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":5.798,"5min":11.452999999999999,"15min":84.608000000000004},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.595,"5min":1.7410000000000001,"15min":2.968},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":5.8380000000000001,"5min":13.106,"15min":87.019999999999996},"last":5.5620000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.9019999999999999,"15min":2.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.2400000000000002,"5min":8.9909999999999997,"15min":84.498000000000005},"last":0.69499999999999995},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9810000000000001,"15min":2.5129999999999999},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":9.8780000000000001,"5min":15.891,"15min":84.664000000000001},"last":0.81100000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.2839999999999998,"15min":2.2839999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4329999999999998,"5min":7.4329999999999998,"15min":7.4329999999999998},"last":0.629},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.46000000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.0470000000000002,"15min":2.5649999999999999},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":10.227,"5min":10.227,"15min":84.596000000000004},"last":1.0649999999999999},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.0840000000000001,"15min":2.4249999999999998},"min":{"1min":0.504,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":7.0099999999999998,"5min":15.933999999999999,"15min":84.783000000000001},"last":0.45100000000000001}]}]},{"osd":3,"up_from":110,"seq":472446402773,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2089760,"kb_used_data":1503320,"kb_used_omap":3,"kb_used_meta":586428,"kb_avail":91655392,"statfs":{"total":95995035648,"available":93855121408,"internally_reserved":0,"allocated":1539399680,"data_stored":1538022075,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":600502441},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.302,"5min":3.113,"15min":3.1880000000000002},"min":{"1min":0.183,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":7.2939999999999996,"5min":33.220999999999997,"15min":79.611000000000004},"last":0.69399999999999995},{"interface":"front","average":{"1min":2.8300000000000001,"5min":3.512,"15min":3.4329999999999998},"min":{"1min":0.21099999999999999,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":36.465000000000003,"5min":36.465000000000003,"15min":79.656000000000006},"last":2.819}]},{"osd":1,"last update":"Sat Sep 23 15:23:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.0830000000000002,"15min":4.9660000000000002},"min":{"1min":0.23799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":126.875,"5min":126.875,"15min":126.875},"last":0.56100000000000005},{"interface":"front","average":{"1min":11.324999999999999,"5min":5.8259999999999996,"15min":5.2320000000000002},"min":{"1min":0.33600000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":128.249,"5min":128.249,"15min":128.249},"last":1.0469999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.55,"5min":2.3109999999999999,"15min":4.6440000000000001},"min":{"1min":0.312,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":7.3449999999999998,"5min":17.199999999999999,"15min":109.077},"last":0.316},{"interface":"front","average":{"1min":3.569,"5min":4.4249999999999998,"15min":5.6589999999999998},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":36.582000000000001,"5min":119.449,"15min":119.72799999999999},"last":5.827}]},{"osd":4,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7389999999999999,"5min":2.1339999999999999,"15min":2.1869999999999998},"min":{"1min":0.33400000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":36.551000000000002,"5min":36.551000000000002,"15min":54.051000000000002},"last":2.9119999999999999},{"interface":"front","average":{"1min":1.5609999999999999,"5min":1.589,"15min":2.0960000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":9.1839999999999993,"5min":12.500999999999999,"15min":52.228999999999999},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":3.1480000000000001,"5min":3.5390000000000001,"15min":2.7829999999999999},"min":{"1min":0.31,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":36.369,"5min":148.619,"15min":148.619},"last":6.8650000000000002},{"interface":"front","average":{"1min":1.8660000000000001,"5min":2.9740000000000002,"15min":2.7349999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.315},"max":{"1min":9.1430000000000007,"5min":148.56999999999999,"15min":148.56999999999999},"last":6.8520000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.2200000000000006,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":0.60099999999999998},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":36.517000000000003,"5min":36.517000000000003,"15min":36.517000000000003},"last":6.7229999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":3.0419999999999998,"15min":2.6880000000000002},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":9.2569999999999997,"5min":145.25,"15min":145.25},"last":0.63400000000000001},{"interface":"front","average":{"1min":1.7370000000000001,"5min":3.1080000000000001,"15min":2.4420000000000002},"min":{"1min":0.40400000000000003,"5min":0.36399999999999999,"15min":0.34799999999999998},"max":{"1min":7.4509999999999996,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.53800000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986538,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1224088,"kb_used_data":918480,"kb_used_omap":0,"kb_used_meta":305600,"kb_avail":92521064,"statfs":{"total":95995035648,"available":94741569536,"internally_reserved":0,"allocated":940523520,"data_stored":939549357,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":312934400},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":5.7130000000000001,"15min":4.0220000000000002},"min":{"1min":0.23599999999999999,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":5.3170000000000002,"5min":162.75999999999999,"15min":162.75999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.5699999999999998,"5min":5.6079999999999997,"15min":4.1219999999999999},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":8.7289999999999992,"5min":162.34800000000001,"15min":162.34800000000001},"last":0.93700000000000006}]},{"osd":1,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8580000000000001,"5min":3.3130000000000002,"15min":3.419},"min":{"1min":0.249,"5min":0.245,"15min":0.245},"max":{"1min":9.5350000000000001,"5min":113.11199999999999,"15min":113.11199999999999},"last":0.995},{"interface":"front","average":{"1min":2.746,"5min":3.4660000000000002,"15min":3.851},"min":{"1min":0.309,"5min":0.252,"15min":0.252},"max":{"1min":12.726000000000001,"5min":113.02,"15min":113.02},"last":0.54600000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":3.734,"15min":4.5199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":7.016,"5min":61.646000000000001,"15min":148.08000000000001},"last":1.048},{"interface":"front","average":{"1min":3.488,"5min":3.8290000000000002,"15min":4.758},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":6.9980000000000002,"5min":64.840000000000003,"15min":147.88900000000001},"last":1.0309999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:22:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8300000000000001,"5min":6.306,"15min":5.9610000000000003},"min":{"1min":0.39400000000000002,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":5.3860000000000001,"5min":156.46000000000001,"15min":163.97499999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.8600000000000001,"5min":6.641,"15min":5.8760000000000003},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.27800000000000002},"max":{"1min":6.2510000000000003,"5min":156.74299999999999,"15min":164.447},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:23:00 2023","interfaces":[{"interface":"back","average":{"1min":2.4620000000000002,"5min":0.91000000000000003,"15min":1.633},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.23499999999999999},"max":{"1min":6.4409999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":1.5289999999999999},{"interface":"front","average":{"1min":3.1680000000000001,"5min":1.131,"15min":1.573},"min":{"1min":0.30199999999999999,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":14.449999999999999,"5min":14.449999999999999,"15min":88.602999999999994},"last":5.7199999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:23:28 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.446,"15min":1.446},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6470000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.373},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.1369999999999996,"5min":6.1369999999999996,"15min":6.1369999999999996},"last":0.97199999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.92500000000000004,"15min":1.1890000000000001},"min":{"1min":0.42599999999999999,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.7599999999999998,"5min":6.0730000000000004,"15min":18.23},"last":1.012},{"interface":"front","average":{"1min":1.819,"5min":1.034,"15min":1.286},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":4.2229999999999999,"5min":5.3520000000000003,"15min":14.653},"last":0.42699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724901,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2136912,"kb_used_data":1766024,"kb_used_omap":0,"kb_used_meta":370816,"kb_avail":91608240,"statfs":{"total":95995035648,"available":93806837760,"internally_reserved":0,"allocated":1808408576,"data_stored":1807438866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":379715584},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.8639999999999999,"15min":3.2309999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":4.1059999999999999,"5min":124.31,"15min":124.31},"last":4.2629999999999999},{"interface":"front","average":{"1min":2.6349999999999998,"5min":4.8070000000000004,"15min":3.2799999999999998},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":9.3889999999999993,"5min":124.374,"15min":124.374},"last":4.2939999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":3.8199999999999998,"15min":2.5880000000000001},"min":{"1min":0.36199999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3149999999999995,"5min":124.38800000000001,"15min":124.38800000000001},"last":4.4249999999999998},{"interface":"front","average":{"1min":2.165,"5min":4.0679999999999996,"15min":2.8919999999999999},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":5.8769999999999998,"5min":124.29000000000001,"15min":124.29000000000001},"last":4.4139999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":3.77,"15min":3.7210000000000001},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":8.3030000000000008,"5min":193.00999999999999,"15min":193.00999999999999},"last":5.8310000000000004},{"interface":"front","average":{"1min":2.4550000000000001,"5min":4.0960000000000001,"15min":3.8340000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":9.3420000000000005,"5min":193.15700000000001,"15min":193.15700000000001},"last":4.4429999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":16.594999999999999,"5min":7.7089999999999996,"15min":6.0309999999999997},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":266.70299999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":6.3330000000000002},{"interface":"front","average":{"1min":20.937000000000001,"5min":8.5090000000000003,"15min":6.0449999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":266.80399999999997,"5min":266.80399999999997,"15min":266.80399999999997},"last":4.46}]},{"osd":4,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":0.73399999999999999,"15min":1.8380000000000001},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":7.3440000000000003,"5min":7.3440000000000003,"15min":138.68600000000001},"last":5.556},{"interface":"front","average":{"1min":1.641,"5min":0.754,"15min":1.9019999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":138.49600000000001},"last":4.7320000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.3969999999999998,"15min":2.3969999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.6050000000000004,"5min":9.6050000000000004,"15min":9.6050000000000004},"last":4.5860000000000003},{"interface":"front","average":{"1min":2.4100000000000001,"5min":2.4100000000000001,"15min":2.4100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":6.6059999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.131,"15min":1.407},"min":{"1min":0.40200000000000002,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":12.065,"5min":12.065,"15min":15.301},"last":6.4619999999999997},{"interface":"front","average":{"1min":2.9809999999999999,"5min":1.0920000000000001,"15min":1.351},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":15.762},"last":6.4359999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1586167808,"data_stored":1586167808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1740898304,"data_stored":1740898304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1769345024,"data_stored":1769340952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1527775232,"data_stored":1527775232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":935198720,"data_stored":935198720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1787891712,"data_stored":1787887640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326107136,"data_stored":326107136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1261309952,"data_stored":1261305880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:23:56.609 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:23:56.673 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:56.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:56.784 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:56.871 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:57.003 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:57.011 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:57.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:57.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:57.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:57.158 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:57.313 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:57.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:57.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:57.414 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 27538/160779 objects degraded (17.128%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:57.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 27538/160779 objects degraded (17.128%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:57.419 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:57.491 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:57.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 27538/160779 objects degraded (17.128%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:23:57.893 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:23:57.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:57.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:23:57.998 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:23:58.183 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:23:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:23:58] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T15:23:58.199 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:58.263 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:23:58.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[215646]: pgmap v939: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 27308/157005 objects degraded (17.393%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/61059070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[222878]: pgmap v939: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 27308/157005 objects degraded (17.393%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/61059070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:58.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:58.590 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:23:58.591 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:23:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:58 smithi027 ceph-mon[195233]: pgmap v939: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 27308/157005 objects degraded (17.393%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T15:23:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/61059070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:23:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:23:58.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:23:59.167 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:23:59.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:59.170 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:23:59.250 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:23:59.271 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:23:59.307 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:23:59 smithi027 ceph-mon[195233]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.624 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:23:59.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:23:59.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:23:59.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:59.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[215646]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:23:59 smithi017 ceph-mon[222878]: failed to encode map e324 with expected crc 2023-09-23T15:23:59.711 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:23:59.736 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:23:59.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:23:59.809 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:23:59.820 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:23:59.862 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:23:59.921 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:23:59.953 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:00.008 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:00.015 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:00.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:00.049 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:00.084 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.087 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:24:00.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:00.188 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:00.273 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:00 smithi017 ceph-mon[222878]: pgmap v941: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 27308/154785 objects degraded (17.643%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:24:00.274 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3037838049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:00.458 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:24:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:00 smithi027 ceph-mon[195233]: pgmap v941: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 27308/154785 objects degraded (17.643%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:24:00.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3037838049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:00.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:00 smithi017 ceph-mon[215646]: pgmap v941: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 462 op/s; 27308/154785 objects degraded (17.643%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T15:24:00.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3037838049' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:01.015 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:01.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:01.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:01.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:01.195 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:01.277 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:24:01.305 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:01.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:01.309 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:01.409 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:01.497 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:01.606 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:01.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:01.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:01.782 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:01.902 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:01.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:01.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:02.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:02.041 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:02.089 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:02.219 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:02.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:02.228 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:02.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:02.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:02 smithi017 ceph-mon[215646]: pgmap v942: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 26670/147891 objects degraded (18.034%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T15:24:02.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26670/147891 objects degraded (18.034%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:02.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:02 smithi017 ceph-mon[222878]: pgmap v942: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 26670/147891 objects degraded (18.034%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T15:24:02.392 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26670/147891 objects degraded (18.034%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:02.400 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:02.517 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:02.518 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 26670/147891 objects degraded (18.034%), 9 pgs degraded, 7 pgs undersized","count":16},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1250,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":324,"num_osds":8,"num_up_osds":8,"osd_up_since":1695482545,"num_in_osds":8,"osd_in_since":1695482516,"num_remapped_pgs":7},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":100},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":6},{"state_name":"active+recovery_wait+degraded","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":109,"num_pools":5,"num_objects":49297,"data_bytes":3233671537,"bytes_used":13469872128,"bytes_avail":754490413056,"bytes_total":767960285184,"degraded_objects":26670,"degraded_total":147891,"degraded_ratio":0.18033551737428241,"recovering_objects_per_sec":85,"recovering_bytes_per_sec":5611009,"recovering_keys_per_sec":0,"num_objects_recovered":855,"num_bytes_recovered":56033280,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":601},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T15:23:24.711679+0000","services":{}},"progress_events":{"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a90e8e5c-cd46-427f-990c-d4f2d8f69255":{"message":"Global Recovery Event (2m)\n [=========================...] (remaining: 15s)","progress":0.91743117570877075,"add_to_ceph_s":true}}} 2023-09-23T15:24:02.524 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:02.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:02.542 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:02.557 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:02 smithi027 ceph-mon[195233]: pgmap v942: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 601 op/s; 26670/147891 objects degraded (18.034%); 5.4 MiB/s, 85 objects/s recovering 2023-09-23T15:24:02.557 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26670/147891 objects degraded (18.034%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:02.643 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:02.699 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:02.885 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:24:02.885 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:24:02.885 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:24:02.885 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:24:02.885 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:24:03.325 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1432866779' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:03.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:24:03.600 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1432866779' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:03.601 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1432866779' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:04.096 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:04.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:04.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:24:04.192 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:24:04.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:24:04.206 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:04.283 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:04.333 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:04 smithi027 ceph-mon[195233]: pgmap v943: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26670/145599 objects degraded (18.317%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:24:04.449 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:04 smithi017 ceph-mon[215646]: pgmap v943: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26670/145599 objects degraded (18.317%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:24:04.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:04 smithi017 ceph-mon[222878]: pgmap v943: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 26670/145599 objects degraded (18.317%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T15:24:04.467 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:04.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:04.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:04.577 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:04.634 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:04.805 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:24:04.832 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:04.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:04.837 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:04.858 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:04.938 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:05.033 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:05.048 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:05.225 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:05.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:05.344 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:05.407 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:05.440 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:24:05.660 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:05.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3767079357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:05.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:05.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3767079357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:05.665 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:05.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:05.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:05.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:05.845 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:05.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3767079357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:05.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:06.403 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: pgmap v944: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26270/142635 objects degraded (18.418%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.404 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: pgmap v944: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26270/142635 objects degraded (18.418%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.405 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.406 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:06 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: pgmap v944: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 604 op/s; 26270/142635 objects degraded (18.418%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: failed to encode map e325 with expected crc 2023-09-23T15:24:06.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:06 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:24:06.914 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:06.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:07.019 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:07.114 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:07.252 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:07.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:45138 8.5 8:a64e88ac:::benchmark_data_smithi017_485823_object45137:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:51.038971+0000", 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 136.24660461600001, 2023-09-23T15:24:07.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29235043399999999, 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 45138 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.038971+0000", 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.038971+0000", 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.038971+0000", 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039057+0000", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6234000000000001e-05 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039068+0000", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1324e-05 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039073+0000", 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7809999999999996e-06 2023-09-23T15:24:07.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039123+0000", 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0306999999999998e-05 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039268+0000", 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014494700000000001 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:24:07.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.039432+0000", 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000164093 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.040767+0000", 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0013351039999999999 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.180999+0000", 2023-09-23T15:24:07.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.14023179999999999 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.331214+0000", 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.15021504099999999 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.331252+0000", 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7450999999999998e-05 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.295 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:51.331322+0000", 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0188e-05 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50571 8.5 8:a23a00b9:::benchmark_data_smithi017_485823_object50570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.161908+0000", 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.123668369, 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.159864754, 2023-09-23T15:24:07.296 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50571 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161908+0000", 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161908+0000", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161906+0000", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161953+0000", 2023-09-23T15:24:07.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7006999999999999e-05 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161970+0000", 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6801000000000001e-05 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.161978+0000", 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.259e-06 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.162074+0000", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5390000000000007e-05 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.162087+0000", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3478999999999999e-05 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128641+0000", 2023-09-23T15:24:07.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.96655348399999996 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128660+0000", 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9219999999999999e-05 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.306730+0000", 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1780699450000001 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.306854+0000", 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012407299999999999 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.307007+0000", 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015353 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.314800+0000", 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077922579999999998 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.317258+0000", 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024586809999999999 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319411+0000", 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021525329999999999 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.303 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319471+0000", 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0341999999999999e-05 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321772+0000", 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023012549999999999 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.304 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50600 8.5 8:a453a8bb:::benchmark_data_smithi017_485823_object50599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.210817+0000", 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.07475894000001, 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1131867729999998, 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50600 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.305 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210817+0000", 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210817+0000", 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.306 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210816+0000", 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210840+0000", 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4669e-05 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210875+0000", 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4459000000000003e-05 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.307 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210885+0000", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0581000000000001e-05 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210973+0000", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7768000000000004e-05 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210999+0000", 2023-09-23T15:24:07.308 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.586e-05 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128701+0000", 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91770172100000003 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128711+0000", 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0056e-05 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.309 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.308891+0000", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.180180292 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.309062+0000", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171311 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.309285+0000", 2023-09-23T15:24:07.310 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000222494 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.314845+0000", 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055603140000000002 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.317303+0000", 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024574200000000001 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.311 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321856+0000", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045531779999999997 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321916+0000", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0343000000000001e-05 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.324004+0000", 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020876720000000001 2023-09-23T15:24:07.312 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50607 8.5 8:abd65503:::benchmark_data_smithi017_485823_object50606:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.219177+0000", 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.06639919, 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.111768401, 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.313 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50607 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219177+0000", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219177+0000", 2023-09-23T15:24:07.314 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219176+0000", 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219203+0000", 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6480000000000001e-05 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.315 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219215+0000", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2386e-05 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219220+0000", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1669999999999998e-06 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219275+0000", 2023-09-23T15:24:07.316 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4651999999999998e-05 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.219290+0000", 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5260999999999998e-05 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.130502+0000", 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91121216900000002 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.317 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.130514+0000", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1962e-05 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310943+0000", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1804293219999999 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.311038+0000", 2023-09-23T15:24:07.318 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4466e-05 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.311209+0000", 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000170805 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.314851+0000", 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036421579999999999 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.319 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.317341+0000", 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024903130000000001 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.328591+0000", 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01124938 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.320 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.328647+0000", 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6650000000000001e-05 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.330945+0000", 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022980470000000001 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.321 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127169 8.5 8:a4bd91d3:::benchmark_data_smithi017_485823_object37032:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.435270+0000", 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.850305826, 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.394636031, 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127169 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435270+0000", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435270+0000", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435268+0000", 2023-09-23T15:24:07.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435277+0000", 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2339999999999994e-06 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435289+0000", 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2642e-05 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.435296+0000", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.674e-06 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.492703+0000", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.057407542999999998 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.493427+0000", 2023-09-23T15:24:07.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00072396500000000003 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.796687+0000", 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30325982899999998 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.796766+0000", 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8782000000000006e-05 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.796874+0000", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010835100000000001 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.815447+0000", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018572305000000001 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.828152+0000", 2023-09-23T15:24:07.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012705118 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.828194+0000", 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2157999999999999e-05 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.828237+0000", 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2540999999999998e-05 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.829906+0000", 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001669741 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127175 8.5 8:a5acb1df:::benchmark_data_smithi017_485823_object37038:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.441732+0000", 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.843844410999999, 2023-09-23T15:24:07.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38921504000000001, 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127175 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441732+0000", 2023-09-23T15:24:07.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441732+0000", 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441730+0000", 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441739+0000", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5099999999999998e-06 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441753+0000", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4141000000000001e-05 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.441760+0000", 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4379999999999998e-06 2023-09-23T15:24:07.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.493472+0000", 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.051711928999999997 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.494170+0000", 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00069842299999999999 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.804465+0000", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31029433899999997 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.804600+0000", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013518800000000001 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.804827+0000", 2023-09-23T15:24:07.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022702399999999999 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.818218+0000", 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013390694 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.829942+0000", 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011724002000000001 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.829984+0000", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2131e-05 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.830024+0000", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9811000000000002e-05 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.830947+0000", 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00092305399999999998 2023-09-23T15:24:07.336 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127177 8.5 8:afdbc28f:::benchmark_data_smithi017_485823_object37040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.444861+0000", 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.840714735000001, 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38701702199999999, 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.337 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127177 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444861+0000", 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444861+0000", 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.338 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444860+0000", 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444867+0000", 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0480000000000001e-06 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444883+0000", 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6263999999999999e-05 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.444891+0000", 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0360000000000006e-06 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.494220+0000", 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.049329389000000001 2023-09-23T15:24:07.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.494866+0000", 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064524099999999996 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.811361+0000", 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31649524400000001 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.811496+0000", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013544100000000001 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.811691+0000", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000195117 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.828167+0000", 2023-09-23T15:24:07.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016475383 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831004+0000", 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028373719999999999 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831028+0000", 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4192000000000001e-05 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831060+0000", 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1674999999999997e-05 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831878+0000", 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000818321 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127184 8.5 8:aa7279d4:::benchmark_data_smithi017_485823_object37047:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.461205+0000", 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.824371392, 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.371829354, 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127184 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.345 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461205+0000", 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461205+0000", 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.346 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461203+0000", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461214+0000", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1895999999999999e-05 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461227+0000", 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2964e-05 2023-09-23T15:24:07.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.461235+0000", 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9829999999999996e-06 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.494902+0000", 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.033666923000000001 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.495538+0000", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00063557700000000002 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.819830+0000", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32429257299999997 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.819912+0000", 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1464000000000005e-05 2023-09-23T15:24:07.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.820024+0000", 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111733 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.830957+0000", 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010933414000000001 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831923+0000", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000965893 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831989+0000", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5994999999999994e-05 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.832031+0000", 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2147999999999997e-05 2023-09-23T15:24:07.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833034+0000", 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001002974 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127190 8.5 8:a13fa353:::benchmark_data_smithi017_485823_object37053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.475611+0000", 2023-09-23T15:24:07.352 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.809965363, 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.358296371, 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127190 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475611+0000", 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475611+0000", 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475609+0000", 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.354 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475617+0000", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4319999999999999e-06 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475631+0000", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4533e-05 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.475639+0000", 2023-09-23T15:24:07.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0870000000000001e-06 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.495570+0000", 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.019931534000000001 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.496290+0000", 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00071965600000000005 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.825066+0000", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32877619800000002 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.825148+0000", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2485999999999996e-05 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.825248+0000", 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9372999999999999e-05 2023-09-23T15:24:07.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.831890+0000", 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066423929999999999 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833072+0000", 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011813800000000001 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833125+0000", 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3887000000000001e-05 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833168+0000", 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2138999999999998e-05 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833907+0000", 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000739491 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.359 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45809.0:127199 8.c e323/287 8:38f7a940:::benchmark_data_smithi017_485823_object37062:head v 323'7785, mlcod=323'7782)", 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.498239+0000", 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.787337388999999, 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.295992796, 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498239+0000", 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498239+0000", 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498238+0000", 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498247+0000", 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9639999999999992e-06 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498259+0000", 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.253e-05 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498266+0000", 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4339999999999997e-06 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498345+0000", 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9264999999999994e-05 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498364+0000", 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9032e-05 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.513163+0000", 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014798731000000001 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794231+0000", 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28106890400000001 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45809.0:127202 8.c e323/287 8:38d6842c:::benchmark_data_smithi017_485823_object37065:head v 323'7786, mlcod=323'7782)", 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.498284+0000", 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.787292186, 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.295983881, 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498284+0000", 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498284+0000", 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498284+0000", 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498288+0000", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9400000000000004e-06 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498293+0000", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1479999999999997e-06 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.498296+0000", 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4189999999999998e-06 2023-09-23T15:24:07.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501686+0000", 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003389389 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501714+0000", 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7977999999999999e-05 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794236+0000", 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29252260499999999 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794268+0000", 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.167e-05 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.369 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45809.0:127203 8.6 e323/305 8:6a63d0a4:::benchmark_data_smithi017_485823_object37066:head v 323'7936, mlcod=0'0)", 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.501688+0000", 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.783887987, 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29254144199999998, 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501688+0000", 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501688+0000", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501687+0000", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501698+0000", 2023-09-23T15:24:07.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1582e-05 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501712+0000", 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3740999999999999e-05 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501719+0000", 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4970000000000004e-06 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501783+0000", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3665999999999997e-05 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.501807+0000", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3983999999999999e-05 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512191+0000", 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010384262 2023-09-23T15:24:07.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794230+0000", 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28203821099999998 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45809.0:127209 8.c e323/287 8:386b74dc:::benchmark_data_smithi017_485823_object37072:head v 323'7787, mlcod=323'7783)", 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.505808+0000", 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.779767592000001, 2023-09-23T15:24:07.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.288482666, 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505808+0000", 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505808+0000", 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505808+0000", 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505815+0000", 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7789999999999998e-06 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505823+0000", 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505829+0000", 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1639999999999998e-06 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505866+0000", 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6956000000000003e-05 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.505885+0000", 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8544000000000001e-05 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794271+0000", 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28838581899999999 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794291+0000", 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0497999999999999e-05 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:127212 8.5 8:a9e7c80b:::benchmark_data_smithi017_485823_object37075:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e323)", 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.506477+0000", 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.779099312, 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.34777157199999997, 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:24:07.379 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 127212 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506477+0000", 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506477+0000", 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506475+0000", 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506483+0000", 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6920000000000002e-06 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506496+0000", 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3867e-05 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.506503+0000", 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.579e-06 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.833935+0000", 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32743144499999999 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.834864+0000", 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00092945899999999999 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.843906+0000", 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090420199999999996 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.844028+0000", 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012217700000000001 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 4,6", 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.844227+0000", 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000198889 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:24:07.384 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.853181+0000", 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089537350000000009 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.853247+0000", 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6217999999999995e-05 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.853280+0000", 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2984000000000001e-05 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.853323+0000", 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3161000000000002e-05 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.854248+0000", 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00092519300000000004 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:a5acb1df:::benchmark_data_smithi017_485823_object37038:head)])", 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.509376+0000", 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.776200462, 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29502916200000001, 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509376+0000", 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509376+0000", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509375+0000", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509384+0000", 2023-09-23T15:24:07.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6330000000000002e-06 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509391+0000", 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2980000000000003e-06 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509395+0000", 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9999999999999998e-06 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.800798+0000", 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29140276199999998 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.804405+0000", 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036067170000000002 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.390 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:afdbc28f:::benchmark_data_smithi017_485823_object37040:head)])", 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.509400+0000", 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.776176255999999, 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30190283400000001, 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509400+0000", 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509400+0000", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509400+0000", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509402+0000", 2023-09-23T15:24:07.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6740000000000001e-06 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509408+0000", 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6149999999999996e-06 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.509412+0000", 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9550000000000002e-06 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.808515+0000", 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.299102916 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.811303+0000", 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027878690000000001 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.394 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:aa7279d4:::benchmark_data_smithi017_485823_object37047:head)])", 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.511807+0000", 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.773769414, 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30540568800000001, 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511807+0000", 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511807+0000", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511806+0000", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511814+0000", 2023-09-23T15:24:07.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1040000000000007e-06 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511827+0000", 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3288000000000001e-05 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511833+0000", 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4169999999999997e-06 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.815505+0000", 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30367134600000001 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.817212+0000", 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017075269999999999 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.398 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:a13fa353:::benchmark_data_smithi017_485823_object37053:head)])", 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.511840+0000", 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.77373594, 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30638040999999999, 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511840+0000", 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511840+0000", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511840+0000", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511843+0000", 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6690000000000002e-06 2023-09-23T15:24:07.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511848+0000", 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5269999999999998e-06 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.511853+0000", 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5090000000000004e-06 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.817237+0000", 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.305384457 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.818221+0000", 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00098348800000000003 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.402 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:aa7279d4:::benchmark_data_smithi017_485823_object37047:head)])", 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.512070+0000", 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.773505596, 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30773837100000001, 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512070+0000", 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512070+0000", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512070+0000", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512077+0000", 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8279999999999997e-06 2023-09-23T15:24:07.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512087+0000", 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0665999999999999e-05 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512093+0000", 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1140000000000002e-06 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.818243+0000", 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30615009999999998 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.819809+0000", 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001565426 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.406 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(8.5 323/305 [PushReplyOp(8:a13fa353:::benchmark_data_smithi017_485823_object37053:head)])", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:23:52.512264+0000", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 14.773312137, 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.31278150700000001, 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512264+0000", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:24:07.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512264+0000", 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512263+0000", 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.512269+0000", 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9830000000000001e-06 2023-09-23T15:24:07.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794202+0000", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28193295400000001 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.794217+0000", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4619e-05 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.822852+0000", 2023-09-23T15:24:07.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.028634624000000001 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:23:52.825045+0000", 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021938880000000002 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:24:07.410 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:24:07.411 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:07.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:07 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:07.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 26270/142635 objects degraded (18.418%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:07.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:07 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:07.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 26270/142635 objects degraded (18.418%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:07.474 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:07.597 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:07 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:07.597 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 26270/142635 objects degraded (18.418%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:07.947 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:07.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:07.959 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:08.060 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:08.091 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:08.297 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:08.304 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:08] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.43.0" 2023-09-23T15:24:08.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:08.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:08.405 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:08.509 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:24:08.533 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:08.568 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:24:08.607 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:24:08.614 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:24:08.651 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:08.652 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:08 smithi017 ceph-mon[222878]: pgmap v946: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 25988/138726 objects degraded (18.733%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T15:24:08.652 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:08 smithi017 ceph-mon[215646]: pgmap v946: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 25988/138726 objects degraded (18.733%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T15:24:08.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:08.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:08.692 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:24:08.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:08.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:08.793 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:08.827 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:08.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:08 smithi027 ceph-mon[195233]: pgmap v946: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 631 op/s; 25988/138726 objects degraded (18.733%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T15:24:08.865 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:08.893 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:08.909 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.909 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:24:08.910 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:24:09.126 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:09.146 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:24:09.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:24:09.689 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1686190999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:09.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1686190999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1686190999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:10.185 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:10.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:10.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:10.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:10.378 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:10.484 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:10.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:10.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:10.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:10.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:10 smithi027 ceph-mon[195233]: pgmap v947: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 25988/136461 objects degraded (19.044%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T15:24:10.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:10 smithi027 ceph-mon[195233]: from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:10.640 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:10.888 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:10.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:10 smithi017 ceph-mon[215646]: pgmap v947: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 25988/136461 objects degraded (19.044%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T15:24:10.892 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:10 smithi017 ceph-mon[215646]: from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:10.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:10 smithi017 ceph-mon[222878]: pgmap v947: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 25988/136461 objects degraded (19.044%); 5.8 MiB/s, 92 objects/s recovering 2023-09-23T15:24:10.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:10 smithi017 ceph-mon[222878]: from='client.36087 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:10.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:10.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:10.937 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":947,"stamp":"2023-09-23T15:24:08.731170+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2983979377,"num_objects":45487,"num_object_clones":0,"num_object_copies":136461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25988,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45487,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":135740,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10407,"num_bytes_recovered":682724725,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110324,"ondisk_log_size":110324,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12601004,"kb_used_data":9484780,"kb_used_omap":44,"kb_used_meta":3116051,"kb_avail":737360212,"statfs":{"total":767960285184,"available":755056857088,"internally_reserved":0,"allocated":9712414720,"data_stored":9702777465,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3190837179},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,5,6,6,2,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-400293888,"num_objects":-6108,"num_object_clones":0,"num_object_copies":-18324,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1320,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6108,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6108,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":930,"num_bytes_recovered":60948480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003994"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.750867+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:23:50.750867+0000","last_peered":"2023-09-23T15:23:50.750867+0000","last_clean":"2023-09-23T15:23:50.750867+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:23:50.750867+0000","last_undegraded":"2023-09-23T15:23:50.750867+0000","last_fullsized":"2023-09-23T15:23:50.750867+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":281,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.267741+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:23:58.267741+0000","last_peered":"2023-09-23T15:23:58.267741+0000","last_clean":"2023-09-23T15:23:58.267741+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:23:58.267741+0000","last_undegraded":"2023-09-23T15:23:58.267741+0000","last_fullsized":"2023-09-23T15:23:58.267741+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":229,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.251620+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:23:58.251620+0000","last_peered":"2023-09-23T15:23:58.251620+0000","last_clean":"2023-09-23T15:23:58.251620+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:23:58.251620+0000","last_undegraded":"2023-09-23T15:23:58.251620+0000","last_fullsized":"2023-09-23T15:23:58.251620+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":302,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.757567+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:23:50.757567+0000","last_peered":"2023-09-23T15:23:50.757567+0000","last_clean":"2023-09-23T15:23:50.757567+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:23:50.757567+0000","last_undegraded":"2023-09-23T15:23:50.757567+0000","last_fullsized":"2023-09-23T15:23:50.757567+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":326,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.266198+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:23:58.266198+0000","last_peered":"2023-09-23T15:23:58.266198+0000","last_clean":"2023-09-23T15:23:58.266198+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:23:58.266198+0000","last_undegraded":"2023-09-23T15:23:58.266198+0000","last_fullsized":"2023-09-23T15:23:58.266198+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":274,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.756841+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:23:50.756841+0000","last_peered":"2023-09-23T15:23:50.756841+0000","last_clean":"2023-09-23T15:23:50.756841+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:23:50.756841+0000","last_undegraded":"2023-09-23T15:23:50.756841+0000","last_fullsized":"2023-09-23T15:23:50.756841+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":272,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.272685+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:23:58.272685+0000","last_peered":"2023-09-23T15:23:58.272685+0000","last_clean":"2023-09-23T15:23:58.272685+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:23:58.272685+0000","last_undegraded":"2023-09-23T15:23:58.272685+0000","last_fullsized":"2023-09-23T15:23:58.272685+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":323,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.266234+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:23:58.266234+0000","last_peered":"2023-09-23T15:23:58.266234+0000","last_clean":"2023-09-23T15:23:58.266234+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:23:58.266234+0000","last_undegraded":"2023-09-23T15:23:58.266234+0000","last_fullsized":"2023-09-23T15:23:58.266234+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":682,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.745292+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:23:50.745292+0000","last_peered":"2023-09-23T15:23:50.745292+0000","last_clean":"2023-09-23T15:23:50.745292+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:23:50.745292+0000","last_undegraded":"2023-09-23T15:23:50.745292+0000","last_fullsized":"2023-09-23T15:23:50.745292+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":324,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.766677+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:23:50.766677+0000","last_peered":"2023-09-23T15:23:50.766677+0000","last_clean":"2023-09-23T15:23:50.766677+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:23:50.766677+0000","last_undegraded":"2023-09-23T15:23:50.766677+0000","last_fullsized":"2023-09-23T15:23:50.766677+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":277,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.265695+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:23:58.265695+0000","last_peered":"2023-09-23T15:23:58.265695+0000","last_clean":"2023-09-23T15:23:58.265695+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:23:58.265695+0000","last_undegraded":"2023-09-23T15:23:58.265695+0000","last_fullsized":"2023-09-23T15:23:58.265695+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":68,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.788496+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:23:50.788496+0000","last_peered":"2023-09-23T15:23:50.788496+0000","last_clean":"2023-09-23T15:23:50.788496+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:23:50.788496+0000","last_undegraded":"2023-09-23T15:23:50.788496+0000","last_fullsized":"2023-09-23T15:23:50.788496+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":215,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.251730+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:23:58.251730+0000","last_peered":"2023-09-23T15:23:58.251730+0000","last_clean":"2023-09-23T15:23:58.251730+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:23:58.251730+0000","last_undegraded":"2023-09-23T15:23:58.251730+0000","last_fullsized":"2023-09-23T15:23:58.251730+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":58,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.752193+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:23:50.752193+0000","last_peered":"2023-09-23T15:23:50.752193+0000","last_clean":"2023-09-23T15:23:50.752193+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:23:50.752193+0000","last_undegraded":"2023-09-23T15:23:50.752193+0000","last_fullsized":"2023-09-23T15:23:50.752193+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751252+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:23:50.751252+0000","last_peered":"2023-09-23T15:23:50.751252+0000","last_clean":"2023-09-23T15:23:50.751252+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:23:50.751252+0000","last_undegraded":"2023-09-23T15:23:50.751252+0000","last_fullsized":"2023-09-23T15:23:50.751252+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":633,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.760117+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:23:50.760117+0000","last_peered":"2023-09-23T15:23:50.760117+0000","last_clean":"2023-09-23T15:23:50.760117+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:23:50.760117+0000","last_undegraded":"2023-09-23T15:23:50.760117+0000","last_fullsized":"2023-09-23T15:23:50.760117+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":72,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.770826+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:23:50.770826+0000","last_peered":"2023-09-23T15:23:50.770826+0000","last_clean":"2023-09-23T15:23:50.770826+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:23:50.770826+0000","last_undegraded":"2023-09-23T15:23:50.770826+0000","last_fullsized":"2023-09-23T15:23:50.770826+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":328,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.268544+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:23:58.268544+0000","last_peered":"2023-09-23T15:23:58.268544+0000","last_clean":"2023-09-23T15:23:58.268544+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:23:58.268544+0000","last_undegraded":"2023-09-23T15:23:58.268544+0000","last_fullsized":"2023-09-23T15:23:58.268544+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":682,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.767291+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:23:50.767291+0000","last_peered":"2023-09-23T15:23:50.767291+0000","last_clean":"2023-09-23T15:23:50.767291+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:23:50.767291+0000","last_undegraded":"2023-09-23T15:23:50.767291+0000","last_fullsized":"2023-09-23T15:23:50.767291+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":301,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.754946+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:23:50.754946+0000","last_peered":"2023-09-23T15:23:50.754946+0000","last_clean":"2023-09-23T15:23:50.754946+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:23:50.754946+0000","last_undegraded":"2023-09-23T15:23:50.754946+0000","last_fullsized":"2023-09-23T15:23:50.754946+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":309,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.768785+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:23:50.768785+0000","last_peered":"2023-09-23T15:23:50.768785+0000","last_clean":"2023-09-23T15:23:50.768785+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:23:50.768785+0000","last_undegraded":"2023-09-23T15:23:50.768785+0000","last_fullsized":"2023-09-23T15:23:50.768785+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":615,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.256932+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:23:58.256932+0000","last_peered":"2023-09-23T15:23:58.256932+0000","last_clean":"2023-09-23T15:23:58.256932+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:23:58.256932+0000","last_undegraded":"2023-09-23T15:23:58.256932+0000","last_fullsized":"2023-09-23T15:23:58.256932+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":72,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.772657+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:23:50.772657+0000","last_peered":"2023-09-23T15:23:50.772657+0000","last_clean":"2023-09-23T15:23:50.772657+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:23:50.772657+0000","last_undegraded":"2023-09-23T15:23:50.772657+0000","last_fullsized":"2023-09-23T15:23:50.772657+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":290,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.277321+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:23:58.277321+0000","last_peered":"2023-09-23T15:23:58.277321+0000","last_clean":"2023-09-23T15:23:58.277321+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:23:58.277321+0000","last_undegraded":"2023-09-23T15:23:58.277321+0000","last_fullsized":"2023-09-23T15:23:58.277321+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":640,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.769425+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:23:50.769425+0000","last_peered":"2023-09-23T15:23:50.769425+0000","last_clean":"2023-09-23T15:23:50.769425+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:23:50.769425+0000","last_undegraded":"2023-09-23T15:23:50.769425+0000","last_fullsized":"2023-09-23T15:23:50.769425+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":260,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755498+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:23:50.755498+0000","last_peered":"2023-09-23T15:23:50.755498+0000","last_clean":"2023-09-23T15:23:50.755498+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:23:50.755498+0000","last_undegraded":"2023-09-23T15:23:50.755498+0000","last_fullsized":"2023-09-23T15:23:50.755498+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":328,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.275643+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:23:58.275643+0000","last_peered":"2023-09-23T15:23:58.275643+0000","last_clean":"2023-09-23T15:23:58.275643+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:23:58.275643+0000","last_undegraded":"2023-09-23T15:23:58.275643+0000","last_fullsized":"2023-09-23T15:23:58.275643+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"324'8540","reported_seq":11132,"reported_epoch":324,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:04.875676+0000","last_change":"2023-09-23T15:23:39.629532+0000","last_active":"2023-09-23T15:24:04.875676+0000","last_peered":"2023-09-23T15:24:04.875676+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:58.288439+0000","last_became_peered":"2023-09-23T15:21:58.288439+0000","last_unstale":"2023-09-23T15:24:04.875676+0000","last_undegraded":"2023-09-23T15:21:58.149099+0000","last_fullsized":"2023-09-23T15:21:58.128266+0000","mapping_epoch":297,"log_start":"292'1700","ondisk_log_start":"292'1700","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6840,"log_dups_size":0,"ondisk_log_size":6840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:22:01.838822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188350464,"num_objects":2874,"num_object_clones":0,"num_object_copies":8622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1974,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2874,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8540,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1268,"num_bytes_recovered":83099648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":900}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":586,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.788648+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:23:50.788648+0000","last_peered":"2023-09-23T15:23:50.788648+0000","last_clean":"2023-09-23T15:23:50.788648+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:23:50.788648+0000","last_undegraded":"2023-09-23T15:23:50.788648+0000","last_fullsized":"2023-09-23T15:23:50.788648+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":64,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.771119+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:23:50.771119+0000","last_peered":"2023-09-23T15:23:50.771119+0000","last_clean":"2023-09-23T15:23:50.771119+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:23:50.771119+0000","last_undegraded":"2023-09-23T15:23:50.771119+0000","last_fullsized":"2023-09-23T15:23:50.771119+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":316,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.766001+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:23:50.766001+0000","last_peered":"2023-09-23T15:23:50.766001+0000","last_clean":"2023-09-23T15:23:50.766001+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:23:50.766001+0000","last_undegraded":"2023-09-23T15:23:50.766001+0000","last_fullsized":"2023-09-23T15:23:50.766001+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"324'8194","reported_seq":16237,"reported_epoch":324,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:04.440595+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:24:04.440595+0000","last_peered":"2023-09-23T15:24:04.440595+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:24:04.440595+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"292'1300","ondisk_log_start":"292'1300","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6894,"log_dups_size":0,"ondisk_log_size":6894,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2294,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8194,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2658,"num_bytes_recovered":174194688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1"],"object_location_counts":[{"shards":"0,1,7","objects":405},{"shards":"0,4,7","objects":1484},{"shards":"4,7","objects":991}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":645,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.268476+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:23:58.268476+0000","last_peered":"2023-09-23T15:23:58.268476+0000","last_clean":"2023-09-23T15:23:58.268476+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:23:58.268476+0000","last_undegraded":"2023-09-23T15:23:58.268476+0000","last_fullsized":"2023-09-23T15:23:58.268476+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":64,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.777466+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:23:50.777466+0000","last_peered":"2023-09-23T15:23:50.777466+0000","last_clean":"2023-09-23T15:23:50.777466+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:23:50.777466+0000","last_undegraded":"2023-09-23T15:23:50.777466+0000","last_fullsized":"2023-09-23T15:23:50.777466+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.750650+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:23:50.750650+0000","last_peered":"2023-09-23T15:23:50.750650+0000","last_clean":"2023-09-23T15:23:50.750650+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:23:50.750650+0000","last_undegraded":"2023-09-23T15:23:50.750650+0000","last_fullsized":"2023-09-23T15:23:50.750650+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"324'8377","reported_seq":8444,"reported_epoch":324,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:04.095113+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:24:04.095113+0000","last_peered":"2023-09-23T15:24:04.095113+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:24:04.095113+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"292'1800","ondisk_log_start":"292'1800","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185925632,"num_objects":2837,"num_object_clones":0,"num_object_copies":8511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8377,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1859},{"shards":"0,7","objects":-1793}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":531,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.757014+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:23:50.757014+0000","last_peered":"2023-09-23T15:23:50.757014+0000","last_clean":"2023-09-23T15:23:50.757014+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:23:50.757014+0000","last_undegraded":"2023-09-23T15:23:50.757014+0000","last_fullsized":"2023-09-23T15:23:50.757014+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":64,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.788410+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:23:50.788410+0000","last_peered":"2023-09-23T15:23:50.788410+0000","last_clean":"2023-09-23T15:23:50.788410+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:23:50.788410+0000","last_undegraded":"2023-09-23T15:23:50.788410+0000","last_fullsized":"2023-09-23T15:23:50.788410+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":287,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755555+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:23:50.755555+0000","last_peered":"2023-09-23T15:23:50.755555+0000","last_clean":"2023-09-23T15:23:50.755555+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:23:50.755555+0000","last_undegraded":"2023-09-23T15:23:50.755555+0000","last_fullsized":"2023-09-23T15:23:50.755555+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"325'8552","reported_seq":8608,"reported_epoch":325,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:06.900955+0000","last_change":"2023-09-23T15:22:27.588207+0000","last_active":"2023-09-23T15:24:06.900955+0000","last_peered":"2023-09-23T15:24:06.900955+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:24:06.900955+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"293'2000","ondisk_log_start":"293'2000","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6552,"log_dups_size":0,"ondisk_log_size":6552,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180879360,"num_objects":2760,"num_object_clones":0,"num_object_copies":8280,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2760,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8552,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-1896}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":539,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751775+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:23:50.751775+0000","last_peered":"2023-09-23T15:23:50.751775+0000","last_clean":"2023-09-23T15:23:50.751775+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:23:50.751775+0000","last_undegraded":"2023-09-23T15:23:50.751775+0000","last_fullsized":"2023-09-23T15:23:50.751775+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":300,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.768475+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:23:50.768475+0000","last_peered":"2023-09-23T15:23:50.768475+0000","last_clean":"2023-09-23T15:23:50.768475+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:23:50.768475+0000","last_undegraded":"2023-09-23T15:23:50.768475+0000","last_fullsized":"2023-09-23T15:23:50.768475+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":305,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755175+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:23:50.755175+0000","last_peered":"2023-09-23T15:23:50.755175+0000","last_clean":"2023-09-23T15:23:50.755175+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:23:50.755175+0000","last_undegraded":"2023-09-23T15:23:50.755175+0000","last_fullsized":"2023-09-23T15:23:50.755175+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"325'8419","reported_seq":8470,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.565256+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:24:05.565256+0000","last_peered":"2023-09-23T15:24:05.565256+0000","last_clean":"2023-09-23T15:24:05.565256+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:24:05.565256+0000","last_undegraded":"2023-09-23T15:24:05.565256+0000","last_fullsized":"2023-09-23T15:24:05.565256+0000","mapping_epoch":284,"log_start":"293'2000","ondisk_log_start":"293'2000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187105280,"num_objects":2855,"num_object_clones":0,"num_object_copies":8565,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2855,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8419,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":560,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.772841+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:23:50.772841+0000","last_peered":"2023-09-23T15:23:50.772841+0000","last_clean":"2023-09-23T15:23:50.772841+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:23:50.772841+0000","last_undegraded":"2023-09-23T15:23:50.772841+0000","last_fullsized":"2023-09-23T15:23:50.772841+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":317,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.266454+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:23:58.266454+0000","last_peered":"2023-09-23T15:23:58.266454+0000","last_clean":"2023-09-23T15:23:58.266454+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:23:58.266454+0000","last_undegraded":"2023-09-23T15:23:58.266454+0000","last_fullsized":"2023-09-23T15:23:58.266454+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.752039+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:23:50.752039+0000","last_peered":"2023-09-23T15:23:50.752039+0000","last_clean":"2023-09-23T15:23:50.752039+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:23:50.752039+0000","last_undegraded":"2023-09-23T15:23:50.752039+0000","last_fullsized":"2023-09-23T15:23:50.752039+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"325'8701","reported_seq":8749,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:06.853288+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:24:06.853288+0000","last_peered":"2023-09-23T15:24:06.853288+0000","last_clean":"2023-09-23T15:24:06.853288+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:24:06.853288+0000","last_undegraded":"2023-09-23T15:24:06.853288+0000","last_fullsized":"2023-09-23T15:24:06.853288+0000","mapping_epoch":284,"log_start":"293'2100","ondisk_log_start":"293'2100","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6601,"log_dups_size":0,"ondisk_log_size":6601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186187776,"num_objects":2841,"num_object_clones":0,"num_object_copies":8523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8701,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":628,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.767365+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:23:50.767365+0000","last_peered":"2023-09-23T15:23:50.767365+0000","last_clean":"2023-09-23T15:23:50.767365+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:23:50.767365+0000","last_undegraded":"2023-09-23T15:23:50.767365+0000","last_fullsized":"2023-09-23T15:23:50.767365+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":271,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755639+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:23:50.755639+0000","last_peered":"2023-09-23T15:23:50.755639+0000","last_clean":"2023-09-23T15:23:50.755639+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:23:50.755639+0000","last_undegraded":"2023-09-23T15:23:50.755639+0000","last_fullsized":"2023-09-23T15:23:50.755639+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.754282+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:23:50.754282+0000","last_peered":"2023-09-23T15:23:50.754282+0000","last_clean":"2023-09-23T15:23:50.754282+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:23:50.754282+0000","last_undegraded":"2023-09-23T15:23:50.754282+0000","last_fullsized":"2023-09-23T15:23:50.754282+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"324'8592","reported_seq":15139,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:24:04.602582+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:24:04.602582+0000","last_peered":"2023-09-23T15:24:04.602582+0000","last_clean":"2023-09-23T15:24:04.602582+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:24:04.602582+0000","last_undegraded":"2023-09-23T15:24:04.602582+0000","last_fullsized":"2023-09-23T15:24:04.602582+0000","mapping_epoch":310,"log_start":"293'2000","ondisk_log_start":"293'2000","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6592,"log_dups_size":0,"ondisk_log_size":6592,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":186777600,"num_objects":2850,"num_object_clones":0,"num_object_copies":8550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8592,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":581,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.256828+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:23:58.256828+0000","last_peered":"2023-09-23T15:23:58.256828+0000","last_clean":"2023-09-23T15:23:58.256828+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:23:58.256828+0000","last_undegraded":"2023-09-23T15:23:58.256828+0000","last_fullsized":"2023-09-23T15:23:58.256828+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.750717+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:23:50.750717+0000","last_peered":"2023-09-23T15:23:50.750717+0000","last_clean":"2023-09-23T15:23:50.750717+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:23:50.750717+0000","last_undegraded":"2023-09-23T15:23:50.750717+0000","last_fullsized":"2023-09-23T15:23:50.750717+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751688+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:23:50.751688+0000","last_peered":"2023-09-23T15:23:50.751688+0000","last_clean":"2023-09-23T15:23:50.751688+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:23:50.751688+0000","last_undegraded":"2023-09-23T15:23:50.751688+0000","last_fullsized":"2023-09-23T15:23:50.751688+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"325'8519","reported_seq":8575,"reported_epoch":325,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:06.945502+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:24:06.945502+0000","last_peered":"2023-09-23T15:24:06.945502+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:24:06.945502+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6619,"log_dups_size":0,"ondisk_log_size":6619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182124544,"num_objects":2779,"num_object_clones":0,"num_object_copies":8337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8519,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-1839}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":640,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.769296+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:23:50.769296+0000","last_peered":"2023-09-23T15:23:50.769296+0000","last_clean":"2023-09-23T15:23:50.769296+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:23:50.769296+0000","last_undegraded":"2023-09-23T15:23:50.769296+0000","last_fullsized":"2023-09-23T15:23:50.769296+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":339,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.266158+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:23:58.266158+0000","last_peered":"2023-09-23T15:23:58.266158+0000","last_clean":"2023-09-23T15:23:58.266158+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:23:58.266158+0000","last_undegraded":"2023-09-23T15:23:58.266158+0000","last_fullsized":"2023-09-23T15:23:58.266158+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":233,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:24:01.958699+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:24:01.958699+0000","last_peered":"2023-09-23T15:24:01.958699+0000","last_clean":"2023-09-23T15:24:01.958699+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:24:01.958699+0000","last_undegraded":"2023-09-23T15:24:01.958699+0000","last_fullsized":"2023-09-23T15:24:01.958699+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"324'8240","reported_seq":8307,"reported_epoch":324,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:04.037984+0000","last_change":"2023-09-23T15:21:58.303371+0000","last_active":"2023-09-23T15:24:04.037984+0000","last_peered":"2023-09-23T15:24:04.037984+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:24:04.037984+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"292'1700","ondisk_log_start":"292'1700","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185729024,"num_objects":2834,"num_object_clones":0,"num_object_copies":8502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8240,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":-259},{"shards":"0,2,7","objects":390}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":521,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.756922+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:23:50.756922+0000","last_peered":"2023-09-23T15:23:50.756922+0000","last_clean":"2023-09-23T15:23:50.756922+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:23:50.756922+0000","last_undegraded":"2023-09-23T15:23:50.756922+0000","last_fullsized":"2023-09-23T15:23:50.756922+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":524,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.252073+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:23:58.252073+0000","last_peered":"2023-09-23T15:23:58.252073+0000","last_clean":"2023-09-23T15:23:58.252073+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:23:58.252073+0000","last_undegraded":"2023-09-23T15:23:58.252073+0000","last_fullsized":"2023-09-23T15:23:58.252073+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.768701+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:23:50.768701+0000","last_peered":"2023-09-23T15:23:50.768701+0000","last_clean":"2023-09-23T15:23:50.768701+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:23:50.768701+0000","last_undegraded":"2023-09-23T15:23:50.768701+0000","last_fullsized":"2023-09-23T15:23:50.768701+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":267,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.743124+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:23:50.743124+0000","last_peered":"2023-09-23T15:23:50.743124+0000","last_clean":"2023-09-23T15:23:50.743124+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:23:50.743124+0000","last_undegraded":"2023-09-23T15:23:50.743124+0000","last_fullsized":"2023-09-23T15:23:50.743124+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"324'8391","reported_seq":8449,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:24:04.784004+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:24:04.784004+0000","last_peered":"2023-09-23T15:24:04.784004+0000","last_clean":"2023-09-23T15:24:04.784004+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:24:04.784004+0000","last_undegraded":"2023-09-23T15:24:04.784004+0000","last_fullsized":"2023-09-23T15:24:04.784004+0000","mapping_epoch":287,"log_start":"292'1500","ondisk_log_start":"292'1500","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6891,"log_dups_size":0,"ondisk_log_size":6891,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185663488,"num_objects":2833,"num_object_clones":0,"num_object_copies":8499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8391,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":621,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.766890+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:23:50.766890+0000","last_peered":"2023-09-23T15:23:50.766890+0000","last_clean":"2023-09-23T15:23:50.766890+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:23:50.766890+0000","last_undegraded":"2023-09-23T15:23:50.766890+0000","last_fullsized":"2023-09-23T15:23:50.766890+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":859,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.769653+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:23:50.769653+0000","last_peered":"2023-09-23T15:23:50.769653+0000","last_clean":"2023-09-23T15:23:50.769653+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:23:50.769653+0000","last_undegraded":"2023-09-23T15:23:50.769653+0000","last_fullsized":"2023-09-23T15:23:50.769653+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751196+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:23:50.751196+0000","last_peered":"2023-09-23T15:23:50.751196+0000","last_clean":"2023-09-23T15:23:50.751196+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:23:50.751196+0000","last_undegraded":"2023-09-23T15:23:50.751196+0000","last_fullsized":"2023-09-23T15:23:50.751196+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":221,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.251889+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:23:58.251889+0000","last_peered":"2023-09-23T15:23:58.251889+0000","last_clean":"2023-09-23T15:23:58.251889+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:23:58.251889+0000","last_undegraded":"2023-09-23T15:23:58.251889+0000","last_fullsized":"2023-09-23T15:23:58.251889+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"325'8314","reported_seq":14601,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.537903+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:24:05.537903+0000","last_peered":"2023-09-23T15:24:05.537903+0000","last_clean":"2023-09-23T15:24:05.537903+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:24:05.537903+0000","last_undegraded":"2023-09-23T15:24:05.537903+0000","last_fullsized":"2023-09-23T15:24:05.537903+0000","mapping_epoch":320,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8314,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":679,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.770590+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:23:50.770590+0000","last_peered":"2023-09-23T15:23:50.770590+0000","last_clean":"2023-09-23T15:23:50.770590+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:23:50.770590+0000","last_undegraded":"2023-09-23T15:23:50.770590+0000","last_fullsized":"2023-09-23T15:23:50.770590+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":940,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.743587+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:23:50.743587+0000","last_peered":"2023-09-23T15:23:50.743587+0000","last_clean":"2023-09-23T15:23:50.743587+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:23:50.743587+0000","last_undegraded":"2023-09-23T15:23:50.743587+0000","last_fullsized":"2023-09-23T15:23:50.743587+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.251368+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:23:58.251368+0000","last_peered":"2023-09-23T15:23:58.251368+0000","last_clean":"2023-09-23T15:23:58.251368+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:23:58.251368+0000","last_undegraded":"2023-09-23T15:23:58.251368+0000","last_fullsized":"2023-09-23T15:23:58.251368+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":225,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.252773+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:23:58.252773+0000","last_peered":"2023-09-23T15:23:58.252773+0000","last_clean":"2023-09-23T15:23:58.252773+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:23:58.252773+0000","last_undegraded":"2023-09-23T15:23:58.252773+0000","last_fullsized":"2023-09-23T15:23:58.252773+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"324'8440","reported_seq":8490,"reported_epoch":324,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:04.090609+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:24:04.090609+0000","last_peered":"2023-09-23T15:24:04.090609+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:24:04.090609+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":191889408,"num_objects":2928,"num_object_clones":0,"num_object_copies":8784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8440,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-1778}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":653,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.767037+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:23:50.767037+0000","last_peered":"2023-09-23T15:23:50.767037+0000","last_clean":"2023-09-23T15:23:50.767037+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:23:50.767037+0000","last_undegraded":"2023-09-23T15:23:50.767037+0000","last_fullsized":"2023-09-23T15:23:50.767037+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":643,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.770535+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:23:50.770535+0000","last_peered":"2023-09-23T15:23:50.770535+0000","last_clean":"2023-09-23T15:23:50.770535+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:23:50.770535+0000","last_undegraded":"2023-09-23T15:23:50.770535+0000","last_fullsized":"2023-09-23T15:23:50.770535+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":258,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.265950+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:23:58.265950+0000","last_peered":"2023-09-23T15:23:58.265950+0000","last_clean":"2023-09-23T15:23:58.265950+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:23:58.265950+0000","last_undegraded":"2023-09-23T15:23:58.265950+0000","last_fullsized":"2023-09-23T15:23:58.265950+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":275,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.756550+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:23:50.756550+0000","last_peered":"2023-09-23T15:23:50.756550+0000","last_clean":"2023-09-23T15:23:50.756550+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:23:50.756550+0000","last_undegraded":"2023-09-23T15:23:50.756550+0000","last_fullsized":"2023-09-23T15:23:50.756550+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"325'8388","reported_seq":8439,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:06.924634+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:24:06.924634+0000","last_peered":"2023-09-23T15:24:06.924634+0000","last_clean":"2023-09-23T15:24:06.924634+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:24:06.924634+0000","last_undegraded":"2023-09-23T15:24:06.924634+0000","last_fullsized":"2023-09-23T15:24:06.924634+0000","mapping_epoch":284,"log_start":"293'1800","ondisk_log_start":"293'1800","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6588,"log_dups_size":0,"ondisk_log_size":6588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181010432,"num_objects":2762,"num_object_clones":0,"num_object_copies":8286,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2762,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8388,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":631,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.256724+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:23:58.256724+0000","last_peered":"2023-09-23T15:23:58.256724+0000","last_clean":"2023-09-23T15:23:58.256724+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:23:58.256724+0000","last_undegraded":"2023-09-23T15:23:58.256724+0000","last_fullsized":"2023-09-23T15:23:58.256724+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":68,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.770448+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:23:50.770448+0000","last_peered":"2023-09-23T15:23:50.770448+0000","last_clean":"2023-09-23T15:23:50.770448+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:23:50.770448+0000","last_undegraded":"2023-09-23T15:23:50.770448+0000","last_fullsized":"2023-09-23T15:23:50.770448+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":291,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.760037+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:23:50.760037+0000","last_peered":"2023-09-23T15:23:50.760037+0000","last_clean":"2023-09-23T15:23:50.760037+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:23:50.760037+0000","last_undegraded":"2023-09-23T15:23:50.760037+0000","last_fullsized":"2023-09-23T15:23:50.760037+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":302,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.757401+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:23:50.757401+0000","last_peered":"2023-09-23T15:23:50.757401+0000","last_clean":"2023-09-23T15:23:50.757401+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:23:50.757401+0000","last_undegraded":"2023-09-23T15:23:50.757401+0000","last_fullsized":"2023-09-23T15:23:50.757401+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"325'8600","reported_seq":8651,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.577717+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:24:05.577717+0000","last_peered":"2023-09-23T15:24:05.577717+0000","last_clean":"2023-09-23T15:24:05.577717+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:24:05.577717+0000","last_undegraded":"2023-09-23T15:24:05.577717+0000","last_fullsized":"2023-09-23T15:24:05.577717+0000","mapping_epoch":284,"log_start":"293'2200","ondisk_log_start":"293'2200","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6400,"log_dups_size":0,"ondisk_log_size":6400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193331200,"num_objects":2950,"num_object_clones":0,"num_object_copies":8850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8600,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":578,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.756597+0000","last_change":"2023-09-23T15:23:50.575734+0000","last_active":"2023-09-23T15:23:50.756597+0000","last_peered":"2023-09-23T15:23:50.756597+0000","last_clean":"2023-09-23T15:23:50.756597+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:23:50.756597+0000","last_undegraded":"2023-09-23T15:23:50.756597+0000","last_fullsized":"2023-09-23T15:23:50.756597+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1017,"reported_epoch":323,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:50.773109+0000","last_change":"2023-09-23T15:22:27.435515+0000","last_active":"2023-09-23T15:23:50.773109+0000","last_peered":"2023-09-23T15:23:50.773109+0000","last_clean":"2023-09-23T15:21:46.756689+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:23:50.773109+0000","last_undegraded":"2023-09-23T15:22:27.290223+0000","last_fullsized":"2023-09-23T15:23:50.773109+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":201,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.251987+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:23:58.251987+0000","last_peered":"2023-09-23T15:23:58.251987+0000","last_clean":"2023-09-23T15:23:58.251987+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:23:58.251987+0000","last_undegraded":"2023-09-23T15:23:58.251987+0000","last_fullsized":"2023-09-23T15:23:58.251987+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":287,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755398+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:23:50.755398+0000","last_peered":"2023-09-23T15:23:50.755398+0000","last_clean":"2023-09-23T15:23:50.755398+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:23:50.755398+0000","last_undegraded":"2023-09-23T15:23:50.755398+0000","last_fullsized":"2023-09-23T15:23:50.755398+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"324'8217","reported_seq":8276,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:24:04.108066+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:24:04.108066+0000","last_peered":"2023-09-23T15:24:04.108066+0000","last_clean":"2023-09-23T15:24:04.108066+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:24:04.108066+0000","last_undegraded":"2023-09-23T15:24:04.108066+0000","last_fullsized":"2023-09-23T15:24:04.108066+0000","mapping_epoch":287,"log_start":"292'1600","ondisk_log_start":"292'1600","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6617,"log_dups_size":0,"ondisk_log_size":6617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190251008,"num_objects":2903,"num_object_clones":0,"num_object_copies":8709,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2903,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8217,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":632,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.754204+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:23:50.754204+0000","last_peered":"2023-09-23T15:23:50.754204+0000","last_clean":"2023-09-23T15:23:50.754204+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:23:50.754204+0000","last_undegraded":"2023-09-23T15:23:50.754204+0000","last_fullsized":"2023-09-23T15:23:50.754204+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":79,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.743397+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:23:50.743397+0000","last_peered":"2023-09-23T15:23:50.743397+0000","last_clean":"2023-09-23T15:23:50.743397+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:23:50.743397+0000","last_undegraded":"2023-09-23T15:23:50.743397+0000","last_fullsized":"2023-09-23T15:23:50.743397+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":856,"reported_epoch":324,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:23:58.252303+0000","last_change":"2023-09-23T15:22:27.316872+0000","last_active":"2023-09-23T15:23:58.252303+0000","last_peered":"2023-09-23T15:23:58.252303+0000","last_clean":"2023-09-23T15:21:04.172426+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:23:58.252303+0000","last_undegraded":"2023-09-23T15:22:27.289016+0000","last_fullsized":"2023-09-23T15:23:58.252303+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":318,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.266277+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:23:58.266277+0000","last_peered":"2023-09-23T15:23:58.266277+0000","last_clean":"2023-09-23T15:23:58.266277+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:23:58.266277+0000","last_undegraded":"2023-09-23T15:23:58.266277+0000","last_fullsized":"2023-09-23T15:23:58.266277+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"324'8327","reported_seq":8383,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:24:04.828807+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:24:04.828807+0000","last_peered":"2023-09-23T15:24:04.828807+0000","last_clean":"2023-09-23T15:24:04.828807+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:24:04.828807+0000","last_undegraded":"2023-09-23T15:24:04.828807+0000","last_fullsized":"2023-09-23T15:24:04.828807+0000","mapping_epoch":287,"log_start":"292'1500","ondisk_log_start":"292'1500","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6827,"log_dups_size":0,"ondisk_log_size":6827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181927960,"num_objects":2777,"num_object_clones":0,"num_object_copies":8331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2777,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8327,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":649,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.268401+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:23:58.268401+0000","last_peered":"2023-09-23T15:23:58.268401+0000","last_clean":"2023-09-23T15:23:58.268401+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:23:58.268401+0000","last_undegraded":"2023-09-23T15:23:58.268401+0000","last_fullsized":"2023-09-23T15:23:58.268401+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":760,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.756767+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:23:50.756767+0000","last_peered":"2023-09-23T15:23:50.756767+0000","last_clean":"2023-09-23T15:23:50.756767+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:23:50.756767+0000","last_undegraded":"2023-09-23T15:23:50.756767+0000","last_fullsized":"2023-09-23T15:23:50.756767+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":329,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.757494+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:23:50.757494+0000","last_peered":"2023-09-23T15:23:50.757494+0000","last_clean":"2023-09-23T15:23:50.757494+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:23:50.757494+0000","last_undegraded":"2023-09-23T15:23:50.757494+0000","last_fullsized":"2023-09-23T15:23:50.757494+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.750631+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:23:50.750631+0000","last_peered":"2023-09-23T15:23:50.750631+0000","last_clean":"2023-09-23T15:23:50.750631+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:23:50.750631+0000","last_undegraded":"2023-09-23T15:23:50.750631+0000","last_fullsized":"2023-09-23T15:23:50.750631+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":306,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755313+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:23:50.755313+0000","last_peered":"2023-09-23T15:23:50.755313+0000","last_clean":"2023-09-23T15:23:50.755313+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:23:50.755313+0000","last_undegraded":"2023-09-23T15:23:50.755313+0000","last_fullsized":"2023-09-23T15:23:50.755313+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":327,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.745422+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:23:50.745422+0000","last_peered":"2023-09-23T15:23:50.745422+0000","last_clean":"2023-09-23T15:23:50.745422+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:23:50.745422+0000","last_undegraded":"2023-09-23T15:23:50.745422+0000","last_fullsized":"2023-09-23T15:23:50.745422+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":337,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.768547+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:23:50.768547+0000","last_peered":"2023-09-23T15:23:50.768547+0000","last_clean":"2023-09-23T15:23:50.768547+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:23:50.768547+0000","last_undegraded":"2023-09-23T15:23:50.768547+0000","last_fullsized":"2023-09-23T15:23:50.768547+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751330+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:23:50.751330+0000","last_peered":"2023-09-23T15:23:50.751330+0000","last_clean":"2023-09-23T15:23:50.751330+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:23:50.751330+0000","last_undegraded":"2023-09-23T15:23:50.751330+0000","last_fullsized":"2023-09-23T15:23:50.751330+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":317,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.768607+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:23:50.768607+0000","last_peered":"2023-09-23T15:23:50.768607+0000","last_clean":"2023-09-23T15:23:50.768607+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:23:50.768607+0000","last_undegraded":"2023-09-23T15:23:50.768607+0000","last_fullsized":"2023-09-23T15:23:50.768607+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":37,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.751001+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:23:50.751001+0000","last_peered":"2023-09-23T15:23:50.751001+0000","last_clean":"2023-09-23T15:23:50.751001+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:23:50.751001+0000","last_undegraded":"2023-09-23T15:23:50.751001+0000","last_fullsized":"2023-09-23T15:23:50.751001+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":315,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.268138+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:23:58.268138+0000","last_peered":"2023-09-23T15:23:58.268138+0000","last_clean":"2023-09-23T15:23:58.268138+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:23:58.268138+0000","last_undegraded":"2023-09-23T15:23:58.268138+0000","last_fullsized":"2023-09-23T15:23:58.268138+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":314,"reported_epoch":324,"state":"active+clean","last_fresh":"2023-09-23T15:23:58.265621+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:23:58.265621+0000","last_peered":"2023-09-23T15:23:58.265621+0000","last_clean":"2023-09-23T15:23:58.265621+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:23:58.265621+0000","last_undegraded":"2023-09-23T15:23:58.265621+0000","last_fullsized":"2023-09-23T15:23:58.265621+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":276,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.755503+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:23:50.755503+0000","last_peered":"2023-09-23T15:23:50.755503+0000","last_clean":"2023-09-23T15:23:50.755503+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:23:50.755503+0000","last_undegraded":"2023-09-23T15:23:50.755503+0000","last_fullsized":"2023-09-23T15:23:50.755503+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":51,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T15:23:50.750429+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:23:50.750429+0000","last_peered":"2023-09-23T15:23:50.750429+0000","last_clean":"2023-09-23T15:23:50.750429+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:23:50.750429+0000","last_undegraded":"2023-09-23T15:23:50.750429+0000","last_fullsized":"2023-09-23T15:23:50.750429+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2979135512,"num_objects":45459,"num_object_clones":0,"num_object_copies":136377,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":25971,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":45459,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134811,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10279,"num_bytes_recovered":673644544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9631117312,"data_stored":9631105096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105911,"ondisk_log_size":105911,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201632,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1249144,"kb_used_data":1118640,"kb_used_omap":0,"kb_used_meta":130496,"kb_avail":92496008,"statfs":{"total":95995035648,"available":94715912192,"internally_reserved":0,"allocated":1145487360,"data_stored":1144497365,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":133627904},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":7.2160000000000002,"15min":3.8849999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":7.7629999999999999,"5min":226.226,"15min":226.226},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.9399999999999999,"5min":8.5329999999999995,"15min":4.5190000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":9.4749999999999996,"5min":226.619,"15min":226.619},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.3680000000000001,"15min":2.1480000000000001},"min":{"1min":0.32600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.4290000000000003,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.768,"5min":1.585,"15min":2.569},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.1840000000000002,"5min":7.2510000000000003,"15min":16.065999999999999},"last":1.76}]},{"osd":2,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.4630000000000001,"15min":3.9340000000000002},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":6.6799999999999997,"5min":90.227999999999994,"15min":170.63800000000001},"last":2.7829999999999999},{"interface":"front","average":{"1min":1.889,"5min":2.778,"15min":4.5090000000000003},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2059999999999995,"5min":90.197999999999993,"15min":170.47999999999999},"last":0.47699999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:24:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8319999999999999,"5min":4.5869999999999997,"15min":3.9220000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":41.588999999999999,"5min":146.166,"15min":146.166},"last":2.5750000000000002},{"interface":"front","average":{"1min":5.4000000000000004,"5min":4.8520000000000003,"15min":3.98},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":41.542000000000002,"5min":146.18299999999999,"15min":146.18299999999999},"last":2.5899999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":1.181,"15min":2.5990000000000002},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6739999999999995,"5min":16.771999999999998,"15min":150.119},"last":1.8260000000000001},{"interface":"front","average":{"1min":2.302,"5min":0.92300000000000004,"15min":2.7919999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.298,"5min":9.298,"15min":150.596},"last":0.441}]},{"osd":5,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":2.8079999999999998,"15min":2.8479999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":10.153,"5min":224.441,"15min":224.441},"last":0.496},{"interface":"front","average":{"1min":2.7400000000000002,"5min":3.056,"15min":2.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.6600000000000001,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.2320000000000002,"15min":2.2320000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.2629999999999999,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":2.0489999999999999},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.336,"5min":10.336,"15min":10.336},"last":0.77600000000000002}]}]},{"osd":6,"up_from":304,"seq":1305670058005,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":389420,"kb_used_data":322148,"kb_used_omap":13,"kb_used_meta":67250,"kb_avail":93355732,"statfs":{"total":95995035648,"available":95596269568,"internally_reserved":0,"allocated":329879552,"data_stored":328901694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":68864355},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":5.7649999999999997},"last":4.4630000000000001},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.8780000000000001,"15min":2.8780000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.044,"5min":10.044,"15min":10.044},"last":6.4169999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.742,"15min":2.742},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.064,"5min":11.064,"15min":11.064},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":17.001999999999999,"5min":17.001999999999999,"15min":17.001999999999999},"last":7.016}]},{"osd":2,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.526,"15min":1.526},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":4.5099999999999998},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":1.1160000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":2.2799999999999998,"15min":2.2799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.0449999999999999,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":4.6319999999999997},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5049999999999999,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":4.7110000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.1550000000000002,"5min":5.1550000000000002,"15min":5.1550000000000002},"last":8.1219999999999999},{"interface":"front","average":{"1min":1.821,"5min":1.821,"15min":1.821},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":9.1929999999999996},"last":7.4939999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":12.711,"5min":12.711,"15min":12.711},"last":5.056},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.1950000000000003,"5min":5.1950000000000003,"15min":5.1950000000000003},"last":4.423}]},{"osd":7,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.0609999999999999,"5min":5.0609999999999999,"15min":5.0609999999999999},"last":4.6749999999999998},{"interface":"front","average":{"1min":1.823,"5min":1.823,"15min":1.823},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":1.2070000000000001}]}]},{"osd":1,"up_from":183,"seq":785979015306,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1909348,"kb_used_data":1462684,"kb_used_omap":6,"kb_used_meta":446649,"kb_avail":91835804,"statfs":{"total":95995035648,"available":94039863296,"internally_reserved":0,"allocated":1497788416,"data_stored":1496255881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":457368651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.4409999999999998,"5min":3.9390000000000001,"15min":3.5049999999999999},"min":{"1min":0.22500000000000001,"5min":0.191,"15min":0.191},"max":{"1min":9.5739999999999998,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.23300000000000001},{"interface":"front","average":{"1min":4.0019999999999998,"5min":4.0300000000000002,"15min":4.2030000000000003},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":15.537000000000001,"5min":63.594999999999999,"15min":63.594999999999999},"last":1.3500000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":4.2759999999999998,"15min":4.1550000000000002},"min":{"1min":0.26900000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":6.1600000000000001,"5min":109.375,"15min":109.375},"last":1.6579999999999999},{"interface":"front","average":{"1min":2.7269999999999999,"5min":3.7309999999999999,"15min":3.722},"min":{"1min":0.32600000000000001,"5min":0.219,"15min":0.219},"max":{"1min":9.6259999999999994,"5min":105.16,"15min":105.16},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":5.5490000000000004,"5min":6.399,"15min":8.2629999999999999},"min":{"1min":0.33100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":64.540999999999997,"5min":110.285,"15min":211.405},"last":0.70699999999999996},{"interface":"front","average":{"1min":6.4770000000000003,"5min":6.6900000000000004,"15min":9.0289999999999999},"min":{"1min":0.31900000000000001,"5min":0.28599999999999998,"15min":0.27200000000000002},"max":{"1min":67.465000000000003,"5min":110.095,"15min":210.988},"last":1.4630000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":3.3580000000000001,"15min":3.294},"min":{"1min":0.34200000000000003,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.3289999999999997,"5min":80.570999999999998,"15min":80.570999999999998},"last":0.64200000000000002},{"interface":"front","average":{"1min":2.7040000000000002,"5min":3.4550000000000001,"15min":3.5249999999999999},"min":{"1min":0.38100000000000001,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":9.6600000000000001,"5min":80.75,"15min":80.75},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":3.1509999999999998,"5min":2.5110000000000001,"15min":3.0649999999999999},"min":{"1min":0.39500000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":9.3659999999999997,"5min":34.512999999999998,"15min":34.512999999999998},"last":1.3879999999999999},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.5870000000000002,"15min":2.8639999999999999},"min":{"1min":0.37,"5min":0.311,"15min":0.311},"max":{"1min":8.9559999999999995,"5min":37.911000000000001,"15min":37.911000000000001},"last":0.82499999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:23:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.0710000000000002,"15min":3.0710000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":12.26,"5min":12.26,"15min":12.26},"last":0.71599999999999997},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.3230000000000004,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":1.423}]},{"osd":7,"last update":"Sat Sep 23 15:23:07 2023","interfaces":[{"interface":"back","average":{"1min":2.3210000000000002,"5min":2.2549999999999999,"15min":2.8050000000000002},"min":{"1min":0.45400000000000001,"5min":0.371,"15min":0.371},"max":{"1min":6.0339999999999998,"5min":12.861000000000001,"15min":12.861000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":2.6120000000000001,"5min":2.0510000000000002,"15min":2.6400000000000001},"min":{"1min":0.442,"5min":0.377,"15min":0.377},"max":{"1min":9.6929999999999996,"5min":13.282999999999999,"15min":16.841000000000001},"last":0.626}]}]},{"osd":0,"up_from":95,"seq":408021893355,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1981640,"kb_used_data":1408960,"kb_used_omap":8,"kb_used_meta":572663,"kb_avail":91763512,"statfs":{"total":95995035648,"available":93965836288,"internally_reserved":0,"allocated":1442775040,"data_stored":1441379718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":586407508},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.8519999999999999,"15min":3.468},"min":{"1min":0.28399999999999997,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":17.542000000000002,"5min":48.442999999999998,"15min":64.412999999999997},"last":17.542000000000002},{"interface":"front","average":{"1min":3.3239999999999998,"5min":2.7250000000000001,"15min":3.387},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":12.058999999999999,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":2.7759999999999998,"15min":4.5229999999999997},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":10.949,"5min":30.510999999999999,"15min":122.20099999999999},"last":13.022},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.3069999999999999,"15min":4.3010000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.18099999999999999},"max":{"1min":10.861000000000001,"5min":64.466999999999999,"15min":122.071},"last":9.2050000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:23:48 2023","interfaces":[{"interface":"back","average":{"1min":12.212999999999999,"5min":5.2119999999999997,"15min":5.7519999999999998},"min":{"1min":0.39100000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":197.78899999999999,"5min":197.78899999999999,"15min":197.78899999999999},"last":4.0570000000000004},{"interface":"front","average":{"1min":12.901999999999999,"5min":5.6210000000000004,"15min":6.226},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":200.58799999999999,"5min":200.58799999999999,"15min":200.58799999999999},"last":17.600999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":1.8899999999999999,"15min":2.1419999999999999},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.358000000000001,"5min":15.358000000000001,"15min":31.341999999999999},"last":3.0830000000000002},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.964,"15min":2.246},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":99.168999999999997},"last":3.1379999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.492,"15min":2.411},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.768000000000001,"5min":64.349999999999994,"15min":109.369},"last":9.1590000000000007},{"interface":"front","average":{"1min":3.6789999999999998,"5min":1.9990000000000001,"15min":2.355},"min":{"1min":0.497,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":15.436999999999999,"5min":15.436999999999999,"15min":109.387},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.50900000000000001},{"interface":"front","average":{"1min":3.3610000000000002,"5min":3.3610000000000002,"15min":3.3610000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":17.385000000000002,"5min":17.385000000000002,"15min":17.385000000000002},"last":2.9129999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.5550000000000002,"15min":2.488},"min":{"1min":0.42499999999999999,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":11.734,"5min":64.304000000000002,"15min":99.549999999999997},"last":17.712},{"interface":"front","average":{"1min":4.9279999999999999,"5min":2.3039999999999998,"15min":2.1720000000000002},"min":{"1min":0.45900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":31.225999999999999},"last":0.45300000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566302,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2175672,"kb_used_data":1578864,"kb_used_omap":11,"kb_used_meta":596788,"kb_avail":91569480,"statfs":{"total":95995035648,"available":93767147520,"internally_reserved":0,"allocated":1616756736,"data_stored":1615371036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":611111696},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":6.9870000000000001,"5min":5.4580000000000002,"15min":4.1790000000000003},"min":{"1min":0.22900000000000001,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":64.022000000000006,"5min":81.218999999999994,"15min":87.132999999999996},"last":0.22700000000000001},{"interface":"front","average":{"1min":7.8109999999999999,"5min":5.6849999999999996,"15min":3.863},"min":{"1min":0.26300000000000001,"5min":0.23799999999999999,"15min":0.23100000000000001},"max":{"1min":64.091999999999999,"5min":81.352000000000004,"15min":84.820999999999998},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:23:04 2023","interfaces":[{"interface":"back","average":{"1min":3.157,"5min":3.8399999999999999,"15min":4.5869999999999997},"min":{"1min":0.22800000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":18.094000000000001,"5min":44.508000000000003,"15min":84.641000000000005},"last":0.214},{"interface":"front","average":{"1min":3.4870000000000001,"5min":3.4319999999999999,"15min":3.3090000000000002},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":18.628,"5min":45.765999999999998,"15min":84.751999999999995},"last":0.317}]},{"osd":3,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":8.5559999999999992,"5min":6.1820000000000004,"15min":5.298},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":77.031999999999996,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.49199999999999999},{"interface":"front","average":{"1min":8.2710000000000008,"5min":6.5289999999999999,"15min":5.3239999999999998},"min":{"1min":0.313,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":76.742000000000004,"5min":76.742000000000004,"15min":86.158000000000001},"last":1.016}]},{"osd":4,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.6779999999999999,"5min":1.716,"15min":2.343},"min":{"1min":0.29999999999999999,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":5.798,"5min":11.452999999999999,"15min":84.608000000000004},"last":3.1920000000000002},{"interface":"front","average":{"1min":1.595,"5min":1.7410000000000001,"15min":2.968},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":5.8380000000000001,"5min":13.106,"15min":87.019999999999996},"last":1.0680000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":1.542,"5min":1.9019999999999999,"15min":2.6819999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.2400000000000002,"5min":8.9909999999999997,"15min":84.498000000000005},"last":3.2240000000000002},{"interface":"front","average":{"1min":2.3929999999999998,"5min":1.9810000000000001,"15min":2.5129999999999999},"min":{"1min":0.34899999999999998,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":9.8780000000000001,"5min":15.891,"15min":84.664000000000001},"last":0.41999999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.2839999999999998,"15min":2.2839999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4329999999999998,"5min":7.4329999999999998,"15min":7.4329999999999998},"last":0.97199999999999998},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:23:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.0470000000000002,"15min":2.5649999999999999},"min":{"1min":0.376,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":10.227,"5min":10.227,"15min":84.596000000000004},"last":1.052},{"interface":"front","average":{"1min":2.1539999999999999,"5min":2.0840000000000001,"15min":2.4249999999999998},"min":{"1min":0.504,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":7.0099999999999998,"5min":15.933999999999999,"15min":84.783000000000001},"last":0.60999999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402776,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1855052,"kb_used_data":1254660,"kb_used_omap":3,"kb_used_meta":600380,"kb_avail":91890100,"statfs":{"total":95995035648,"available":94095462400,"internally_reserved":0,"allocated":1284771840,"data_stored":1283375336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":614789289},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":8.5269999999999992,"5min":3.7440000000000002,"15min":3.6949999999999998},"min":{"1min":0.20599999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":108.318,"5min":108.318,"15min":108.318},"last":0.34999999999999998},{"interface":"front","average":{"1min":9.3879999999999999,"5min":4.1669999999999998,"15min":3.9740000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":108.889,"5min":108.889,"15min":108.889},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:23:02 2023","interfaces":[{"interface":"back","average":{"1min":9.2219999999999995,"5min":5.0830000000000002,"15min":4.9660000000000002},"min":{"1min":0.23799999999999999,"5min":0.214,"15min":0.214},"max":{"1min":126.875,"5min":126.875,"15min":126.875},"last":1.6359999999999999},{"interface":"front","average":{"1min":11.324999999999999,"5min":5.8259999999999996,"15min":5.2320000000000002},"min":{"1min":0.33600000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":128.249,"5min":128.249,"15min":128.249},"last":2.3220000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.2349999999999999,"15min":4.7539999999999996},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":12.210000000000001,"5min":17.199999999999999,"15min":109.077},"last":0.28100000000000003},{"interface":"front","average":{"1min":3.5670000000000002,"5min":3.3410000000000002,"15min":5.7969999999999997},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.763,"5min":36.582000000000001,"15min":119.72799999999999},"last":0.625}]},{"osd":4,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":2.1429999999999998,"15min":2.2669999999999999},"min":{"1min":0.35299999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.417999999999999,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.67700000000000005},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.5660000000000001,"15min":2.121},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":8.0050000000000008,"5min":12.500999999999999,"15min":52.228999999999999},"last":1.7330000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":3.7330000000000001,"15min":2.9249999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":11.372999999999999,"5min":148.619,"15min":148.619},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.909,"5min":2.9780000000000002,"15min":2.7789999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":6.8520000000000003,"5min":148.56999999999999,"15min":148.56999999999999},"last":1.694}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.2200000000000006,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":1.7649999999999999},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":36.517000000000003,"5min":36.517000000000003,"15min":36.517000000000003},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.9279999999999999,"15min":2.681},"min":{"1min":0.41799999999999998,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":4.6699999999999999,"5min":145.25,"15min":145.25},"last":1.796},{"interface":"front","average":{"1min":1.877,"5min":3.121,"15min":2.4990000000000001},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":7.1829999999999998,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.48699999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986541,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190384,"kb_used_data":863080,"kb_used_omap":0,"kb_used_meta":327232,"kb_avail":92554768,"statfs":{"total":95995035648,"available":94776082432,"internally_reserved":0,"allocated":883793920,"data_stored":882812023,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":335085568},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":5.7130000000000001,"15min":4.0220000000000002},"min":{"1min":0.23599999999999999,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":5.3170000000000002,"5min":162.75999999999999,"15min":162.75999999999999},"last":4.6299999999999999},{"interface":"front","average":{"1min":2.5699999999999998,"5min":5.6079999999999997,"15min":4.1219999999999999},"min":{"1min":0.25900000000000001,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":8.7289999999999992,"5min":162.34800000000001,"15min":162.34800000000001},"last":1.3080000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":5.1509999999999998,"5min":4.1280000000000001,"15min":3.4590000000000001},"min":{"1min":0.254,"5min":0.249,"15min":0.245},"max":{"1min":60.122,"5min":113.11199999999999,"15min":113.11199999999999},"last":2.0419999999999998},{"interface":"front","average":{"1min":5.5640000000000001,"5min":4.1980000000000004,"15min":3.8839999999999999},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":60.802,"5min":113.02,"15min":113.02},"last":0.80000000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:23:14 2023","interfaces":[{"interface":"back","average":{"1min":2.4369999999999998,"5min":3.734,"15min":4.5199999999999996},"min":{"1min":0.33600000000000002,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":7.016,"5min":61.646000000000001,"15min":148.08000000000001},"last":1.3460000000000001},{"interface":"front","average":{"1min":3.488,"5min":3.8290000000000002,"15min":4.758},"min":{"1min":0.35199999999999998,"5min":0.28999999999999998,"15min":0.25700000000000001},"max":{"1min":6.9980000000000002,"5min":64.840000000000003,"15min":147.88900000000001},"last":2.0099999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":6.3780000000000001,"15min":5.9969999999999999},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":6.5060000000000002,"5min":156.46000000000001,"15min":163.97499999999999},"last":1.173},{"interface":"front","average":{"1min":2.1360000000000001,"5min":6.5069999999999997,"15min":5.9400000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27800000000000002},"max":{"1min":8.3829999999999991,"5min":156.74299999999999,"15min":164.447},"last":0.35999999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.069,"15min":1.6279999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":6.4089999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.458,"5min":1.2789999999999999,"15min":1.573},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":5.7199999999999998,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:23:28 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.446,"15min":1.446},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6470000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.1369999999999996,"5min":6.1369999999999996,"15min":6.1369999999999996},"last":2.7429999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.92500000000000004,"15min":1.1890000000000001},"min":{"1min":0.42599999999999999,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.7599999999999998,"5min":6.0730000000000004,"15min":18.23},"last":1.375},{"interface":"front","average":{"1min":1.819,"5min":1.034,"15min":1.286},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":4.2229999999999999,"5min":5.3520000000000003,"15min":14.653},"last":1.149}]}]},{"osd":5,"up_from":39,"seq":167503724904,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1850344,"kb_used_data":1475744,"kb_used_omap":0,"kb_used_meta":374592,"kb_avail":91894808,"statfs":{"total":95995035648,"available":94100283392,"internally_reserved":0,"allocated":1511161856,"data_stored":1510184412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":383582208},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.7070000000000001,"5min":4.8639999999999999,"15min":3.2309999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":4.1059999999999999,"5min":124.31,"15min":124.31},"last":1.1779999999999999},{"interface":"front","average":{"1min":2.6349999999999998,"5min":4.8070000000000004,"15min":3.2799999999999998},"min":{"1min":0.25800000000000001,"5min":0.23300000000000001,"15min":0.222},"max":{"1min":9.3889999999999993,"5min":124.374,"15min":124.374},"last":0.72999999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":2.8690000000000002,"5min":3.8199999999999998,"15min":2.5880000000000001},"min":{"1min":0.36199999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.3149999999999995,"5min":124.38800000000001,"15min":124.38800000000001},"last":1.212},{"interface":"front","average":{"1min":2.165,"5min":4.0679999999999996,"15min":2.8919999999999999},"min":{"1min":0.29199999999999998,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":5.8769999999999998,"5min":124.29000000000001,"15min":124.29000000000001},"last":0.77000000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:23:16 2023","interfaces":[{"interface":"back","average":{"1min":1.716,"5min":3.77,"15min":3.7210000000000001},"min":{"1min":0.32400000000000001,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":8.3030000000000008,"5min":193.00999999999999,"15min":193.00999999999999},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.4550000000000001,"5min":4.0960000000000001,"15min":3.8340000000000001},"min":{"1min":0.36299999999999999,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":9.3420000000000005,"5min":193.15700000000001,"15min":193.15700000000001},"last":0.79500000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":16.594999999999999,"5min":7.7089999999999996,"15min":6.0309999999999997},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":266.70299999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":1.1659999999999999},{"interface":"front","average":{"1min":20.937000000000001,"5min":8.5090000000000003,"15min":6.0449999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":266.80399999999997,"5min":266.80399999999997,"15min":266.80399999999997},"last":1.24}]},{"osd":4,"last update":"Sat Sep 23 15:23:05 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":0.73399999999999999,"15min":1.8380000000000001},"min":{"1min":0.27800000000000002,"5min":0.26500000000000001,"15min":0.26200000000000001},"max":{"1min":7.3440000000000003,"5min":7.3440000000000003,"15min":138.68600000000001},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.641,"5min":0.754,"15min":1.9019999999999999},"min":{"1min":0.29599999999999999,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":138.49600000000001},"last":0.81699999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.3969999999999998,"15min":2.3969999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.6050000000000004,"5min":9.6050000000000004,"15min":9.6050000000000004},"last":1.0209999999999999},{"interface":"front","average":{"1min":2.4100000000000001,"5min":2.4100000000000001,"15min":2.4100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":0.999}]},{"osd":7,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.131,"15min":1.407},"min":{"1min":0.40200000000000002,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":12.065,"5min":12.065,"15min":15.301},"last":15.849},{"interface":"front","average":{"1min":2.9809999999999999,"5min":1.0920000000000001,"15min":1.351},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":15.762},"last":16.134}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1433862144,"data_stored":1433862144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1484062720,"data_stored":1484062720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1600196608,"data_stored":1600192536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1279393792,"data_stored":1279393792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":878444544,"data_stored":878444544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1493962752,"data_stored":1493958680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326172672,"data_stored":326172672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1135022080,"data_stored":1135018008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:24:11.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:11.076 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:11.078 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:11.140 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:11.172 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:11.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:11.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:11.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:11.339 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:11.552 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:11.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:11.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:11.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:11.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:11.848 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:11.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:11.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:11.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:12.008 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:12.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:24:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[222878]: pgmap v948: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 25302/129852 objects degraded (19.485%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 25302/129852 objects degraded (19.485%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:12.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:12.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[215646]: pgmap v948: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 25302/129852 objects degraded (19.485%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:12.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 25302/129852 objects degraded (19.485%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:12.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:12.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:12.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:12 smithi027 ceph-mon[195233]: pgmap v948: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 600 op/s; 25302/129852 objects degraded (19.485%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:12.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 25302/129852 objects degraded (19.485%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:12.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:12.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:13.347 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:13.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:13.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:13.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:13.514 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:13.538 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:13.611 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:13.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:13.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:13.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:13.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:13.946 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:24:14.127 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:24:14.127 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:24:14.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:14.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:14.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:14.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:14.361 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:14.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:14 smithi017 ceph-mon[215646]: pgmap v949: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 25302/127524 objects degraded (19.841%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:14.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:14 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:14.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:14 smithi017 ceph-mon[222878]: pgmap v949: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 25302/127524 objects degraded (19.841%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:14.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:14 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:14.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:14 smithi027 ceph-mon[195233]: pgmap v949: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 25302/127524 objects degraded (19.841%); 4.7 MiB/s, 97 objects/s recovering 2023-09-23T15:24:14.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:14 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:15.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:24:15.619 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:15.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:15.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:15.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:15.808 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:15 smithi017 ceph-mon[215646]: pgmap v950: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 24908/124512 objects degraded (20.004%); 4.6 MiB/s, 96 objects/s recovering 2023-09-23T15:24:15.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:15 smithi017 ceph-mon[222878]: pgmap v950: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 24908/124512 objects degraded (20.004%); 4.6 MiB/s, 96 objects/s recovering 2023-09-23T15:24:15.824 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:15.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:15 smithi027 ceph-mon[195233]: pgmap v950: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 603 op/s; 24908/124512 objects degraded (20.004%); 4.6 MiB/s, 96 objects/s recovering 2023-09-23T15:24:16.157 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:16.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:16.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:16.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:16.319 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:16.407 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:16.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:16.412 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:16.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:16.521 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:16.558 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:16.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:16 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:16.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1251786200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.568 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.590 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:24:16.625 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:16.646 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:16.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:16.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:16.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:16.835 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:16.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:16.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1251786200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:16.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1251786200' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.904 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:16.909 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:16.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:17.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:17.074 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:17.152 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:17.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:17.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:17.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:17.285 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:17.332 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:17.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:17.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:17.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:17.493 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:17.573 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:24:17.573 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: pgmap v952: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24563/120063 objects degraded (20.458%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:17.574 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24908/124512 objects degraded (20.004%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:17.805 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:17.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:24:17.808 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: pgmap v952: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24563/120063 objects degraded (20.458%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:17.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24908/124512 objects degraded (20.004%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:17.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: failed to encode map e326 with expected crc 2023-09-23T15:24:17.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: pgmap v952: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 621 op/s; 24563/120063 objects degraded (20.458%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:17.811 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24908/124512 objects degraded (20.004%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:17.811 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:17.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:17.958 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:18.034 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:18.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:18.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:18.085 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:18] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T15:24:18.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:18.229 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:18.277 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:24:18.326 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:24:18.369 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:24:18.410 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:24:18.465 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:18.483 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:18.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:18.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:18.494 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:18.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:18.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:18.594 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.606 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:24:18.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2341790291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:18.643 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2341790291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:18.667 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:18.717 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:18.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:18.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:18.822 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:18.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2341790291' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:18.866 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:19.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:19 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:19.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:19 smithi017 ceph-mon[215646]: pgmap v953: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24563/117408 objects degraded (20.921%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:19.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:19 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:19.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:19 smithi017 ceph-mon[222878]: pgmap v953: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24563/117408 objects degraded (20.921%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:19.778 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:19.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:19.784 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:19.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:19 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:19.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:19 smithi027 ceph-mon[195233]: pgmap v953: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s; 24563/117408 objects degraded (20.921%); 2.9 MiB/s, 103 objects/s recovering 2023-09-23T15:24:19.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:19.949 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:19.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:24:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:20 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:20 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:20 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:20.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:20 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:21.158 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:21.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:21.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:21.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:21.377 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:21.558 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:21.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:21.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:21.624 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:21 smithi017 ceph-mon[215646]: pgmap v954: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 24075/109512 objects degraded (21.984%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:21.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:21.719 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:21.724 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:24:21.725 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:24:21.725 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:24:21.725 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:24:21.725 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:24:21.825 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:21.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:21.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:21.878 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:21.940 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:21.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:21 smithi017 ceph-mon[222878]: pgmap v954: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 24075/109512 objects degraded (21.984%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:22.012 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:22.103 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:21 smithi027 ceph-mon[195233]: pgmap v954: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 24075/109512 objects degraded (21.984%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:22.414 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:22.431 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 24075/109512 objects degraded (21.984%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:22.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[215646]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 24075/109512 objects degraded (21.984%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:22.805 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:22 smithi017 ceph-mon[222878]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:22.805 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:22.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:22.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:22.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:24:22.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:24:22.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:24:22.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:24:22.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 24075/109512 objects degraded (21.984%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:22.962 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:22 smithi027 ceph-mon[195233]: from='client.46025 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:22.964 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:23.011 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:23.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:23.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:23.164 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:24:23.183 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:23.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:24:23.602 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:23.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:23.613 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:23.701 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":954,"stamp":"2023-09-23T15:24:20.739175+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2395269489,"num_objects":36504,"num_object_clones":0,"num_object_copies":109512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24075,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36504,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":144723,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11915,"num_bytes_recovered":713592181,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110507,"ondisk_log_size":110507,"up":327,"acting":320,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11066268,"kb_used_data":7925212,"kb_used_omap":44,"kb_used_meta":3140563,"kb_avail":738894948,"statfs":{"total":767960285184,"available":756628426752,"internally_reserved":0,"allocated":8115417088,"data_stored":8105711317,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3215937467},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,4,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-444334080,"num_objects":-6780,"num_object_clones":0,"num_object_copies":-20340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1227,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6780,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1019,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005829"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430292+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:24:05.430292+0000","last_peered":"2023-09-23T15:24:05.430292+0000","last_clean":"2023-09-23T15:24:05.430292+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:24:05.430292+0000","last_undegraded":"2023-09-23T15:24:05.430292+0000","last_fullsized":"2023-09-23T15:24:05.430292+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":282,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430065+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:24:05.430065+0000","last_peered":"2023-09-23T15:24:05.430065+0000","last_clean":"2023-09-23T15:24:05.430065+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:24:05.430065+0000","last_undegraded":"2023-09-23T15:24:05.430065+0000","last_fullsized":"2023-09-23T15:24:05.430065+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":230,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.420792+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:24:05.420792+0000","last_peered":"2023-09-23T15:24:05.420792+0000","last_clean":"2023-09-23T15:24:05.420792+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:24:05.420792+0000","last_undegraded":"2023-09-23T15:24:05.420792+0000","last_fullsized":"2023-09-23T15:24:05.420792+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":304,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421793+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:24:05.421793+0000","last_peered":"2023-09-23T15:24:05.421793+0000","last_clean":"2023-09-23T15:24:05.421793+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:24:05.421793+0000","last_undegraded":"2023-09-23T15:24:05.421793+0000","last_fullsized":"2023-09-23T15:24:05.421793+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":327,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430329+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:24:05.430329+0000","last_peered":"2023-09-23T15:24:05.430329+0000","last_clean":"2023-09-23T15:24:05.430329+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:24:05.430329+0000","last_undegraded":"2023-09-23T15:24:05.430329+0000","last_fullsized":"2023-09-23T15:24:05.430329+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":276,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434370+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:24:05.434370+0000","last_peered":"2023-09-23T15:24:05.434370+0000","last_clean":"2023-09-23T15:24:05.434370+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:24:05.434370+0000","last_undegraded":"2023-09-23T15:24:05.434370+0000","last_fullsized":"2023-09-23T15:24:05.434370+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":274,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.414680+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:24:05.414680+0000","last_peered":"2023-09-23T15:24:05.414680+0000","last_clean":"2023-09-23T15:24:05.414680+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:24:05.414680+0000","last_undegraded":"2023-09-23T15:24:05.414680+0000","last_fullsized":"2023-09-23T15:24:05.414680+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":324,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430413+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:24:05.430413+0000","last_peered":"2023-09-23T15:24:05.430413+0000","last_clean":"2023-09-23T15:24:05.430413+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:24:05.430413+0000","last_undegraded":"2023-09-23T15:24:05.430413+0000","last_fullsized":"2023-09-23T15:24:05.430413+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":684,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.411334+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:24:05.411334+0000","last_peered":"2023-09-23T15:24:05.411334+0000","last_clean":"2023-09-23T15:24:05.411334+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:24:05.411334+0000","last_undegraded":"2023-09-23T15:24:05.411334+0000","last_fullsized":"2023-09-23T15:24:05.411334+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":326,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427291+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:24:05.427291+0000","last_peered":"2023-09-23T15:24:05.427291+0000","last_clean":"2023-09-23T15:24:05.427291+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:24:05.427291+0000","last_undegraded":"2023-09-23T15:24:05.427291+0000","last_fullsized":"2023-09-23T15:24:05.427291+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":278,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429712+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:24:05.429712+0000","last_peered":"2023-09-23T15:24:05.429712+0000","last_clean":"2023-09-23T15:24:05.429712+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:24:05.429712+0000","last_undegraded":"2023-09-23T15:24:05.429712+0000","last_fullsized":"2023-09-23T15:24:05.429712+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":70,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432798+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:24:05.432798+0000","last_peered":"2023-09-23T15:24:05.432798+0000","last_clean":"2023-09-23T15:24:05.432798+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:24:05.432798+0000","last_undegraded":"2023-09-23T15:24:05.432798+0000","last_fullsized":"2023-09-23T15:24:05.432798+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":216,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421892+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:24:05.421892+0000","last_peered":"2023-09-23T15:24:05.421892+0000","last_clean":"2023-09-23T15:24:05.421892+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:24:05.421892+0000","last_undegraded":"2023-09-23T15:24:05.421892+0000","last_fullsized":"2023-09-23T15:24:05.421892+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":60,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430945+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:24:05.430945+0000","last_peered":"2023-09-23T15:24:05.430945+0000","last_clean":"2023-09-23T15:24:05.430945+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:24:05.430945+0000","last_undegraded":"2023-09-23T15:24:05.430945+0000","last_fullsized":"2023-09-23T15:24:05.430945+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432547+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:24:05.432547+0000","last_peered":"2023-09-23T15:24:05.432547+0000","last_clean":"2023-09-23T15:24:05.432547+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:24:05.432547+0000","last_undegraded":"2023-09-23T15:24:05.432547+0000","last_fullsized":"2023-09-23T15:24:05.432547+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":635,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429923+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:24:05.429923+0000","last_peered":"2023-09-23T15:24:05.429923+0000","last_clean":"2023-09-23T15:24:05.429923+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:24:05.429923+0000","last_undegraded":"2023-09-23T15:24:05.429923+0000","last_fullsized":"2023-09-23T15:24:05.429923+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432672+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:24:05.432672+0000","last_peered":"2023-09-23T15:24:05.432672+0000","last_clean":"2023-09-23T15:24:05.432672+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:24:05.432672+0000","last_undegraded":"2023-09-23T15:24:05.432672+0000","last_fullsized":"2023-09-23T15:24:05.432672+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":329,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429267+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:24:05.429267+0000","last_peered":"2023-09-23T15:24:05.429267+0000","last_clean":"2023-09-23T15:24:05.429267+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:24:05.429267+0000","last_undegraded":"2023-09-23T15:24:05.429267+0000","last_fullsized":"2023-09-23T15:24:05.429267+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":684,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427345+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:24:05.427345+0000","last_peered":"2023-09-23T15:24:05.427345+0000","last_clean":"2023-09-23T15:24:05.427345+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:24:05.427345+0000","last_undegraded":"2023-09-23T15:24:05.427345+0000","last_fullsized":"2023-09-23T15:24:05.427345+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":303,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.413901+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:24:05.413901+0000","last_peered":"2023-09-23T15:24:05.413901+0000","last_clean":"2023-09-23T15:24:05.413901+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:24:05.413901+0000","last_undegraded":"2023-09-23T15:24:05.413901+0000","last_fullsized":"2023-09-23T15:24:05.413901+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":311,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427920+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:24:05.427920+0000","last_peered":"2023-09-23T15:24:05.427920+0000","last_clean":"2023-09-23T15:24:05.427920+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:24:05.427920+0000","last_undegraded":"2023-09-23T15:24:05.427920+0000","last_fullsized":"2023-09-23T15:24:05.427920+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":616,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421423+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:24:05.421423+0000","last_peered":"2023-09-23T15:24:05.421423+0000","last_clean":"2023-09-23T15:24:05.421423+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:24:05.421423+0000","last_undegraded":"2023-09-23T15:24:05.421423+0000","last_fullsized":"2023-09-23T15:24:05.421423+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":74,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.431948+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:24:05.431948+0000","last_peered":"2023-09-23T15:24:05.431948+0000","last_clean":"2023-09-23T15:24:05.431948+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:24:05.431948+0000","last_undegraded":"2023-09-23T15:24:05.431948+0000","last_fullsized":"2023-09-23T15:24:05.431948+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":292,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421874+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:24:05.421874+0000","last_peered":"2023-09-23T15:24:05.421874+0000","last_clean":"2023-09-23T15:24:05.421874+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:24:05.421874+0000","last_undegraded":"2023-09-23T15:24:05.421874+0000","last_fullsized":"2023-09-23T15:24:05.421874+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":642,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433045+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:24:05.433045+0000","last_peered":"2023-09-23T15:24:05.433045+0000","last_clean":"2023-09-23T15:24:05.433045+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:24:05.433045+0000","last_undegraded":"2023-09-23T15:24:05.433045+0000","last_fullsized":"2023-09-23T15:24:05.433045+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":262,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.428906+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:24:05.428906+0000","last_peered":"2023-09-23T15:24:05.428906+0000","last_clean":"2023-09-23T15:24:05.428906+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:24:05.428906+0000","last_undegraded":"2023-09-23T15:24:05.428906+0000","last_fullsized":"2023-09-23T15:24:05.428906+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":329,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430457+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:24:05.430457+0000","last_peered":"2023-09-23T15:24:05.430457+0000","last_clean":"2023-09-23T15:24:05.430457+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:24:05.430457+0000","last_undegraded":"2023-09-23T15:24:05.430457+0000","last_fullsized":"2023-09-23T15:24:05.430457+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"326'9180","reported_seq":13978,"reported_epoch":326,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:19.907960+0000","last_change":"2023-09-23T15:23:39.629532+0000","last_active":"2023-09-23T15:24:19.907960+0000","last_peered":"2023-09-23T15:24:19.907960+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:21:58.288439+0000","last_became_peered":"2023-09-23T15:21:58.288439+0000","last_unstale":"2023-09-23T15:24:19.907960+0000","last_undegraded":"2023-09-23T15:21:58.149099+0000","last_fullsized":"2023-09-23T15:21:58.128266+0000","mapping_epoch":297,"log_start":"293'2300","ondisk_log_start":"293'2300","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6880,"log_dups_size":0,"ondisk_log_size":6880,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:22:01.838822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":871,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9180,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2371,"num_bytes_recovered":87425024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1363}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":588,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433298+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:24:05.433298+0000","last_peered":"2023-09-23T15:24:05.433298+0000","last_clean":"2023-09-23T15:24:05.433298+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:24:05.433298+0000","last_undegraded":"2023-09-23T15:24:05.433298+0000","last_fullsized":"2023-09-23T15:24:05.433298+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":66,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434153+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:24:05.434153+0000","last_peered":"2023-09-23T15:24:05.434153+0000","last_clean":"2023-09-23T15:24:05.434153+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:24:05.434153+0000","last_undegraded":"2023-09-23T15:24:05.434153+0000","last_fullsized":"2023-09-23T15:24:05.434153+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":318,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427092+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:24:05.427092+0000","last_peered":"2023-09-23T15:24:05.427092+0000","last_clean":"2023-09-23T15:24:05.427092+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:24:05.427092+0000","last_undegraded":"2023-09-23T15:24:05.427092+0000","last_fullsized":"2023-09-23T15:24:05.427092+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"326'8799","reported_seq":18059,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:19.421485+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:24:19.421485+0000","last_peered":"2023-09-23T15:24:19.421485+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:24:19.421485+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"293'1900","ondisk_log_start":"293'1900","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149094400,"num_objects":2275,"num_object_clones":0,"num_object_copies":6825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1484,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8799,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1","4"],"object_location_counts":[{"shards":"0,4,7","objects":1285},{"shards":"4,7","objects":791}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":646,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429217+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:24:05.429217+0000","last_peered":"2023-09-23T15:24:05.429217+0000","last_clean":"2023-09-23T15:24:05.429217+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:24:05.429217+0000","last_undegraded":"2023-09-23T15:24:05.429217+0000","last_fullsized":"2023-09-23T15:24:05.429217+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":66,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433564+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:24:05.433564+0000","last_peered":"2023-09-23T15:24:05.433564+0000","last_clean":"2023-09-23T15:24:05.433564+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:24:05.433564+0000","last_undegraded":"2023-09-23T15:24:05.433564+0000","last_fullsized":"2023-09-23T15:24:05.433564+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430060+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:24:05.430060+0000","last_peered":"2023-09-23T15:24:05.430060+0000","last_clean":"2023-09-23T15:24:05.430060+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:24:05.430060+0000","last_undegraded":"2023-09-23T15:24:05.430060+0000","last_fullsized":"2023-09-23T15:24:05.430060+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"326'8980","reported_seq":9051,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:19.107242+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:24:19.107242+0000","last_peered":"2023-09-23T15:24:19.107242+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:24:19.107242+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"294'2400","ondisk_log_start":"294'2400","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6580,"log_dups_size":0,"ondisk_log_size":6580,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146407424,"num_objects":2234,"num_object_clones":0,"num_object_copies":6702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8980,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":1256},{"shards":"0,7","objects":-2396}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":533,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434550+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:24:05.434550+0000","last_peered":"2023-09-23T15:24:05.434550+0000","last_clean":"2023-09-23T15:24:05.434550+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:24:05.434550+0000","last_undegraded":"2023-09-23T15:24:05.434550+0000","last_fullsized":"2023-09-23T15:24:05.434550+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":66,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.431986+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:24:05.431986+0000","last_peered":"2023-09-23T15:24:05.431986+0000","last_clean":"2023-09-23T15:24:05.431986+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:24:05.431986+0000","last_undegraded":"2023-09-23T15:24:05.431986+0000","last_fullsized":"2023-09-23T15:24:05.431986+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":289,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433905+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:24:05.433905+0000","last_peered":"2023-09-23T15:24:05.433905+0000","last_clean":"2023-09-23T15:24:05.433905+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:24:05.433905+0000","last_undegraded":"2023-09-23T15:24:05.433905+0000","last_fullsized":"2023-09-23T15:24:05.433905+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"326'8965","reported_seq":9022,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:16.922607+0000","last_change":"2023-09-23T15:22:27.588207+0000","last_active":"2023-09-23T15:24:16.922607+0000","last_peered":"2023-09-23T15:24:16.922607+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:24:16.922607+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"294'2400","ondisk_log_start":"294'2400","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6565,"log_dups_size":0,"ondisk_log_size":6565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153812992,"num_objects":2347,"num_object_clones":0,"num_object_copies":7041,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4656,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2347,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8965,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-2309}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":541,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434281+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:24:05.434281+0000","last_peered":"2023-09-23T15:24:05.434281+0000","last_clean":"2023-09-23T15:24:05.434281+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:24:05.434281+0000","last_undegraded":"2023-09-23T15:24:05.434281+0000","last_fullsized":"2023-09-23T15:24:05.434281+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.428741+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:24:05.428741+0000","last_peered":"2023-09-23T15:24:05.428741+0000","last_clean":"2023-09-23T15:24:05.428741+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:24:05.428741+0000","last_undegraded":"2023-09-23T15:24:05.428741+0000","last_fullsized":"2023-09-23T15:24:05.428741+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":307,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.414093+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:24:05.414093+0000","last_peered":"2023-09-23T15:24:05.414093+0000","last_clean":"2023-09-23T15:24:05.414093+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:24:05.414093+0000","last_undegraded":"2023-09-23T15:24:05.414093+0000","last_fullsized":"2023-09-23T15:24:05.414093+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"326'9019","reported_seq":9071,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:20.512009+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:24:20.512009+0000","last_peered":"2023-09-23T15:24:20.512009+0000","last_clean":"2023-09-23T15:24:20.512009+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:24:20.512009+0000","last_undegraded":"2023-09-23T15:24:20.512009+0000","last_fullsized":"2023-09-23T15:24:20.512009+0000","mapping_epoch":284,"log_start":"295'2600","ondisk_log_start":"295'2600","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6419,"log_dups_size":0,"ondisk_log_size":6419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147783680,"num_objects":2255,"num_object_clones":0,"num_object_copies":6765,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2255,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9019,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":562,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432168+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:24:05.432168+0000","last_peered":"2023-09-23T15:24:05.432168+0000","last_clean":"2023-09-23T15:24:05.432168+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:24:05.432168+0000","last_undegraded":"2023-09-23T15:24:05.432168+0000","last_fullsized":"2023-09-23T15:24:05.432168+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":319,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:15.627284+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:24:15.627284+0000","last_peered":"2023-09-23T15:24:15.627284+0000","last_clean":"2023-09-23T15:24:15.627284+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:24:15.627284+0000","last_undegraded":"2023-09-23T15:24:15.627284+0000","last_fullsized":"2023-09-23T15:24:15.627284+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430825+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:24:05.430825+0000","last_peered":"2023-09-23T15:24:05.430825+0000","last_clean":"2023-09-23T15:24:05.430825+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:24:05.430825+0000","last_undegraded":"2023-09-23T15:24:05.430825+0000","last_fullsized":"2023-09-23T15:24:05.430825+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"326'9149","reported_seq":9198,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:16.973655+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:24:16.973655+0000","last_peered":"2023-09-23T15:24:16.973655+0000","last_clean":"2023-09-23T15:24:16.973655+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:24:16.973655+0000","last_undegraded":"2023-09-23T15:24:16.973655+0000","last_fullsized":"2023-09-23T15:24:16.973655+0000","mapping_epoch":284,"log_start":"295'2600","ondisk_log_start":"295'2600","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6549,"log_dups_size":0,"ondisk_log_size":6549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156827648,"num_objects":2393,"num_object_clones":0,"num_object_copies":7179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9149,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":630,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427425+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:24:05.427425+0000","last_peered":"2023-09-23T15:24:05.427425+0000","last_clean":"2023-09-23T15:24:05.427425+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:24:05.427425+0000","last_undegraded":"2023-09-23T15:24:05.427425+0000","last_fullsized":"2023-09-23T15:24:05.427425+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":273,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433960+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:24:05.433960+0000","last_peered":"2023-09-23T15:24:05.433960+0000","last_clean":"2023-09-23T15:24:05.433960+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:24:05.433960+0000","last_undegraded":"2023-09-23T15:24:05.433960+0000","last_fullsized":"2023-09-23T15:24:05.433960+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430790+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:24:05.430790+0000","last_peered":"2023-09-23T15:24:05.430790+0000","last_clean":"2023-09-23T15:24:05.430790+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:24:05.430790+0000","last_undegraded":"2023-09-23T15:24:05.430790+0000","last_fullsized":"2023-09-23T15:24:05.430790+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"326'9176","reported_seq":15725,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:19.700373+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:24:19.700373+0000","last_peered":"2023-09-23T15:24:19.700373+0000","last_clean":"2023-09-23T15:24:19.700373+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:24:19.700373+0000","last_undegraded":"2023-09-23T15:24:19.700373+0000","last_fullsized":"2023-09-23T15:24:19.700373+0000","mapping_epoch":310,"log_start":"294'2600","ondisk_log_start":"294'2600","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6576,"log_dups_size":0,"ondisk_log_size":6576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9176,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":582,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421353+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:24:05.421353+0000","last_peered":"2023-09-23T15:24:05.421353+0000","last_clean":"2023-09-23T15:24:05.421353+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:24:05.421353+0000","last_undegraded":"2023-09-23T15:24:05.421353+0000","last_fullsized":"2023-09-23T15:24:05.421353+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430129+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:24:05.430129+0000","last_peered":"2023-09-23T15:24:05.430129+0000","last_clean":"2023-09-23T15:24:05.430129+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:24:05.430129+0000","last_undegraded":"2023-09-23T15:24:05.430129+0000","last_fullsized":"2023-09-23T15:24:05.430129+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434204+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:24:05.434204+0000","last_peered":"2023-09-23T15:24:05.434204+0000","last_clean":"2023-09-23T15:24:05.434204+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:24:05.434204+0000","last_undegraded":"2023-09-23T15:24:05.434204+0000","last_fullsized":"2023-09-23T15:24:05.434204+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"326'8916","reported_seq":8973,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:16.878376+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:24:16.878376+0000","last_peered":"2023-09-23T15:24:16.878376+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:24:16.878376+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"294'2300","ondisk_log_start":"294'2300","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6616,"log_dups_size":0,"ondisk_log_size":6616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156106752,"num_objects":2382,"num_object_clones":0,"num_object_copies":7146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8916,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-2236}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":642,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432894+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:24:05.432894+0000","last_peered":"2023-09-23T15:24:05.432894+0000","last_clean":"2023-09-23T15:24:05.432894+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:24:05.432894+0000","last_undegraded":"2023-09-23T15:24:05.432894+0000","last_fullsized":"2023-09-23T15:24:05.432894+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":340,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.428986+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:24:05.428986+0000","last_peered":"2023-09-23T15:24:05.428986+0000","last_clean":"2023-09-23T15:24:05.428986+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:24:05.428986+0000","last_undegraded":"2023-09-23T15:24:05.428986+0000","last_fullsized":"2023-09-23T15:24:05.428986+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":234,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427049+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:24:05.427049+0000","last_peered":"2023-09-23T15:24:05.427049+0000","last_clean":"2023-09-23T15:24:05.427049+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:24:05.427049+0000","last_undegraded":"2023-09-23T15:24:05.427049+0000","last_fullsized":"2023-09-23T15:24:05.427049+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"326'8820","reported_seq":8889,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:19.134760+0000","last_change":"2023-09-23T15:21:58.303371+0000","last_active":"2023-09-23T15:24:19.134760+0000","last_peered":"2023-09-23T15:24:19.134760+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:24:19.134760+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"293'2200","ondisk_log_start":"293'2200","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6620,"log_dups_size":0,"ondisk_log_size":6620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3093,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8820,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":-839}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":523,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434456+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:24:05.434456+0000","last_peered":"2023-09-23T15:24:05.434456+0000","last_clean":"2023-09-23T15:24:05.434456+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:24:05.434456+0000","last_undegraded":"2023-09-23T15:24:05.434456+0000","last_fullsized":"2023-09-23T15:24:05.434456+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":525,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421247+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:24:05.421247+0000","last_peered":"2023-09-23T15:24:05.421247+0000","last_clean":"2023-09-23T15:24:05.421247+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:24:05.421247+0000","last_undegraded":"2023-09-23T15:24:05.421247+0000","last_fullsized":"2023-09-23T15:24:05.421247+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427845+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:24:05.427845+0000","last_peered":"2023-09-23T15:24:05.427845+0000","last_clean":"2023-09-23T15:24:05.427845+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:24:05.427845+0000","last_undegraded":"2023-09-23T15:24:05.427845+0000","last_fullsized":"2023-09-23T15:24:05.427845+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":269,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.411444+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:24:05.411444+0000","last_peered":"2023-09-23T15:24:05.411444+0000","last_clean":"2023-09-23T15:24:05.411444+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:24:05.411444+0000","last_undegraded":"2023-09-23T15:24:05.411444+0000","last_fullsized":"2023-09-23T15:24:05.411444+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"326'9010","reported_seq":9070,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:19.904586+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:24:19.904586+0000","last_peered":"2023-09-23T15:24:19.904586+0000","last_clean":"2023-09-23T15:24:19.904586+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:24:19.904586+0000","last_undegraded":"2023-09-23T15:24:19.904586+0000","last_fullsized":"2023-09-23T15:24:19.904586+0000","mapping_epoch":287,"log_start":"293'2100","ondisk_log_start":"293'2100","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6910,"log_dups_size":0,"ondisk_log_size":6910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145096704,"num_objects":2214,"num_object_clones":0,"num_object_copies":6642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9010,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":623,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434743+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:24:05.434743+0000","last_peered":"2023-09-23T15:24:05.434743+0000","last_clean":"2023-09-23T15:24:05.434743+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:24:05.434743+0000","last_undegraded":"2023-09-23T15:24:05.434743+0000","last_fullsized":"2023-09-23T15:24:05.434743+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":861,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433279+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:24:05.433279+0000","last_peered":"2023-09-23T15:24:05.433279+0000","last_clean":"2023-09-23T15:24:05.433279+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:24:05.433279+0000","last_undegraded":"2023-09-23T15:24:05.433279+0000","last_fullsized":"2023-09-23T15:24:05.433279+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432471+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:24:05.432471+0000","last_peered":"2023-09-23T15:24:05.432471+0000","last_clean":"2023-09-23T15:24:05.432471+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:24:05.432471+0000","last_undegraded":"2023-09-23T15:24:05.432471+0000","last_fullsized":"2023-09-23T15:24:05.432471+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":222,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.420915+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:24:05.420915+0000","last_peered":"2023-09-23T15:24:05.420915+0000","last_clean":"2023-09-23T15:24:05.420915+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:24:05.420915+0000","last_undegraded":"2023-09-23T15:24:05.420915+0000","last_fullsized":"2023-09-23T15:24:05.420915+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"326'8911","reported_seq":15199,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:20.505679+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:24:20.505679+0000","last_peered":"2023-09-23T15:24:20.505679+0000","last_clean":"2023-09-23T15:24:20.505679+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:24:20.505679+0000","last_undegraded":"2023-09-23T15:24:20.505679+0000","last_fullsized":"2023-09-23T15:24:20.505679+0000","mapping_epoch":320,"log_start":"295'2500","ondisk_log_start":"295'2500","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6411,"log_dups_size":0,"ondisk_log_size":6411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144113664,"num_objects":2199,"num_object_clones":0,"num_object_copies":6597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8911,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":681,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432381+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:24:05.432381+0000","last_peered":"2023-09-23T15:24:05.432381+0000","last_clean":"2023-09-23T15:24:05.432381+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:24:05.432381+0000","last_undegraded":"2023-09-23T15:24:05.432381+0000","last_fullsized":"2023-09-23T15:24:05.432381+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":942,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.411180+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:24:05.411180+0000","last_peered":"2023-09-23T15:24:05.411180+0000","last_clean":"2023-09-23T15:24:05.411180+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:24:05.411180+0000","last_undegraded":"2023-09-23T15:24:05.411180+0000","last_fullsized":"2023-09-23T15:24:05.411180+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":217,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.420756+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:24:05.420756+0000","last_peered":"2023-09-23T15:24:05.420756+0000","last_clean":"2023-09-23T15:24:05.420756+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:24:05.420756+0000","last_undegraded":"2023-09-23T15:24:05.420756+0000","last_fullsized":"2023-09-23T15:24:05.420756+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":226,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.426975+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:24:05.426975+0000","last_peered":"2023-09-23T15:24:05.426975+0000","last_clean":"2023-09-23T15:24:05.426975+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:24:05.426975+0000","last_undegraded":"2023-09-23T15:24:05.426975+0000","last_fullsized":"2023-09-23T15:24:05.426975+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"326'9102","reported_seq":9154,"reported_epoch":326,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:19.063577+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:24:19.063577+0000","last_peered":"2023-09-23T15:24:19.063577+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:24:19.063577+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"294'2500","ondisk_log_start":"294'2500","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148504576,"num_objects":2266,"num_object_clones":0,"num_object_copies":6798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9102,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-2440}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":655,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427878+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:24:05.427878+0000","last_peered":"2023-09-23T15:24:05.427878+0000","last_clean":"2023-09-23T15:24:05.427878+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:24:05.427878+0000","last_undegraded":"2023-09-23T15:24:05.427878+0000","last_fullsized":"2023-09-23T15:24:05.427878+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":645,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432126+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:24:05.432126+0000","last_peered":"2023-09-23T15:24:05.432126+0000","last_clean":"2023-09-23T15:24:05.432126+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:24:05.432126+0000","last_undegraded":"2023-09-23T15:24:05.432126+0000","last_fullsized":"2023-09-23T15:24:05.432126+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":259,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.428928+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:24:05.428928+0000","last_peered":"2023-09-23T15:24:05.428928+0000","last_clean":"2023-09-23T15:24:05.428928+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:24:05.428928+0000","last_undegraded":"2023-09-23T15:24:05.428928+0000","last_fullsized":"2023-09-23T15:24:05.428928+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":277,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.433053+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:24:05.433053+0000","last_peered":"2023-09-23T15:24:05.433053+0000","last_clean":"2023-09-23T15:24:05.433053+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:24:05.433053+0000","last_undegraded":"2023-09-23T15:24:05.433053+0000","last_fullsized":"2023-09-23T15:24:05.433053+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"326'8791","reported_seq":8843,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:16.943551+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:24:16.943551+0000","last_peered":"2023-09-23T15:24:16.943551+0000","last_clean":"2023-09-23T15:24:16.943551+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:24:16.943551+0000","last_undegraded":"2023-09-23T15:24:16.943551+0000","last_fullsized":"2023-09-23T15:24:16.943551+0000","mapping_epoch":284,"log_start":"293'2200","ondisk_log_start":"293'2200","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":154599424,"num_objects":2359,"num_object_clones":0,"num_object_copies":7077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8791,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":632,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421309+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:24:05.421309+0000","last_peered":"2023-09-23T15:24:05.421309+0000","last_clean":"2023-09-23T15:24:05.421309+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:24:05.421309+0000","last_undegraded":"2023-09-23T15:24:05.421309+0000","last_fullsized":"2023-09-23T15:24:05.421309+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":70,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432077+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:24:05.432077+0000","last_peered":"2023-09-23T15:24:05.432077+0000","last_clean":"2023-09-23T15:24:05.432077+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:24:05.432077+0000","last_undegraded":"2023-09-23T15:24:05.432077+0000","last_fullsized":"2023-09-23T15:24:05.432077+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":293,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429858+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:24:05.429858+0000","last_peered":"2023-09-23T15:24:05.429858+0000","last_clean":"2023-09-23T15:24:05.429858+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:24:05.429858+0000","last_undegraded":"2023-09-23T15:24:05.429858+0000","last_fullsized":"2023-09-23T15:24:05.429858+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":305,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:09.138470+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:24:09.138470+0000","last_peered":"2023-09-23T15:24:09.138470+0000","last_clean":"2023-09-23T15:24:09.138470+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:24:09.138470+0000","last_undegraded":"2023-09-23T15:24:09.138470+0000","last_fullsized":"2023-09-23T15:24:09.138470+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"326'9214","reported_seq":9266,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:20.627509+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:24:20.627509+0000","last_peered":"2023-09-23T15:24:20.627509+0000","last_clean":"2023-09-23T15:24:20.627509+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:24:20.627509+0000","last_undegraded":"2023-09-23T15:24:20.627509+0000","last_fullsized":"2023-09-23T15:24:20.627509+0000","mapping_epoch":284,"log_start":"295'2800","ondisk_log_start":"295'2800","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6414,"log_dups_size":0,"ondisk_log_size":6414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153092096,"num_objects":2336,"num_object_clones":0,"num_object_copies":7008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9214,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":592,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:18.595697+0000","last_change":"2023-09-23T15:24:18.595697+0000","last_active":"2023-09-23T15:24:18.595697+0000","last_peered":"2023-09-23T15:24:18.595697+0000","last_clean":"2023-09-23T15:24:18.595697+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:24:18.595697+0000","last_undegraded":"2023-09-23T15:24:18.595697+0000","last_fullsized":"2023-09-23T15:24:18.595697+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1019,"reported_epoch":325,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:24:05.431760+0000","last_change":"2023-09-23T15:22:27.435515+0000","last_active":"2023-09-23T15:24:05.431760+0000","last_peered":"2023-09-23T15:24:05.431760+0000","last_clean":"2023-09-23T15:21:46.756689+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:24:05.431760+0000","last_undegraded":"2023-09-23T15:22:27.290223+0000","last_fullsized":"2023-09-23T15:24:05.431760+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":8,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":["7","0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":202,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.421112+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:24:05.421112+0000","last_peered":"2023-09-23T15:24:05.421112+0000","last_clean":"2023-09-23T15:24:05.421112+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:24:05.421112+0000","last_undegraded":"2023-09-23T15:24:05.421112+0000","last_fullsized":"2023-09-23T15:24:05.421112+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":289,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.414179+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:24:05.414179+0000","last_peered":"2023-09-23T15:24:05.414179+0000","last_clean":"2023-09-23T15:24:05.414179+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:24:05.414179+0000","last_undegraded":"2023-09-23T15:24:05.414179+0000","last_fullsized":"2023-09-23T15:24:05.414179+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"326'8815","reported_seq":8876,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:19.107984+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:24:19.107984+0000","last_peered":"2023-09-23T15:24:19.107984+0000","last_clean":"2023-09-23T15:24:19.107984+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:24:19.107984+0000","last_undegraded":"2023-09-23T15:24:19.107984+0000","last_fullsized":"2023-09-23T15:24:19.107984+0000","mapping_epoch":287,"log_start":"294'2200","ondisk_log_start":"294'2200","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6615,"log_dups_size":0,"ondisk_log_size":6615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151060480,"num_objects":2305,"num_object_clones":0,"num_object_copies":6915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2305,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8815,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":634,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430712+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:24:05.430712+0000","last_peered":"2023-09-23T15:24:05.430712+0000","last_clean":"2023-09-23T15:24:05.430712+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:24:05.430712+0000","last_undegraded":"2023-09-23T15:24:05.430712+0000","last_fullsized":"2023-09-23T15:24:05.430712+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":81,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.411659+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:24:05.411659+0000","last_peered":"2023-09-23T15:24:05.411659+0000","last_clean":"2023-09-23T15:24:05.411659+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:24:05.411659+0000","last_undegraded":"2023-09-23T15:24:05.411659+0000","last_fullsized":"2023-09-23T15:24:05.411659+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":857,"reported_epoch":325,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T15:24:05.422323+0000","last_change":"2023-09-23T15:22:27.316872+0000","last_active":"2023-09-23T15:24:05.422323+0000","last_peered":"2023-09-23T15:24:05.422323+0000","last_clean":"2023-09-23T15:21:04.172426+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:24:05.422323+0000","last_undegraded":"2023-09-23T15:22:27.289016+0000","last_fullsized":"2023-09-23T15:24:05.422323+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":258,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":["1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":320,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:15.627219+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:24:15.627219+0000","last_peered":"2023-09-23T15:24:15.627219+0000","last_clean":"2023-09-23T15:24:15.627219+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:24:15.627219+0000","last_undegraded":"2023-09-23T15:24:15.627219+0000","last_fullsized":"2023-09-23T15:24:15.627219+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"326'8947","reported_seq":9005,"reported_epoch":326,"state":"active+clean","last_fresh":"2023-09-23T15:24:19.910033+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:24:19.910033+0000","last_peered":"2023-09-23T15:24:19.910033+0000","last_clean":"2023-09-23T15:24:19.910033+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:24:19.910033+0000","last_undegraded":"2023-09-23T15:24:19.910033+0000","last_fullsized":"2023-09-23T15:24:19.910033+0000","mapping_epoch":287,"log_start":"293'2100","ondisk_log_start":"293'2100","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6847,"log_dups_size":0,"ondisk_log_size":6847,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141295640,"num_objects":2157,"num_object_clones":0,"num_object_copies":6471,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2157,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8947,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":650,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429171+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:24:05.429171+0000","last_peered":"2023-09-23T15:24:05.429171+0000","last_clean":"2023-09-23T15:24:05.429171+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:24:05.429171+0000","last_undegraded":"2023-09-23T15:24:05.429171+0000","last_fullsized":"2023-09-23T15:24:05.429171+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":762,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.434296+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:24:05.434296+0000","last_peered":"2023-09-23T15:24:05.434296+0000","last_clean":"2023-09-23T15:24:05.434296+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:24:05.434296+0000","last_undegraded":"2023-09-23T15:24:05.434296+0000","last_fullsized":"2023-09-23T15:24:05.434296+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":332,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:09.138492+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:24:09.138492+0000","last_peered":"2023-09-23T15:24:09.138492+0000","last_clean":"2023-09-23T15:24:09.138492+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:24:09.138492+0000","last_undegraded":"2023-09-23T15:24:09.138492+0000","last_fullsized":"2023-09-23T15:24:09.138492+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429998+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:24:05.429998+0000","last_peered":"2023-09-23T15:24:05.429998+0000","last_clean":"2023-09-23T15:24:05.429998+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:24:05.429998+0000","last_undegraded":"2023-09-23T15:24:05.429998+0000","last_fullsized":"2023-09-23T15:24:05.429998+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":308,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.414206+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:24:05.414206+0000","last_peered":"2023-09-23T15:24:05.414206+0000","last_clean":"2023-09-23T15:24:05.414206+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:24:05.414206+0000","last_undegraded":"2023-09-23T15:24:05.414206+0000","last_fullsized":"2023-09-23T15:24:05.414206+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":329,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.411499+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:24:05.411499+0000","last_peered":"2023-09-23T15:24:05.411499+0000","last_clean":"2023-09-23T15:24:05.411499+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:24:05.411499+0000","last_undegraded":"2023-09-23T15:24:05.411499+0000","last_fullsized":"2023-09-23T15:24:05.411499+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":339,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.436391+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:24:05.436391+0000","last_peered":"2023-09-23T15:24:05.436391+0000","last_clean":"2023-09-23T15:24:05.436391+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:24:05.436391+0000","last_undegraded":"2023-09-23T15:24:05.436391+0000","last_fullsized":"2023-09-23T15:24:05.436391+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.432621+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:24:05.432621+0000","last_peered":"2023-09-23T15:24:05.432621+0000","last_clean":"2023-09-23T15:24:05.432621+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:24:05.432621+0000","last_undegraded":"2023-09-23T15:24:05.432621+0000","last_fullsized":"2023-09-23T15:24:05.432621+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":319,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.427760+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:24:05.427760+0000","last_peered":"2023-09-23T15:24:05.427760+0000","last_clean":"2023-09-23T15:24:05.427760+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:24:05.427760+0000","last_undegraded":"2023-09-23T15:24:05.427760+0000","last_fullsized":"2023-09-23T15:24:05.427760+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":39,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.430444+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:24:05.430444+0000","last_peered":"2023-09-23T15:24:05.430444+0000","last_clean":"2023-09-23T15:24:05.430444+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:24:05.430444+0000","last_undegraded":"2023-09-23T15:24:05.430444+0000","last_fullsized":"2023-09-23T15:24:05.430444+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":317,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:15.627334+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:24:15.627334+0000","last_peered":"2023-09-23T15:24:15.627334+0000","last_clean":"2023-09-23T15:24:15.627334+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:24:15.627334+0000","last_undegraded":"2023-09-23T15:24:15.627334+0000","last_fullsized":"2023-09-23T15:24:15.627334+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":315,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429643+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:24:05.429643+0000","last_peered":"2023-09-23T15:24:05.429643+0000","last_clean":"2023-09-23T15:24:05.429643+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:24:05.429643+0000","last_undegraded":"2023-09-23T15:24:05.429643+0000","last_fullsized":"2023-09-23T15:24:05.429643+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":278,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.414785+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:24:05.414785+0000","last_peered":"2023-09-23T15:24:05.414785+0000","last_clean":"2023-09-23T15:24:05.414785+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:24:05.414785+0000","last_undegraded":"2023-09-23T15:24:05.414785+0000","last_fullsized":"2023-09-23T15:24:05.414785+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":53,"reported_epoch":325,"state":"active+clean","last_fresh":"2023-09-23T15:24:05.429726+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:24:05.429726+0000","last_peered":"2023-09-23T15:24:05.429726+0000","last_clean":"2023-09-23T15:24:05.429726+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:24:05.429726+0000","last_undegraded":"2023-09-23T15:24:05.429726+0000","last_fullsized":"2023-09-23T15:24:05.429726+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":2390425624,"num_objects":36476,"num_object_clones":0,"num_object_copies":109428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":24058,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":36476,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":143794,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11787,"num_bytes_recovered":704512000,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7995797504,"data_stored":7995785288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":106094,"ondisk_log_size":106094,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":95,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201635,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1069904,"kb_used_data":924296,"kb_used_omap":0,"kb_used_meta":145600,"kb_avail":92675248,"statfs":{"total":95995035648,"available":94899453952,"internally_reserved":0,"allocated":946479104,"data_stored":945481375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":149094400},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.4729999999999999,"5min":7.2160000000000002,"15min":3.8849999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":7.7629999999999999,"5min":226.226,"15min":226.226},"last":9.9619999999999997},{"interface":"front","average":{"1min":2.9399999999999999,"5min":8.5329999999999995,"15min":4.5190000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":9.4749999999999996,"5min":226.619,"15min":226.619},"last":1.929}]},{"osd":1,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.425,"15min":2.056},"min":{"1min":0.29999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.9589999999999996,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":4.1799999999999997},{"interface":"front","average":{"1min":3.2010000000000001,"5min":1.79,"15min":2.5499999999999998},"min":{"1min":0.434,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.865,"5min":11.865,"15min":16.065999999999999},"last":4.1500000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:23:23 2023","interfaces":[{"interface":"back","average":{"1min":2.407,"5min":2.4630000000000001,"15min":3.9340000000000002},"min":{"1min":0.36799999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":6.6799999999999997,"5min":90.227999999999994,"15min":170.63800000000001},"last":5.4669999999999996},{"interface":"front","average":{"1min":1.889,"5min":2.778,"15min":4.5090000000000003},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.2059999999999995,"5min":90.197999999999993,"15min":170.47999999999999},"last":6.6879999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:24:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8319999999999999,"5min":4.5869999999999997,"15min":3.9220000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":41.588999999999999,"5min":146.166,"15min":146.166},"last":4.2370000000000001},{"interface":"front","average":{"1min":5.4000000000000004,"5min":4.8520000000000003,"15min":3.98},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":41.542000000000002,"5min":146.18299999999999,"15min":146.18299999999999},"last":4.3230000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":1.181,"15min":2.5990000000000002},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6739999999999995,"5min":16.771999999999998,"15min":150.119},"last":6.3460000000000001},{"interface":"front","average":{"1min":2.302,"5min":0.92300000000000004,"15min":2.7919999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.298,"5min":9.298,"15min":150.596},"last":6.6619999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":2.8079999999999998,"15min":2.8479999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":10.153,"5min":224.441,"15min":224.441},"last":7.5279999999999996},{"interface":"front","average":{"1min":2.7400000000000002,"5min":3.056,"15min":2.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.6600000000000001,"5min":224.28700000000001,"15min":224.28700000000001},"last":9.9039999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:23:27 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.2320000000000002,"15min":2.2320000000000002},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":9.2629999999999999,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":6.6239999999999997},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.336,"5min":10.336,"15min":10.336},"last":3.4830000000000001}]}]},{"osd":6,"up_from":304,"seq":1305670058008,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":390148,"kb_used_data":322108,"kb_used_omap":13,"kb_used_meta":68018,"kb_avail":93355004,"statfs":{"total":95995035648,"available":95595524096,"internally_reserved":0,"allocated":329838592,"data_stored":328853000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":69650787},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7649999999999997,"5min":5.7649999999999997,"15min":5.7649999999999997},"last":86.492999999999995},{"interface":"front","average":{"1min":2.8780000000000001,"5min":2.8780000000000001,"15min":2.8780000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.044,"5min":10.044,"15min":10.044},"last":86.227999999999994}]},{"osd":1,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.742,"5min":2.742,"15min":2.742},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":11.064,"5min":11.064,"15min":11.064},"last":86.534000000000006},{"interface":"front","average":{"1min":2.8620000000000001,"5min":2.8620000000000001,"15min":2.8620000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":17.001999999999999,"5min":17.001999999999999,"15min":17.001999999999999},"last":86.144000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.526,"15min":1.526},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":5.1420000000000003,"5min":5.1420000000000003,"15min":5.1420000000000003},"last":0.496},{"interface":"front","average":{"1min":1.7270000000000001,"5min":1.7270000000000001,"15min":1.7270000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7430000000000003,"5min":6.7430000000000003,"15min":6.7430000000000003},"last":0.307}]},{"osd":3,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":2.2799999999999998,"15min":2.2799999999999998},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":7.0449999999999999,"5min":7.0449999999999999,"15min":7.0449999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.2719999999999998,"5min":2.2719999999999998,"15min":2.2719999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":5.5049999999999999,"5min":5.5049999999999999,"15min":5.5049999999999999},"last":0.52900000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":0.93899999999999995,"5min":0.93899999999999995,"15min":0.93899999999999995},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":5.1550000000000002,"5min":5.1550000000000002,"15min":5.1550000000000002},"last":0.75},{"interface":"front","average":{"1min":1.821,"5min":1.821,"15min":1.821},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":9.1929999999999996},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7170000000000001,"5min":1.7170000000000001,"15min":1.7170000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":12.711,"5min":12.711,"15min":12.711},"last":3.746},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":1.484},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":5.1950000000000003,"5min":5.1950000000000003,"15min":5.1950000000000003},"last":2.75}]},{"osd":7,"last update":"Sat Sep 23 15:23:29 2023","interfaces":[{"interface":"back","average":{"1min":1.3999999999999999,"5min":1.3999999999999999,"15min":1.3999999999999999},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":5.0609999999999999,"5min":5.0609999999999999,"15min":5.0609999999999999},"last":0.624},{"interface":"front","average":{"1min":1.823,"5min":1.823,"15min":1.823},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":0.47999999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015308,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1720804,"kb_used_data":1282268,"kb_used_omap":6,"kb_used_meta":438457,"kb_avail":92024348,"statfs":{"total":95995035648,"available":94232932352,"internally_reserved":0,"allocated":1313042432,"data_stored":1311509897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":448980043},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":3.3540000000000001,"15min":3.3820000000000001},"min":{"1min":0.23300000000000001,"5min":0.191,"15min":0.191},"max":{"1min":12.298,"5min":60.817999999999998,"15min":60.817999999999998},"last":2.8069999999999999},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.1550000000000002,"15min":4.165},"min":{"1min":0.36499999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.779,"5min":63.594999999999999,"15min":63.594999999999999},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.073,"5min":2.3530000000000002,"15min":4.085},"min":{"1min":0.34699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.9619999999999997,"5min":10.329000000000001,"15min":109.375},"last":1.837},{"interface":"front","average":{"1min":1.8580000000000001,"5min":2.2789999999999999,"15min":3.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":12.259,"5min":13.314,"15min":105.16},"last":2.9220000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":4.399,"15min":7.6849999999999996},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":12.218999999999999,"5min":64.540999999999997,"15min":211.405},"last":2.843},{"interface":"front","average":{"1min":2.4279999999999999,"5min":4.601,"15min":8.3529999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.551,"5min":67.465000000000003,"15min":210.988},"last":0.78900000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.0430000000000001,"5min":3.375,"15min":3.3170000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.379,"5min":80.570999999999998,"15min":80.570999999999998},"last":2.903},{"interface":"front","average":{"1min":2.016,"5min":3.5099999999999998,"15min":3.4220000000000002},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":12.279999999999999,"5min":80.75,"15min":80.75},"last":2.7589999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":9.7769999999999992,"5min":4.0979999999999999,"15min":3.4649999999999999},"min":{"1min":0.33900000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":115.401,"5min":115.401,"15min":115.401},"last":4.3090000000000002},{"interface":"front","average":{"1min":9.9770000000000003,"5min":3.9180000000000001,"15min":3.3359999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":115.169,"5min":115.169,"15min":115.169},"last":1.8999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:23:25 2023","interfaces":[{"interface":"back","average":{"1min":3.0710000000000002,"5min":3.0710000000000002,"15min":3.0710000000000002},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":12.26,"5min":12.26,"15min":12.26},"last":1.802},{"interface":"front","average":{"1min":2.3580000000000001,"5min":2.3580000000000001,"15min":2.3580000000000001},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":9.3230000000000004,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":1.1990000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":5.2889999999999997,"5min":2.6520000000000001,"15min":2.948},"min":{"1min":0.55500000000000005,"5min":0.371,"15min":0.371},"max":{"1min":19.702999999999999,"5min":19.702999999999999,"15min":19.702999999999999},"last":1.8740000000000001},{"interface":"front","average":{"1min":3.9550000000000001,"5min":2.4849999999999999,"15min":2.6800000000000002},"min":{"1min":0.5,"5min":0.377,"15min":0.377},"max":{"1min":20.722000000000001,"5min":20.722000000000001,"15min":20.722000000000001},"last":2.7120000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893358,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1776296,"kb_used_data":1189344,"kb_used_omap":8,"kb_used_meta":586615,"kb_avail":91968856,"statfs":{"total":95995035648,"available":94176108544,"internally_reserved":0,"allocated":1217888256,"data_stored":1216477902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":600694356},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,4,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.8519999999999999,"15min":3.468},"min":{"1min":0.28399999999999997,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":17.542000000000002,"5min":48.442999999999998,"15min":64.412999999999997},"last":13.276999999999999},{"interface":"front","average":{"1min":3.3239999999999998,"5min":2.7250000000000001,"15min":3.387},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":12.058999999999999,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.432}]},{"osd":2,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8199999999999998,"5min":2.7759999999999998,"15min":4.5229999999999997},"min":{"1min":0.23200000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":10.949,"5min":30.510999999999999,"15min":122.20099999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":1.9119999999999999,"5min":3.3069999999999999,"15min":4.3010000000000002},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.18099999999999999},"max":{"1min":10.861000000000001,"5min":64.466999999999999,"15min":122.071},"last":7.6230000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:23:48 2023","interfaces":[{"interface":"back","average":{"1min":12.212999999999999,"5min":5.2119999999999997,"15min":5.7519999999999998},"min":{"1min":0.39100000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":197.78899999999999,"5min":197.78899999999999,"15min":197.78899999999999},"last":0.66400000000000003},{"interface":"front","average":{"1min":12.901999999999999,"5min":5.6210000000000004,"15min":6.226},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":200.58799999999999,"5min":200.58799999999999,"15min":200.58799999999999},"last":4.8209999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":3.569,"5min":1.8899999999999999,"15min":2.1419999999999999},"min":{"1min":0.38300000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":15.358000000000001,"5min":15.358000000000001,"15min":31.341999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":2.5110000000000001,"5min":1.964,"15min":2.246},"min":{"1min":0.34100000000000003,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":9.8249999999999993,"5min":9.8249999999999993,"15min":99.168999999999997},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":2.492,"15min":2.411},"min":{"1min":0.44700000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.768000000000001,"5min":64.349999999999994,"15min":109.369},"last":10.618},{"interface":"front","average":{"1min":3.6789999999999998,"5min":1.9990000000000001,"15min":2.355},"min":{"1min":0.497,"5min":0.312,"15min":0.28599999999999998},"max":{"1min":15.436999999999999,"5min":15.436999999999999,"15min":109.387},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.8100000000000001,"5min":2.8100000000000001,"15min":2.8100000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":6.7229999999999999,"5min":6.7229999999999999,"15min":6.7229999999999999},"last":0.503},{"interface":"front","average":{"1min":3.3610000000000002,"5min":3.3610000000000002,"15min":3.3610000000000002},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":17.385000000000002,"5min":17.385000000000002,"15min":17.385000000000002},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":2.5550000000000002,"15min":2.488},"min":{"1min":0.42499999999999999,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":11.734,"5min":64.304000000000002,"15min":99.549999999999997},"last":7.5819999999999999},{"interface":"front","average":{"1min":4.9279999999999999,"5min":2.3039999999999998,"15min":2.1720000000000002},"min":{"1min":0.45900000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":15.407999999999999,"5min":15.407999999999999,"15min":31.225999999999999},"last":0.57099999999999995}]}]},{"osd":2,"up_from":105,"seq":450971566305,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1911128,"kb_used_data":1318416,"kb_used_omap":11,"kb_used_meta":592692,"kb_avail":91834024,"statfs":{"total":95995035648,"available":94038040576,"internally_reserved":0,"allocated":1350057984,"data_stored":1348657252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":606917392},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":13.015000000000001,"5min":7.2009999999999996,"15min":4.9459999999999997},"min":{"1min":0.214,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":88.622,"5min":88.622,"15min":88.622},"last":1.165},{"interface":"front","average":{"1min":12.861000000000001,"5min":7.21,"15min":4.6269999999999998},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":88.649000000000001,"5min":88.649000000000001,"15min":88.649000000000001},"last":7.218}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":7.0439999999999996,"5min":4.4900000000000002,"15min":4.7210000000000001},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":71.819000000000003,"5min":71.819000000000003,"15min":84.641000000000005},"last":0.66000000000000003},{"interface":"front","average":{"1min":7.0810000000000004,"5min":4.1989999999999998,"15min":3.6110000000000002},"min":{"1min":0.317,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":71.805000000000007,"5min":71.805000000000007,"15min":84.751999999999995},"last":7.335}]},{"osd":3,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":8.5559999999999992,"5min":6.1820000000000004,"15min":5.298},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":77.031999999999996,"5min":77.031999999999996,"15min":87.108000000000004},"last":1.0800000000000001},{"interface":"front","average":{"1min":8.2710000000000008,"5min":6.5289999999999999,"15min":5.3239999999999998},"min":{"1min":0.313,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":76.742000000000004,"5min":76.742000000000004,"15min":86.158000000000001},"last":0.73099999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.6020000000000001,"15min":2.3969999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":16.712,"5min":16.712,"15min":84.608000000000004},"last":7.2690000000000001},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.579,"15min":2.9980000000000002},"min":{"1min":0.376,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":6.3789999999999996,"5min":7.7880000000000003,"15min":87.019999999999996},"last":1.137}]},{"osd":5,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.839,"15min":2.6480000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.6980000000000004,"5min":8.9909999999999997,"15min":84.498000000000005},"last":3.1070000000000002},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.7210000000000001,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.5099999999999998,"5min":10.808999999999999,"15min":84.664000000000001},"last":7.3019999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2839999999999998,"5min":2.2839999999999998,"15min":2.2839999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":7.4329999999999998,"5min":7.4329999999999998,"15min":7.4329999999999998},"last":1.399},{"interface":"front","average":{"1min":2.0510000000000002,"5min":2.0510000000000002,"15min":2.0510000000000002},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":7.6879999999999997,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":1.1899999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.9970000000000001,"15min":2.593},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":9.3859999999999992,"5min":10.227,"15min":84.596000000000004},"last":1.3069999999999999},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.9550000000000001,"15min":2.4910000000000001},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":16.074999999999999,"5min":16.074999999999999,"15min":84.783000000000001},"last":7.6150000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402779,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1608028,"kb_used_data":993684,"kb_used_omap":3,"kb_used_meta":614332,"kb_avail":92137124,"statfs":{"total":95995035648,"available":94348414976,"internally_reserved":0,"allocated":1017532416,"data_stored":1016128396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":629076137},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":8.5269999999999992,"5min":3.7440000000000002,"15min":3.6949999999999998},"min":{"1min":0.20599999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":108.318,"5min":108.318,"15min":108.318},"last":0.36599999999999999},{"interface":"front","average":{"1min":9.3879999999999999,"5min":4.1669999999999998,"15min":3.9740000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":108.889,"5min":108.889,"15min":108.889},"last":0.22500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.8179999999999996,"15min":4.8339999999999996},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":9.2390000000000008,"5min":126.875,"15min":126.875},"last":0.432},{"interface":"front","average":{"1min":2.1400000000000001,"5min":5.5970000000000004,"15min":5.0519999999999996},"min":{"1min":0.28599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.805999999999999,"5min":128.249,"15min":128.249},"last":0.46600000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.2349999999999999,"15min":4.7539999999999996},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":12.210000000000001,"5min":17.199999999999999,"15min":109.077},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.5670000000000002,"5min":3.3410000000000002,"15min":5.7969999999999997},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.763,"5min":36.582000000000001,"15min":119.72799999999999},"last":0.33200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":2.1429999999999998,"15min":2.2669999999999999},"min":{"1min":0.35299999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.417999999999999,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.38700000000000001},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.5660000000000001,"15min":2.121},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":8.0050000000000008,"5min":12.500999999999999,"15min":52.228999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":3.7330000000000001,"15min":2.9249999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":11.372999999999999,"5min":148.619,"15min":148.619},"last":7.9260000000000002},{"interface":"front","average":{"1min":1.909,"5min":2.9780000000000002,"15min":2.7789999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":6.8520000000000003,"5min":148.56999999999999,"15min":148.56999999999999},"last":7.9530000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":9.2200000000000006,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":5.2370000000000001},{"interface":"front","average":{"1min":3.6509999999999998,"5min":3.6509999999999998,"15min":3.6509999999999998},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":36.517000000000003,"5min":36.517000000000003,"15min":36.517000000000003},"last":1.946}]},{"osd":7,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.9279999999999999,"15min":2.681},"min":{"1min":0.41799999999999998,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":4.6699999999999999,"5min":145.25,"15min":145.25},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.877,"5min":3.121,"15min":2.4990000000000001},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":7.1829999999999998,"5min":145.00899999999999,"15min":145.00899999999999},"last":5.3300000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986544,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1047176,"kb_used_data":731008,"kb_used_omap":0,"kb_used_meta":316160,"kb_avail":92697976,"statfs":{"total":95995035648,"available":94922727424,"internally_reserved":0,"allocated":748552192,"data_stored":747562561,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":323747840},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":5.2699999999999996,"5min":5.1219999999999999,"15min":4.2460000000000004},"min":{"1min":0.28299999999999997,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":53.771000000000001,"5min":162.75999999999999,"15min":162.75999999999999},"last":3.5110000000000001},{"interface":"front","average":{"1min":4.0990000000000002,"5min":4.8369999999999997,"15min":4.1959999999999997},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":53.145000000000003,"5min":162.34800000000001,"15min":162.34800000000001},"last":0.81699999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":5.1509999999999998,"5min":4.1280000000000001,"15min":3.4590000000000001},"min":{"1min":0.254,"5min":0.249,"15min":0.245},"max":{"1min":60.122,"5min":113.11199999999999,"15min":113.11199999999999},"last":2.774},{"interface":"front","average":{"1min":5.5640000000000001,"5min":4.1980000000000004,"15min":3.8839999999999999},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":60.802,"5min":113.02,"15min":113.02},"last":7.0129999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":3.3599999999999999,"15min":4.3040000000000003},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":4.8010000000000002,"5min":61.646000000000001,"15min":148.08000000000001},"last":3.1429999999999998},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.536,"15min":4.5309999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.25700000000000001},"max":{"1min":6.1980000000000004,"5min":64.840000000000003,"15min":147.88900000000001},"last":3.2200000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":6.3780000000000001,"15min":5.9969999999999999},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":6.5060000000000002,"5min":156.46000000000001,"15min":163.97499999999999},"last":3.5489999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":6.5069999999999997,"15min":5.9400000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27800000000000002},"max":{"1min":8.3829999999999991,"5min":156.74299999999999,"15min":164.447},"last":3.3159999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.069,"15min":1.6279999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":6.4089999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":0.31},{"interface":"front","average":{"1min":1.458,"5min":1.2789999999999999,"15min":1.573},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":5.7199999999999998,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:23:28 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.446,"15min":1.446},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.6470000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.2250000000000001,"15min":2.2250000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":6.1369999999999996,"5min":6.1369999999999996,"15min":6.1369999999999996},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.92500000000000004,"15min":1.1890000000000001},"min":{"1min":0.42599999999999999,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.7599999999999998,"5min":6.0730000000000004,"15min":18.23},"last":0.495},{"interface":"front","average":{"1min":1.819,"5min":1.034,"15min":1.286},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":4.2229999999999999,"5min":5.3520000000000003,"15min":14.653},"last":3.1030000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724907,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1542784,"kb_used_data":1164088,"kb_used_omap":0,"kb_used_meta":378688,"kb_avail":92202368,"statfs":{"total":95995035648,"available":94415224832,"internally_reserved":0,"allocated":1192026112,"data_stored":1191040934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":387776512},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":4.1900000000000004,"15min":3.3079999999999998},"min":{"1min":0.251,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.178999999999998,"5min":94.372,"15min":124.31},"last":0.44600000000000001},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.0720000000000001,"15min":3.3769999999999998},"min":{"1min":0.44400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.106000000000002,"5min":94.396000000000001,"15min":124.374},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":10.42,"5min":4.4420000000000002,"15min":3.1709999999999998},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":86.900999999999996,"5min":86.900999999999996,"15min":124.38800000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":5.2599999999999998,"5min":3.6110000000000002,"15min":3.0910000000000002},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":40.939,"5min":66.825000000000003,"15min":124.29000000000001},"last":1.018}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":2.444,"15min":3.7770000000000001},"min":{"1min":0.35899999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":11.754,"5min":74.019000000000005,"15min":193.00999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.8900000000000001,"5min":2.5489999999999999,"15min":3.8239999999999998},"min":{"1min":0.37,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":10.542999999999999,"5min":73.712000000000003,"15min":193.15700000000001},"last":3.3010000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":16.594999999999999,"5min":7.7089999999999996,"15min":6.0309999999999997},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":266.70299999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":0.93600000000000005},{"interface":"front","average":{"1min":20.937000000000001,"5min":8.5090000000000003,"15min":6.0449999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":266.80399999999997,"5min":266.80399999999997,"15min":266.80399999999997},"last":0.51800000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":1.292,"15min":1.9470000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":10.749000000000001,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":3.222,"5min":1.3009999999999999,"15min":2.0449999999999999},"min":{"1min":0.41699999999999998,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":10.516,"5min":10.516,"15min":138.49600000000001},"last":0.36399999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:23:26 2023","interfaces":[{"interface":"back","average":{"1min":2.3969999999999998,"5min":2.3969999999999998,"15min":2.3969999999999998},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":9.6050000000000004,"5min":9.6050000000000004,"15min":9.6050000000000004},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.4100000000000001,"5min":2.4100000000000001,"15min":2.4100000000000001},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":9.6370000000000005,"5min":9.6370000000000005,"15min":9.6370000000000005},"last":0.42399999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.131,"15min":1.407},"min":{"1min":0.40200000000000002,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":12.065,"5min":12.065,"15min":15.301},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.9809999999999999,"5min":1.0920000000000001,"15min":1.351},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":15.762},"last":0.79400000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1193213952,"data_stored":1193213952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1295646720,"data_stored":1295646720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1327435776,"data_stored":1327431704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1012072448,"data_stored":1012072448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":734658560,"data_stored":734658560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1170345984,"data_stored":1170341912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":326107136,"data_stored":326107136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":936316928,"data_stored":936312856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:24:23.708 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:24:23.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:23.857 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:23.897 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:23 smithi017 ceph-mon[215646]: 1.6 deep-scrub starts 2023-09-23T15:24:23.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:23 smithi017 ceph-mon[215646]: pgmap v955: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24075/107145 objects degraded (22.470%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:23.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:23 smithi017 ceph-mon[222878]: 1.6 deep-scrub starts 2023-09-23T15:24:23.898 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:23 smithi017 ceph-mon[222878]: pgmap v955: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24075/107145 objects degraded (22.470%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:24.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:23 smithi027 ceph-mon[195233]: 1.6 deep-scrub starts 2023-09-23T15:24:24.104 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:23 smithi027 ceph-mon[195233]: pgmap v955: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 678 op/s; 24075/107145 objects degraded (22.470%); 1.3 MiB/s, 101 objects/s recovering 2023-09-23T15:24:24.175 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:24.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:24.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:24.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:24.311 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:24.742 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:24.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:24.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:24.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:24.856 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:24.906 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:24.966 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:24.974 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:24.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:24.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:24.987 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:24 smithi027 ceph-mon[195233]: 1.6 deep-scrub starts 2023-09-23T15:24:24.989 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:25.087 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:25.159 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:25.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:24 smithi017 ceph-mon[215646]: 1.6 deep-scrub starts 2023-09-23T15:24:25.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:24 smithi017 ceph-mon[222878]: 1.6 deep-scrub starts 2023-09-23T15:24:25.285 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:25.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:25.391 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:25.470 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:25.540 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:25.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:25.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:25.650 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:25.693 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:25.799 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:25.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:25.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:25.809 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:25 smithi017 ceph-mon[215646]: pgmap v956: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 24075/104016 objects degraded (23.145%); 70 KiB/s, 82 objects/s recovering 2023-09-23T15:24:25.809 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:25 smithi017 ceph-mon[222878]: pgmap v956: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 24075/104016 objects degraded (23.145%); 70 KiB/s, 82 objects/s recovering 2023-09-23T15:24:25.812 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:25 smithi027 ceph-mon[195233]: pgmap v956: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.1 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 24075/104016 objects degraded (23.145%); 70 KiB/s, 82 objects/s recovering 2023-09-23T15:24:25.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:25.981 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:26.789 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:24:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:24:27.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:24:27.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:24:27.934 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:27.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:28.032 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:28] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.43.0" 2023-09-23T15:24:28.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: pgmap v957: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 23337/99864 objects degraded (23.369%); 69 KiB/s, 119 objects/s recovering 2023-09-23T15:24:28.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:28.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.032 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 23337/99864 objects degraded (23.369%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: pgmap v957: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 23337/99864 objects degraded (23.369%); 69 KiB/s, 119 objects/s recovering 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:28.033 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.034 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 23337/99864 objects degraded (23.369%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:28.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:28.071 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: pgmap v957: 109 pgs: 2 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 100 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 23337/99864 objects degraded (23.369%); 69 KiB/s, 119 objects/s recovering 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1970427957' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: failed to encode map e327 with expected crc 2023-09-23T15:24:28.072 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 23337/99864 objects degraded (23.369%), 9 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:28.076 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:28.178 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:28.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:28.184 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:28.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:28.513 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[215646]: failed to encode map e328 with expected crc 2023-09-23T15:24:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[215646]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T15:24:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[215646]: failed to encode map e328 with expected crc 2023-09-23T15:24:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[215646]: failed to encode map e328 with expected crc 2023-09-23T15:24:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:29.011 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:29.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:29.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:29.023 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[222878]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[222878]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T15:24:29.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[222878]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[222878]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:28 smithi027 ceph-mon[195233]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:28 smithi027 ceph-mon[195233]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T15:24:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:28 smithi027 ceph-mon[195233]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:28 smithi027 ceph-mon[195233]: failed to encode map e328 with expected crc 2023-09-23T15:24:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:29.122 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:29.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:29.265 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:29.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:29.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:29.471 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:30.094 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:30.096 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:29 smithi017 ceph-mon[215646]: pgmap v960: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 23295/97431 objects degraded (23.909%); 264 KiB/s, 97 objects/s recovering 2023-09-23T15:24:30.096 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:29 smithi017 ceph-mon[222878]: pgmap v960: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 23295/97431 objects degraded (23.909%); 264 KiB/s, 97 objects/s recovering 2023-09-23T15:24:30.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:30.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:29 smithi027 ceph-mon[195233]: pgmap v960: 109 pgs: 1 active+recovery_wait+degraded, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 2.0 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 503 op/s; 23295/97431 objects degraded (23.909%); 264 KiB/s, 97 objects/s recovering 2023-09-23T15:24:30.198 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:30.229 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:30.303 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:30.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:30.410 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:30.478 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:30.759 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:30.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:30.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:30.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:30.938 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:31.029 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:31.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:31.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:31.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:31.192 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:31.295 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:31.303 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:31.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:31.510 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:31.525 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:24:31.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:24:31.661 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:24:31.688 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:31.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:31.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:24:31.748 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:31.783 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:31.783 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:24:31.783 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:24:31.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:31.870 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:31.915 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:31.919 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:31.928 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:31.928 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22926/90582 objects degraded (25.310%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1279,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":328,"num_osds":8,"num_up_osds":8,"osd_up_since":1695482545,"num_in_osds":8,"osd_in_since":1695482516,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":109,"num_pools":5,"num_objects":30194,"data_bytes":1981737329,"bytes_used":10238005248,"bytes_avail":757722279936,"bytes_total":767960285184,"degraded_objects":22926,"degraded_total":90582,"degraded_ratio":0.25309664171689739,"recovering_objects_per_sec":143,"recovering_bytes_per_sec":270245,"recovering_keys_per_sec":0,"num_objects_recovered":1149,"num_bytes_recovered":2162688,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":689},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T15:23:24.711679+0000","services":{}},"progress_events":{"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a90e8e5c-cd46-427f-990c-d4f2d8f69255":{"message":"Global Recovery Event (3m)\n [=========================...] (remaining: 15s)","progress":0.92660552263259888,"add_to_ceph_s":true}}} 2023-09-23T15:24:31.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.951 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.952 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:24:31.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:31 smithi017 ceph-mon[215646]: pgmap v961: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 22926/90582 objects degraded (25.310%); 264 KiB/s, 143 objects/s recovering 2023-09-23T15:24:31.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2868967985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:31.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:31 smithi017 ceph-mon[222878]: pgmap v961: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 22926/90582 objects degraded (25.310%); 264 KiB/s, 143 objects/s recovering 2023-09-23T15:24:31.974 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2868967985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:32.042 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:32.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:31 smithi027 ceph-mon[195233]: pgmap v961: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 689 op/s; 22926/90582 objects degraded (25.310%); 264 KiB/s, 143 objects/s recovering 2023-09-23T15:24:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2868967985' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:32.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:32.229 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:32.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:24:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1253332391' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:33.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22926/90582 objects degraded (25.310%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:33.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1253332391' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:33.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22926/90582 objects degraded (25.310%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:33.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1253332391' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:24:33.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22926/90582 objects degraded (25.310%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:33.278 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:33.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:33.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:33.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:33.461 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:34.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:34 smithi017 ceph-mon[215646]: pgmap v962: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 22732/88500 objects degraded (25.686%); 1.8 MiB/s, 167 objects/s recovering 2023-09-23T15:24:34.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:34 smithi017 ceph-mon[222878]: pgmap v962: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 22732/88500 objects degraded (25.686%); 1.8 MiB/s, 167 objects/s recovering 2023-09-23T15:24:34.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:34 smithi027 ceph-mon[195233]: pgmap v962: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 646 op/s; 22732/88500 objects degraded (25.686%); 1.8 MiB/s, 167 objects/s recovering 2023-09-23T15:24:34.987 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:34.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:34.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:35.096 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:35.133 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:35.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:24:35.640 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:35.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:35.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:35.748 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:35.791 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:24:35.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:24:35.793 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:35.899 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:35.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:35.948 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:36.012 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:36.086 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:36.095 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:24:36.181 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:36.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:36.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:36.200 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:36 smithi017 ceph-mon[215646]: pgmap v963: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22446/85590 objects degraded (26.225%); 1.8 MiB/s, 111 objects/s recovering 2023-09-23T15:24:36.204 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:36 smithi017 ceph-mon[222878]: pgmap v963: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22446/85590 objects degraded (26.225%); 1.8 MiB/s, 111 objects/s recovering 2023-09-23T15:24:36.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:36.335 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:36 smithi027 ceph-mon[195233]: pgmap v963: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 594 op/s; 22446/85590 objects degraded (26.225%); 1.8 MiB/s, 111 objects/s recovering 2023-09-23T15:24:36.339 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:36.447 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:36.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:36.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:36.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:36.637 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:36.735 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:36.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:36.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:36.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:36.921 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:37.030 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:37.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:37.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:37.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:37.232 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:37.285 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:37.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:37.288 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:37.389 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:37.416 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:37.461 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:37.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:37.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:37.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:37.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 22446/85590 objects degraded (26.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:37.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 22446/85590 objects degraded (26.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:37.593 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:37.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 22446/85590 objects degraded (26.225%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:37.716 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:37.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:37.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:24:37.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:37.914 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:38.050 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:38.053 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:38.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:38.058 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:38.158 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:38.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:38] "GET /metrics HTTP/1.1" 200 36388 "" "Prometheus/2.43.0" 2023-09-23T15:24:38.243 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:38.479 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:38.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:38 smithi017 ceph-mon[215646]: pgmap v964: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 22446/81183 objects degraded (27.649%); 1.4 MiB/s, 89 objects/s recovering 2023-09-23T15:24:38.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:38.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:38.491 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:38 smithi017 ceph-mon[222878]: pgmap v964: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 22446/81183 objects degraded (27.649%); 1.4 MiB/s, 89 objects/s recovering 2023-09-23T15:24:38.565 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:24:38.566 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:24:38.589 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:38.590 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:38 smithi027 ceph-mon[195233]: pgmap v964: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 22446/81183 objects degraded (27.649%); 1.4 MiB/s, 89 objects/s recovering 2023-09-23T15:24:38.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:38.938 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:38.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:38.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:39.044 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:39.100 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:39.346 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:39.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:39.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:39.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:24:39.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:39.545 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:39.662 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:39.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:39.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:39.781 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:39.824 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:39.935 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:39.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:39.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:40.045 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:24:40.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:40.121 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:40.218 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:40.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:40.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:40.263 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:24:40.348 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:40.411 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:40.494 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:40.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:40.499 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:40.584 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:40 smithi027 ceph-mon[195233]: pgmap v965: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 22258/78582 objects degraded (28.325%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T15:24:40.584 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:40.600 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:40.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:40 smithi017 ceph-mon[215646]: pgmap v965: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 22258/78582 objects degraded (28.325%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T15:24:40.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:40.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:40 smithi017 ceph-mon[222878]: pgmap v965: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 22258/78582 objects degraded (28.325%); 2.2 MiB/s, 94 objects/s recovering 2023-09-23T15:24:40.667 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:24:40.676 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:41.038 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:41.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:41.045 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:41.146 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:41.173 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:41.246 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:41.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:41.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:41.353 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:41.401 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:41.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[215646]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:41.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T15:24:41.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:41 smithi017 ceph-mon[222878]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.537 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3296956149' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T15:24:41.543 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:41 smithi027 ceph-mon[195233]: failed to encode map e329 with expected crc 2023-09-23T15:24:41.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:41.544 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:41.645 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:41.707 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:41.830 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:41.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:41.848 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:41.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:42.009 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:42.148 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:24:42.148 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:24:42.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:42.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:42.384 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:42.468 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:42.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:42 smithi017 ceph-mon[215646]: pgmap v967: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 21980/71586 objects degraded (30.704%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:42.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21980/71586 objects degraded (30.704%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:42.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:42 smithi017 ceph-mon[222878]: pgmap v967: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 21980/71586 objects degraded (30.704%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:42.581 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21980/71586 objects degraded (30.704%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:42 smithi027 ceph-mon[195233]: pgmap v967: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.5 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 21980/71586 objects degraded (30.704%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21980/71586 objects degraded (30.704%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:42.742 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:42.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:42.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:42.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:42.887 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:42.968 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:42.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:24:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:42.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:42.998 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:43.027 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:24:43.090 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:43.162 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:43.283 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:43.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:43.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:43.390 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:43.431 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:43.431 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:43.436 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:43.711 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:24:43.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:24:44.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:44 smithi027 ceph-mon[195233]: pgmap v968: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 21791/69405 objects degraded (31.397%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:44.316 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:44 smithi027 ceph-mon[195233]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:44.379 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:44.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:44.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:44.532 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:44.580 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:44.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:44.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:44.683 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:44 smithi017 ceph-mon[215646]: pgmap v968: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 21791/69405 objects degraded (31.397%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:44 smithi017 ceph-mon[215646]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:44.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:44 smithi017 ceph-mon[222878]: pgmap v968: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 21791/69405 objects degraded (31.397%); 2.4 MiB/s, 94 objects/s recovering 2023-09-23T15:24:44.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:44 smithi017 ceph-mon[222878]: from='client.46052 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:44.753 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:45.010 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:45.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:45.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:45.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:45.171 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:45.458 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":968,"stamp":"2023-09-23T15:24:42.750912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1519118705,"num_objects":23135,"num_object_clones":0,"num_object_copies":69405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21791,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23135,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":158092,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14199,"num_bytes_recovered":753175925,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112476,"ondisk_log_size":112476,"up":327,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":342,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8798732,"kb_used_data":5591116,"kb_used_omap":44,"kb_used_meta":3207507,"kb_avail":741162484,"statfs":{"total":767960285184,"available":758950383616,"internally_reserved":0,"allocated":5725302784,"data_stored":5715505144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3284488123},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417136640,"num_objects":-6365,"num_object_clones":0,"num_object_copies":-19095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-941,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6365,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":941,"num_bytes_recovered":24707072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003290"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.778198+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:24:26.778198+0000","last_peered":"2023-09-23T15:24:26.778198+0000","last_clean":"2023-09-23T15:24:26.778198+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:24:26.778198+0000","last_undegraded":"2023-09-23T15:24:26.778198+0000","last_fullsized":"2023-09-23T15:24:26.778198+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":285,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757545+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:24:27.757545+0000","last_peered":"2023-09-23T15:24:27.757545+0000","last_clean":"2023-09-23T15:24:27.757545+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:24:27.757545+0000","last_undegraded":"2023-09-23T15:24:27.757545+0000","last_fullsized":"2023-09-23T15:24:27.757545+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":233,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765004+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:24:27.765004+0000","last_peered":"2023-09-23T15:24:27.765004+0000","last_clean":"2023-09-23T15:24:27.765004+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:24:27.765004+0000","last_undegraded":"2023-09-23T15:24:27.765004+0000","last_fullsized":"2023-09-23T15:24:27.765004+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":306,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.776000+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:24:26.776000+0000","last_peered":"2023-09-23T15:24:26.776000+0000","last_clean":"2023-09-23T15:24:26.776000+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:24:26.776000+0000","last_undegraded":"2023-09-23T15:24:26.776000+0000","last_fullsized":"2023-09-23T15:24:26.776000+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":330,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756432+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:24:27.756432+0000","last_peered":"2023-09-23T15:24:27.756432+0000","last_clean":"2023-09-23T15:24:27.756432+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:24:27.756432+0000","last_undegraded":"2023-09-23T15:24:27.756432+0000","last_fullsized":"2023-09-23T15:24:27.756432+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":278,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.772599+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:24:26.772599+0000","last_peered":"2023-09-23T15:24:26.772599+0000","last_clean":"2023-09-23T15:24:26.772599+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:24:26.772599+0000","last_undegraded":"2023-09-23T15:24:26.772599+0000","last_fullsized":"2023-09-23T15:24:26.772599+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":276,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.773201+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:24:26.773201+0000","last_peered":"2023-09-23T15:24:26.773201+0000","last_clean":"2023-09-23T15:24:26.773201+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:24:26.773201+0000","last_undegraded":"2023-09-23T15:24:26.773201+0000","last_fullsized":"2023-09-23T15:24:26.773201+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":327,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756464+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:24:27.756464+0000","last_peered":"2023-09-23T15:24:27.756464+0000","last_clean":"2023-09-23T15:24:27.756464+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:24:27.756464+0000","last_undegraded":"2023-09-23T15:24:27.756464+0000","last_fullsized":"2023-09-23T15:24:27.756464+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":686,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.764680+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:24:26.764680+0000","last_peered":"2023-09-23T15:24:26.764680+0000","last_clean":"2023-09-23T15:24:26.764680+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:24:26.764680+0000","last_undegraded":"2023-09-23T15:24:26.764680+0000","last_fullsized":"2023-09-23T15:24:26.764680+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":328,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.770201+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:24:26.770201+0000","last_peered":"2023-09-23T15:24:26.770201+0000","last_clean":"2023-09-23T15:24:26.770201+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:24:26.770201+0000","last_undegraded":"2023-09-23T15:24:26.770201+0000","last_fullsized":"2023-09-23T15:24:26.770201+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":281,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757016+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:24:27.757016+0000","last_peered":"2023-09-23T15:24:27.757016+0000","last_clean":"2023-09-23T15:24:27.757016+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:24:27.757016+0000","last_undegraded":"2023-09-23T15:24:27.757016+0000","last_fullsized":"2023-09-23T15:24:27.757016+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":72,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.783950+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:24:26.783950+0000","last_peered":"2023-09-23T15:24:26.783950+0000","last_clean":"2023-09-23T15:24:26.783950+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:24:26.783950+0000","last_undegraded":"2023-09-23T15:24:26.783950+0000","last_fullsized":"2023-09-23T15:24:26.783950+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":220,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765361+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:24:27.765361+0000","last_peered":"2023-09-23T15:24:27.765361+0000","last_clean":"2023-09-23T15:24:27.765361+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:24:27.765361+0000","last_undegraded":"2023-09-23T15:24:27.765361+0000","last_fullsized":"2023-09-23T15:24:27.765361+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":62,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777783+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:24:26.777783+0000","last_peered":"2023-09-23T15:24:26.777783+0000","last_clean":"2023-09-23T15:24:26.777783+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:24:26.777783+0000","last_undegraded":"2023-09-23T15:24:26.777783+0000","last_fullsized":"2023-09-23T15:24:26.777783+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.778324+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:24:26.778324+0000","last_peered":"2023-09-23T15:24:26.778324+0000","last_clean":"2023-09-23T15:24:26.778324+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:24:26.778324+0000","last_undegraded":"2023-09-23T15:24:26.778324+0000","last_fullsized":"2023-09-23T15:24:26.778324+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":637,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769897+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:24:26.769897+0000","last_peered":"2023-09-23T15:24:26.769897+0000","last_clean":"2023-09-23T15:24:26.769897+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:24:26.769897+0000","last_undegraded":"2023-09-23T15:24:26.769897+0000","last_fullsized":"2023-09-23T15:24:26.769897+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":76,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.780993+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:24:26.780993+0000","last_peered":"2023-09-23T15:24:26.780993+0000","last_clean":"2023-09-23T15:24:26.780993+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:24:26.780993+0000","last_undegraded":"2023-09-23T15:24:26.780993+0000","last_fullsized":"2023-09-23T15:24:26.780993+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":332,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757501+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:24:27.757501+0000","last_peered":"2023-09-23T15:24:27.757501+0000","last_clean":"2023-09-23T15:24:27.757501+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:24:27.757501+0000","last_undegraded":"2023-09-23T15:24:27.757501+0000","last_fullsized":"2023-09-23T15:24:27.757501+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":686,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769437+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:24:26.769437+0000","last_peered":"2023-09-23T15:24:26.769437+0000","last_clean":"2023-09-23T15:24:26.769437+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:24:26.769437+0000","last_undegraded":"2023-09-23T15:24:26.769437+0000","last_fullsized":"2023-09-23T15:24:26.769437+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":305,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771502+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:24:26.771502+0000","last_peered":"2023-09-23T15:24:26.771502+0000","last_clean":"2023-09-23T15:24:26.771502+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:24:26.771502+0000","last_undegraded":"2023-09-23T15:24:26.771502+0000","last_fullsized":"2023-09-23T15:24:26.771502+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":313,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.770339+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:24:26.770339+0000","last_peered":"2023-09-23T15:24:26.770339+0000","last_clean":"2023-09-23T15:24:26.770339+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:24:26.770339+0000","last_undegraded":"2023-09-23T15:24:26.770339+0000","last_fullsized":"2023-09-23T15:24:26.770339+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":619,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765636+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:24:27.765636+0000","last_peered":"2023-09-23T15:24:27.765636+0000","last_clean":"2023-09-23T15:24:27.765636+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:24:27.765636+0000","last_undegraded":"2023-09-23T15:24:27.765636+0000","last_fullsized":"2023-09-23T15:24:27.765636+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":76,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.780618+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:24:26.780618+0000","last_peered":"2023-09-23T15:24:26.780618+0000","last_clean":"2023-09-23T15:24:26.780618+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:24:26.780618+0000","last_undegraded":"2023-09-23T15:24:26.780618+0000","last_fullsized":"2023-09-23T15:24:26.780618+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":294,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.776040+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:24:26.776040+0000","last_peered":"2023-09-23T15:24:26.776040+0000","last_clean":"2023-09-23T15:24:26.776040+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:24:26.776040+0000","last_undegraded":"2023-09-23T15:24:26.776040+0000","last_fullsized":"2023-09-23T15:24:26.776040+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":644,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.768398+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:24:26.768398+0000","last_peered":"2023-09-23T15:24:26.768398+0000","last_clean":"2023-09-23T15:24:26.768398+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:24:26.768398+0000","last_undegraded":"2023-09-23T15:24:26.768398+0000","last_fullsized":"2023-09-23T15:24:26.768398+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":264,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771646+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:24:26.771646+0000","last_peered":"2023-09-23T15:24:26.771646+0000","last_clean":"2023-09-23T15:24:26.771646+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:24:26.771646+0000","last_undegraded":"2023-09-23T15:24:26.771646+0000","last_fullsized":"2023-09-23T15:24:26.771646+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":332,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757285+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:24:27.757285+0000","last_peered":"2023-09-23T15:24:27.757285+0000","last_clean":"2023-09-23T15:24:27.757285+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:24:27.757285+0000","last_undegraded":"2023-09-23T15:24:27.757285+0000","last_fullsized":"2023-09-23T15:24:27.757285+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"328'9946","reported_seq":16501,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:39.880681+0000","last_change":"2023-09-23T15:24:27.826512+0000","last_active":"2023-09-23T15:24:39.880681+0000","last_peered":"2023-09-23T15:24:39.880681+0000","last_clean":"2023-09-23T15:24:39.880681+0000","last_became_active":"2023-09-23T15:24:27.764262+0000","last_became_peered":"2023-09-23T15:24:27.764262+0000","last_unstale":"2023-09-23T15:24:39.880681+0000","last_undegraded":"2023-09-23T15:24:39.880681+0000","last_fullsized":"2023-09-23T15:24:39.880681+0000","mapping_epoch":327,"log_start":"294'2500","ondisk_log_start":"294'2500","created":284,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7446,"log_dups_size":0,"ondisk_log_size":7446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:39.059424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96206848,"num_objects":1468,"num_object_clones":0,"num_object_copies":4404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9946,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":87425024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":590,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.784106+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:24:26.784106+0000","last_peered":"2023-09-23T15:24:26.784106+0000","last_clean":"2023-09-23T15:24:26.784106+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:24:26.784106+0000","last_undegraded":"2023-09-23T15:24:26.784106+0000","last_fullsized":"2023-09-23T15:24:26.784106+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":68,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.781286+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:24:26.781286+0000","last_peered":"2023-09-23T15:24:26.781286+0000","last_clean":"2023-09-23T15:24:26.781286+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:24:26.781286+0000","last_undegraded":"2023-09-23T15:24:26.781286+0000","last_fullsized":"2023-09-23T15:24:26.781286+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":320,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769196+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:24:26.769196+0000","last_peered":"2023-09-23T15:24:26.769196+0000","last_clean":"2023-09-23T15:24:26.769196+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:24:26.769196+0000","last_undegraded":"2023-09-23T15:24:26.769196+0000","last_fullsized":"2023-09-23T15:24:26.769196+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"328'9599","reported_seq":18861,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:39.407386+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:24:39.407386+0000","last_peered":"2023-09-23T15:24:39.407386+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:24:39.407386+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"295'2700","ondisk_log_start":"295'2700","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6899,"log_dups_size":0,"ondisk_log_size":6899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96665600,"num_objects":1475,"num_object_clones":0,"num_object_copies":4425,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1484,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1475,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9599,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1","4"],"object_location_counts":[{"shards":"0,4,7","objects":485},{"shards":"4,7","objects":-9}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":649,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757476+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:24:27.757476+0000","last_peered":"2023-09-23T15:24:27.757476+0000","last_clean":"2023-09-23T15:24:27.757476+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:24:27.757476+0000","last_undegraded":"2023-09-23T15:24:27.757476+0000","last_fullsized":"2023-09-23T15:24:27.757476+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":68,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.783746+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:24:26.783746+0000","last_peered":"2023-09-23T15:24:26.783746+0000","last_clean":"2023-09-23T15:24:26.783746+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:24:26.783746+0000","last_undegraded":"2023-09-23T15:24:26.783746+0000","last_fullsized":"2023-09-23T15:24:26.783746+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777056+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:24:26.777056+0000","last_peered":"2023-09-23T15:24:26.777056+0000","last_clean":"2023-09-23T15:24:26.777056+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:24:26.777056+0000","last_undegraded":"2023-09-23T15:24:26.777056+0000","last_fullsized":"2023-09-23T15:24:26.777056+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"328'9774","reported_seq":9847,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:39.191263+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:24:39.191263+0000","last_peered":"2023-09-23T15:24:39.191263+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:24:39.191263+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"298'3200","ondisk_log_start":"298'3200","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94371840,"num_objects":1440,"num_object_clones":0,"num_object_copies":4320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9774,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2,7","objects":462},{"shards":"0,7","objects":-3190}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":535,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.772761+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:24:26.772761+0000","last_peered":"2023-09-23T15:24:26.772761+0000","last_clean":"2023-09-23T15:24:26.772761+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:24:26.772761+0000","last_undegraded":"2023-09-23T15:24:26.772761+0000","last_fullsized":"2023-09-23T15:24:26.772761+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":68,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.783902+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:24:26.783902+0000","last_peered":"2023-09-23T15:24:26.783902+0000","last_clean":"2023-09-23T15:24:26.783902+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:24:26.783902+0000","last_undegraded":"2023-09-23T15:24:26.783902+0000","last_fullsized":"2023-09-23T15:24:26.783902+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":291,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771859+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:24:26.771859+0000","last_peered":"2023-09-23T15:24:26.771859+0000","last_clean":"2023-09-23T15:24:26.771859+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:24:26.771859+0000","last_undegraded":"2023-09-23T15:24:26.771859+0000","last_fullsized":"2023-09-23T15:24:26.771859+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"329'9908","reported_seq":11177,"reported_epoch":329,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:42.016686+0000","last_change":"2023-09-23T15:24:25.988237+0000","last_active":"2023-09-23T15:24:42.016686+0000","last_peered":"2023-09-23T15:24:42.016686+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:24:42.016686+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"299'3300","ondisk_log_start":"299'3300","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6608,"log_dups_size":0,"ondisk_log_size":6608,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9908,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":604,"num_bytes_recovered":39583744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-2648}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":543,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777687+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:24:26.777687+0000","last_peered":"2023-09-23T15:24:26.777687+0000","last_clean":"2023-09-23T15:24:26.777687+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:24:26.777687+0000","last_undegraded":"2023-09-23T15:24:26.777687+0000","last_fullsized":"2023-09-23T15:24:26.777687+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":304,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.774579+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:24:26.774579+0000","last_peered":"2023-09-23T15:24:26.774579+0000","last_clean":"2023-09-23T15:24:26.774579+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:24:26.774579+0000","last_undegraded":"2023-09-23T15:24:26.774579+0000","last_fullsized":"2023-09-23T15:24:26.774579+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":309,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771755+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:24:26.771755+0000","last_peered":"2023-09-23T15:24:26.771755+0000","last_clean":"2023-09-23T15:24:26.771755+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:24:26.771755+0000","last_undegraded":"2023-09-23T15:24:26.771755+0000","last_fullsized":"2023-09-23T15:24:26.771755+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"329'9862","reported_seq":9916,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.612475+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:24:40.612475+0000","last_peered":"2023-09-23T15:24:40.612475+0000","last_clean":"2023-09-23T15:24:40.612475+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:24:40.612475+0000","last_undegraded":"2023-09-23T15:24:40.612475+0000","last_fullsized":"2023-09-23T15:24:40.612475+0000","mapping_epoch":284,"log_start":"300'3300","ondisk_log_start":"300'3300","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92536832,"num_objects":1412,"num_object_clones":0,"num_object_copies":4236,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1412,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9862,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":564,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.780849+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:24:26.780849+0000","last_peered":"2023-09-23T15:24:26.780849+0000","last_clean":"2023-09-23T15:24:26.780849+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:24:26.780849+0000","last_undegraded":"2023-09-23T15:24:26.780849+0000","last_fullsized":"2023-09-23T15:24:26.780849+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":322,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757392+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:24:27.757392+0000","last_peered":"2023-09-23T15:24:27.757392+0000","last_clean":"2023-09-23T15:24:27.757392+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:24:27.757392+0000","last_undegraded":"2023-09-23T15:24:27.757392+0000","last_fullsized":"2023-09-23T15:24:27.757392+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777629+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:24:26.777629+0000","last_peered":"2023-09-23T15:24:26.777629+0000","last_clean":"2023-09-23T15:24:26.777629+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:24:26.777629+0000","last_undegraded":"2023-09-23T15:24:26.777629+0000","last_fullsized":"2023-09-23T15:24:26.777629+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"329'10110","reported_seq":10165,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:41.922032+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:24:41.922032+0000","last_peered":"2023-09-23T15:24:41.922032+0000","last_clean":"2023-09-23T15:24:41.922032+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:24:41.922032+0000","last_undegraded":"2023-09-23T15:24:41.922032+0000","last_fullsized":"2023-09-23T15:24:41.922032+0000","mapping_epoch":284,"log_start":"300'3500","ondisk_log_start":"300'3500","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93847552,"num_objects":1432,"num_object_clones":0,"num_object_copies":4296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1432,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10110,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":632,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769514+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:24:26.769514+0000","last_peered":"2023-09-23T15:24:26.769514+0000","last_clean":"2023-09-23T15:24:26.769514+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:24:26.769514+0000","last_undegraded":"2023-09-23T15:24:26.769514+0000","last_fullsized":"2023-09-23T15:24:26.769514+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":275,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771904+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:24:26.771904+0000","last_peered":"2023-09-23T15:24:26.771904+0000","last_clean":"2023-09-23T15:24:26.771904+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:24:26.771904+0000","last_undegraded":"2023-09-23T15:24:26.771904+0000","last_fullsized":"2023-09-23T15:24:26.771904+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777794+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:24:26.777794+0000","last_peered":"2023-09-23T15:24:26.777794+0000","last_clean":"2023-09-23T15:24:26.777794+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:24:26.777794+0000","last_undegraded":"2023-09-23T15:24:26.777794+0000","last_fullsized":"2023-09-23T15:24:26.777794+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"328'9969","reported_seq":16520,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:39.730387+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:24:39.730387+0000","last_peered":"2023-09-23T15:24:39.730387+0000","last_clean":"2023-09-23T15:24:39.730387+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:24:39.730387+0000","last_undegraded":"2023-09-23T15:24:39.730387+0000","last_fullsized":"2023-09-23T15:24:39.730387+0000","mapping_epoch":310,"log_start":"300'3400","ondisk_log_start":"300'3400","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96534528,"num_objects":1473,"num_object_clones":0,"num_object_copies":4419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9969,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":585,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765567+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:24:27.765567+0000","last_peered":"2023-09-23T15:24:27.765567+0000","last_clean":"2023-09-23T15:24:27.765567+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:24:27.765567+0000","last_undegraded":"2023-09-23T15:24:27.765567+0000","last_fullsized":"2023-09-23T15:24:27.765567+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777128+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:24:26.777128+0000","last_peered":"2023-09-23T15:24:26.777128+0000","last_clean":"2023-09-23T15:24:26.777128+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:24:26.777128+0000","last_undegraded":"2023-09-23T15:24:26.777128+0000","last_fullsized":"2023-09-23T15:24:26.777128+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777606+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:24:26.777606+0000","last_peered":"2023-09-23T15:24:26.777606+0000","last_clean":"2023-09-23T15:24:26.777606+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:24:26.777606+0000","last_undegraded":"2023-09-23T15:24:26.777606+0000","last_fullsized":"2023-09-23T15:24:26.777606+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"329'9904","reported_seq":9964,"reported_epoch":329,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:41.987455+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:24:41.987455+0000","last_peered":"2023-09-23T15:24:41.987455+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:24:41.987455+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"300'3300","ondisk_log_start":"300'3300","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6604,"log_dups_size":0,"ondisk_log_size":6604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91357184,"num_objects":1394,"num_object_clones":0,"num_object_copies":4182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9904,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-3224}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":644,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.768266+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:24:26.768266+0000","last_peered":"2023-09-23T15:24:26.768266+0000","last_clean":"2023-09-23T15:24:26.768266+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:24:26.768266+0000","last_undegraded":"2023-09-23T15:24:26.768266+0000","last_fullsized":"2023-09-23T15:24:26.768266+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":343,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756374+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:24:27.756374+0000","last_peered":"2023-09-23T15:24:27.756374+0000","last_clean":"2023-09-23T15:24:27.756374+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:24:27.756374+0000","last_undegraded":"2023-09-23T15:24:27.756374+0000","last_fullsized":"2023-09-23T15:24:27.756374+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":237,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.766515+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:24:27.766515+0000","last_peered":"2023-09-23T15:24:27.766515+0000","last_clean":"2023-09-23T15:24:27.766515+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:24:27.766515+0000","last_undegraded":"2023-09-23T15:24:27.766515+0000","last_fullsized":"2023-09-23T15:24:27.766515+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"328'9633","reported_seq":11289,"reported_epoch":328,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:39.187016+0000","last_change":"2023-09-23T15:24:25.991143+0000","last_active":"2023-09-23T15:24:39.187016+0000","last_peered":"2023-09-23T15:24:39.187016+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:24:39.187016+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"298'3100","ondisk_log_start":"298'3100","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6533,"log_dups_size":0,"ondisk_log_size":6533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94437376,"num_objects":1441,"num_object_clones":0,"num_object_copies":4323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2301,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9633,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":792,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":-860}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":525,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.772679+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:24:26.772679+0000","last_peered":"2023-09-23T15:24:26.772679+0000","last_clean":"2023-09-23T15:24:26.772679+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:24:26.772679+0000","last_undegraded":"2023-09-23T15:24:26.772679+0000","last_fullsized":"2023-09-23T15:24:26.772679+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":528,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765358+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:24:27.765358+0000","last_peered":"2023-09-23T15:24:27.765358+0000","last_clean":"2023-09-23T15:24:27.765358+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:24:27.765358+0000","last_undegraded":"2023-09-23T15:24:27.765358+0000","last_fullsized":"2023-09-23T15:24:27.765358+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":304,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.770273+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:24:26.770273+0000","last_peered":"2023-09-23T15:24:26.770273+0000","last_clean":"2023-09-23T15:24:26.770273+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:24:26.770273+0000","last_undegraded":"2023-09-23T15:24:26.770273+0000","last_fullsized":"2023-09-23T15:24:26.770273+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":272,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.763344+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:24:27.763344+0000","last_peered":"2023-09-23T15:24:27.763344+0000","last_clean":"2023-09-23T15:24:27.763344+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:24:27.763344+0000","last_undegraded":"2023-09-23T15:24:27.763344+0000","last_fullsized":"2023-09-23T15:24:27.763344+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"328'9768","reported_seq":9830,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:39.880817+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:24:39.880817+0000","last_peered":"2023-09-23T15:24:39.880817+0000","last_clean":"2023-09-23T15:24:39.880817+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:24:39.880817+0000","last_undegraded":"2023-09-23T15:24:39.880817+0000","last_fullsized":"2023-09-23T15:24:39.880817+0000","mapping_epoch":287,"log_start":"294'2400","ondisk_log_start":"294'2400","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95420416,"num_objects":1456,"num_object_clones":0,"num_object_copies":4368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9768,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":625,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769735+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:24:26.769735+0000","last_peered":"2023-09-23T15:24:26.769735+0000","last_clean":"2023-09-23T15:24:26.769735+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:24:26.769735+0000","last_undegraded":"2023-09-23T15:24:26.769735+0000","last_fullsized":"2023-09-23T15:24:26.769735+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":863,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.768531+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:24:26.768531+0000","last_peered":"2023-09-23T15:24:26.768531+0000","last_clean":"2023-09-23T15:24:26.768531+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:24:26.768531+0000","last_undegraded":"2023-09-23T15:24:26.768531+0000","last_fullsized":"2023-09-23T15:24:26.768531+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.778249+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:24:26.778249+0000","last_peered":"2023-09-23T15:24:26.778249+0000","last_clean":"2023-09-23T15:24:26.778249+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:24:26.778249+0000","last_undegraded":"2023-09-23T15:24:26.778249+0000","last_fullsized":"2023-09-23T15:24:26.778249+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":225,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765167+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:24:27.765167+0000","last_peered":"2023-09-23T15:24:27.765167+0000","last_clean":"2023-09-23T15:24:27.765167+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:24:27.765167+0000","last_undegraded":"2023-09-23T15:24:27.765167+0000","last_fullsized":"2023-09-23T15:24:27.765167+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"329'9711","reported_seq":16002,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.594188+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:24:40.594188+0000","last_peered":"2023-09-23T15:24:40.594188+0000","last_clean":"2023-09-23T15:24:40.594188+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:24:40.594188+0000","last_undegraded":"2023-09-23T15:24:40.594188+0000","last_fullsized":"2023-09-23T15:24:40.594188+0000","mapping_epoch":320,"log_start":"295'3100","ondisk_log_start":"295'3100","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6611,"log_dups_size":0,"ondisk_log_size":6611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91684864,"num_objects":1399,"num_object_clones":0,"num_object_copies":4197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9711,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":683,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.780903+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:24:26.780903+0000","last_peered":"2023-09-23T15:24:26.780903+0000","last_clean":"2023-09-23T15:24:26.780903+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:24:26.780903+0000","last_undegraded":"2023-09-23T15:24:26.780903+0000","last_fullsized":"2023-09-23T15:24:26.780903+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":945,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.763764+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:24:27.763764+0000","last_peered":"2023-09-23T15:24:27.763764+0000","last_clean":"2023-09-23T15:24:27.763764+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:24:27.763764+0000","last_undegraded":"2023-09-23T15:24:27.763764+0000","last_fullsized":"2023-09-23T15:24:27.763764+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":220,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765046+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:24:27.765046+0000","last_peered":"2023-09-23T15:24:27.765046+0000","last_clean":"2023-09-23T15:24:27.765046+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:24:27.765046+0000","last_undegraded":"2023-09-23T15:24:27.765046+0000","last_fullsized":"2023-09-23T15:24:27.765046+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":229,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.766476+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:24:27.766476+0000","last_peered":"2023-09-23T15:24:27.766476+0000","last_clean":"2023-09-23T15:24:27.766476+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:24:27.766476+0000","last_undegraded":"2023-09-23T15:24:27.766476+0000","last_fullsized":"2023-09-23T15:24:27.766476+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"328'9862","reported_seq":9917,"reported_epoch":328,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:39.177539+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:24:39.177539+0000","last_peered":"2023-09-23T15:24:39.177539+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:24:39.177539+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"299'3300","ondisk_log_start":"299'3300","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6562,"log_dups_size":0,"ondisk_log_size":6562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98697216,"num_objects":1506,"num_object_clones":0,"num_object_copies":4518,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1506,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9862,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-3200}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":657,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.770102+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:24:26.770102+0000","last_peered":"2023-09-23T15:24:26.770102+0000","last_clean":"2023-09-23T15:24:26.770102+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:24:26.770102+0000","last_undegraded":"2023-09-23T15:24:26.770102+0000","last_fullsized":"2023-09-23T15:24:26.770102+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":647,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.783258+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:24:26.783258+0000","last_peered":"2023-09-23T15:24:26.783258+0000","last_clean":"2023-09-23T15:24:26.783258+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:24:26.783258+0000","last_undegraded":"2023-09-23T15:24:26.783258+0000","last_fullsized":"2023-09-23T15:24:26.783258+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":262,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756556+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:24:27.756556+0000","last_peered":"2023-09-23T15:24:27.756556+0000","last_clean":"2023-09-23T15:24:27.756556+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:24:27.756556+0000","last_undegraded":"2023-09-23T15:24:27.756556+0000","last_fullsized":"2023-09-23T15:24:27.756556+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":279,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.772297+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:24:26.772297+0000","last_peered":"2023-09-23T15:24:26.772297+0000","last_clean":"2023-09-23T15:24:26.772297+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:24:26.772297+0000","last_undegraded":"2023-09-23T15:24:26.772297+0000","last_fullsized":"2023-09-23T15:24:26.772297+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"329'9787","reported_seq":9842,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:41.960776+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:24:41.960776+0000","last_peered":"2023-09-23T15:24:41.960776+0000","last_clean":"2023-09-23T15:24:41.960776+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:24:41.960776+0000","last_undegraded":"2023-09-23T15:24:41.960776+0000","last_fullsized":"2023-09-23T15:24:41.960776+0000","mapping_epoch":284,"log_start":"299'3200","ondisk_log_start":"299'3200","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6587,"log_dups_size":0,"ondisk_log_size":6587,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89325568,"num_objects":1363,"num_object_clones":0,"num_object_copies":4089,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1363,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9787,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":635,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765496+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:24:27.765496+0000","last_peered":"2023-09-23T15:24:27.765496+0000","last_clean":"2023-09-23T15:24:27.765496+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:24:27.765496+0000","last_undegraded":"2023-09-23T15:24:27.765496+0000","last_fullsized":"2023-09-23T15:24:27.765496+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":72,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.783257+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:24:26.783257+0000","last_peered":"2023-09-23T15:24:26.783257+0000","last_clean":"2023-09-23T15:24:26.783257+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:24:26.783257+0000","last_undegraded":"2023-09-23T15:24:26.783257+0000","last_fullsized":"2023-09-23T15:24:26.783257+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":295,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.769813+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:24:26.769813+0000","last_peered":"2023-09-23T15:24:26.769813+0000","last_clean":"2023-09-23T15:24:26.769813+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:24:26.769813+0000","last_undegraded":"2023-09-23T15:24:26.769813+0000","last_fullsized":"2023-09-23T15:24:26.769813+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":307,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.775882+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:24:26.775882+0000","last_peered":"2023-09-23T15:24:26.775882+0000","last_clean":"2023-09-23T15:24:26.775882+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:24:26.775882+0000","last_undegraded":"2023-09-23T15:24:26.775882+0000","last_fullsized":"2023-09-23T15:24:26.775882+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"329'10034","reported_seq":10089,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.572235+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:24:40.572235+0000","last_peered":"2023-09-23T15:24:40.572235+0000","last_clean":"2023-09-23T15:24:40.572235+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:24:40.572235+0000","last_undegraded":"2023-09-23T15:24:40.572235+0000","last_fullsized":"2023-09-23T15:24:40.572235+0000","mapping_epoch":284,"log_start":"300'3500","ondisk_log_start":"300'3500","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6534,"log_dups_size":0,"ondisk_log_size":6534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99352576,"num_objects":1516,"num_object_clones":0,"num_object_copies":4548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10034,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":606,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.773984+0000","last_change":"2023-09-23T15:24:24.592597+0000","last_active":"2023-09-23T15:24:26.773984+0000","last_peered":"2023-09-23T15:24:26.773984+0000","last_clean":"2023-09-23T15:24:26.773984+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:24:26.773984+0000","last_undegraded":"2023-09-23T15:24:26.773984+0000","last_fullsized":"2023-09-23T15:24:26.773984+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1041,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.784051+0000","last_change":"2023-09-23T15:24:25.990916+0000","last_active":"2023-09-23T15:24:26.784051+0000","last_peered":"2023-09-23T15:24:26.784051+0000","last_clean":"2023-09-23T15:24:26.784051+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:24:26.784051+0000","last_undegraded":"2023-09-23T15:24:26.784051+0000","last_fullsized":"2023-09-23T15:24:26.784051+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":205,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765260+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:24:27.765260+0000","last_peered":"2023-09-23T15:24:27.765260+0000","last_clean":"2023-09-23T15:24:27.765260+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:24:27.765260+0000","last_undegraded":"2023-09-23T15:24:27.765260+0000","last_fullsized":"2023-09-23T15:24:27.765260+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":291,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.771976+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:24:26.771976+0000","last_peered":"2023-09-23T15:24:26.771976+0000","last_clean":"2023-09-23T15:24:26.771976+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:24:26.771976+0000","last_undegraded":"2023-09-23T15:24:26.771976+0000","last_fullsized":"2023-09-23T15:24:26.771976+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"328'9602","reported_seq":9665,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:39.142068+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:24:39.142068+0000","last_peered":"2023-09-23T15:24:39.142068+0000","last_clean":"2023-09-23T15:24:39.142068+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:24:39.142068+0000","last_undegraded":"2023-09-23T15:24:39.142068+0000","last_fullsized":"2023-09-23T15:24:39.142068+0000","mapping_epoch":287,"log_start":"295'3000","ondisk_log_start":"295'3000","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99483648,"num_objects":1518,"num_object_clones":0,"num_object_copies":4554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":636,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.777722+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:24:26.777722+0000","last_peered":"2023-09-23T15:24:26.777722+0000","last_clean":"2023-09-23T15:24:26.777722+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:24:26.777722+0000","last_undegraded":"2023-09-23T15:24:26.777722+0000","last_fullsized":"2023-09-23T15:24:26.777722+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":83,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.764995+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:24:26.764995+0000","last_peered":"2023-09-23T15:24:26.764995+0000","last_clean":"2023-09-23T15:24:26.764995+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:24:26.764995+0000","last_undegraded":"2023-09-23T15:24:26.764995+0000","last_fullsized":"2023-09-23T15:24:26.764995+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":882,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.765469+0000","last_change":"2023-09-23T15:24:25.979031+0000","last_active":"2023-09-23T15:24:27.765469+0000","last_peered":"2023-09-23T15:24:27.765469+0000","last_clean":"2023-09-23T15:24:27.765469+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:24:27.765469+0000","last_undegraded":"2023-09-23T15:24:27.765469+0000","last_fullsized":"2023-09-23T15:24:27.765469+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":323,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756516+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:24:27.756516+0000","last_peered":"2023-09-23T15:24:27.756516+0000","last_clean":"2023-09-23T15:24:27.756516+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:24:27.756516+0000","last_undegraded":"2023-09-23T15:24:27.756516+0000","last_fullsized":"2023-09-23T15:24:27.756516+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"328'9694","reported_seq":9754,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:39.877680+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:24:39.877680+0000","last_peered":"2023-09-23T15:24:39.877680+0000","last_clean":"2023-09-23T15:24:39.877680+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:24:39.877680+0000","last_undegraded":"2023-09-23T15:24:39.877680+0000","last_fullsized":"2023-09-23T15:24:39.877680+0000","mapping_epoch":287,"log_start":"294'2300","ondisk_log_start":"294'2300","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7394,"log_dups_size":0,"ondisk_log_size":7394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92340248,"num_objects":1410,"num_object_clones":0,"num_object_copies":4230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1410,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9694,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":653,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.757367+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:24:27.757367+0000","last_peered":"2023-09-23T15:24:27.757367+0000","last_clean":"2023-09-23T15:24:27.757367+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:24:27.757367+0000","last_undegraded":"2023-09-23T15:24:27.757367+0000","last_fullsized":"2023-09-23T15:24:27.757367+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":764,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.772520+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:24:26.772520+0000","last_peered":"2023-09-23T15:24:26.772520+0000","last_clean":"2023-09-23T15:24:26.772520+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:24:26.772520+0000","last_undegraded":"2023-09-23T15:24:26.772520+0000","last_fullsized":"2023-09-23T15:24:26.772520+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":334,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.775955+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:24:26.775955+0000","last_peered":"2023-09-23T15:24:26.775955+0000","last_clean":"2023-09-23T15:24:26.775955+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:24:26.775955+0000","last_undegraded":"2023-09-23T15:24:26.775955+0000","last_fullsized":"2023-09-23T15:24:26.775955+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.776988+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:24:26.776988+0000","last_peered":"2023-09-23T15:24:26.776988+0000","last_clean":"2023-09-23T15:24:26.776988+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:24:26.776988+0000","last_undegraded":"2023-09-23T15:24:26.776988+0000","last_fullsized":"2023-09-23T15:24:26.776988+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":311,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.768138+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:24:27.768138+0000","last_peered":"2023-09-23T15:24:27.768138+0000","last_clean":"2023-09-23T15:24:27.768138+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:24:27.768138+0000","last_undegraded":"2023-09-23T15:24:27.768138+0000","last_fullsized":"2023-09-23T15:24:27.768138+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":331,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.764803+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:24:26.764803+0000","last_peered":"2023-09-23T15:24:26.764803+0000","last_clean":"2023-09-23T15:24:26.764803+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:24:26.764803+0000","last_undegraded":"2023-09-23T15:24:26.764803+0000","last_fullsized":"2023-09-23T15:24:26.764803+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":341,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.774671+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:24:26.774671+0000","last_peered":"2023-09-23T15:24:26.774671+0000","last_clean":"2023-09-23T15:24:26.774671+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:24:26.774671+0000","last_undegraded":"2023-09-23T15:24:26.774671+0000","last_fullsized":"2023-09-23T15:24:26.774671+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.778309+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:24:26.778309+0000","last_peered":"2023-09-23T15:24:26.778309+0000","last_clean":"2023-09-23T15:24:26.778309+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:24:26.778309+0000","last_undegraded":"2023-09-23T15:24:26.778309+0000","last_fullsized":"2023-09-23T15:24:26.778309+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":321,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.770194+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:24:26.770194+0000","last_peered":"2023-09-23T15:24:26.770194+0000","last_clean":"2023-09-23T15:24:26.770194+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:24:26.770194+0000","last_undegraded":"2023-09-23T15:24:26.770194+0000","last_fullsized":"2023-09-23T15:24:26.770194+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":41,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.778341+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:24:26.778341+0000","last_peered":"2023-09-23T15:24:26.778341+0000","last_clean":"2023-09-23T15:24:26.778341+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:24:26.778341+0000","last_undegraded":"2023-09-23T15:24:26.778341+0000","last_fullsized":"2023-09-23T15:24:26.778341+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":320,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756845+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:24:27.756845+0000","last_peered":"2023-09-23T15:24:27.756845+0000","last_clean":"2023-09-23T15:24:27.756845+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:24:27.756845+0000","last_undegraded":"2023-09-23T15:24:27.756845+0000","last_fullsized":"2023-09-23T15:24:27.756845+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":318,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.756951+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:24:27.756951+0000","last_peered":"2023-09-23T15:24:27.756951+0000","last_clean":"2023-09-23T15:24:27.756951+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:24:27.756951+0000","last_undegraded":"2023-09-23T15:24:27.756951+0000","last_fullsized":"2023-09-23T15:24:27.756951+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":280,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.773224+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:24:26.773224+0000","last_peered":"2023-09-23T15:24:26.773224+0000","last_clean":"2023-09-23T15:24:26.773224+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:24:26.773224+0000","last_undegraded":"2023-09-23T15:24:26.773224+0000","last_fullsized":"2023-09-23T15:24:26.773224+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":55,"reported_epoch":327,"state":"active+clean","last_fresh":"2023-09-23T15:24:26.776730+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:24:26.776730+0000","last_peered":"2023-09-23T15:24:26.776730+0000","last_clean":"2023-09-23T15:24:26.776730+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:24:26.776730+0000","last_undegraded":"2023-09-23T15:24:26.776730+0000","last_fullsized":"2023-09-23T15:24:26.776730+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1514274840,"num_objects":23107,"num_object_clones":0,"num_object_copies":69321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21791,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":23107,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157163,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14054,"num_bytes_recovered":744095744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5618741248,"data_stored":5618729032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108063,"ondisk_log_size":108063,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":112,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201639,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":826280,"kb_used_data":677344,"kb_used_omap":0,"kb_used_meta":148928,"kb_avail":92918872,"statfs":{"total":95995035648,"available":95148924928,"internally_reserved":0,"allocated":693600256,"data_stored":692594735,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":152502272},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":6.6959999999999997,"15min":3.9710000000000001},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":12.096,"5min":226.226,"15min":226.226},"last":35.908000000000001},{"interface":"front","average":{"1min":4.1159999999999997,"5min":7.9660000000000002,"15min":4.6340000000000003},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":13.196,"5min":226.619,"15min":226.619},"last":35.939999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.425,"15min":2.056},"min":{"1min":0.29999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.9589999999999996,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":311.16699999999997},{"interface":"front","average":{"1min":3.2010000000000001,"5min":1.79,"15min":2.5499999999999998},"min":{"1min":0.434,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.865,"5min":11.865,"15min":16.065999999999999},"last":311.279}]},{"osd":2,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.7450000000000001,"15min":3.327},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":7.7889999999999997,"5min":90.227999999999994,"15min":101.807},"last":0.36699999999999999},{"interface":"front","average":{"1min":4.2880000000000003,"5min":3.2400000000000002,"15min":4.0030000000000001},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.127000000000001,"5min":90.197999999999993,"15min":101.84099999999999},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:24:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8319999999999999,"5min":4.5869999999999997,"15min":3.9220000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":41.588999999999999,"5min":146.166,"15min":146.166},"last":0.39500000000000002},{"interface":"front","average":{"1min":5.4000000000000004,"5min":4.8520000000000003,"15min":3.98},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":41.542000000000002,"5min":146.18299999999999,"15min":146.18299999999999},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.7189999999999999,"5min":1.181,"15min":2.5990000000000002},"min":{"1min":0.33700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.6739999999999995,"5min":16.771999999999998,"15min":150.119},"last":0.316},{"interface":"front","average":{"1min":2.302,"5min":0.92300000000000004,"15min":2.7919999999999998},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.298,"5min":9.298,"15min":150.596},"last":0.33700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0089999999999999,"5min":2.8079999999999998,"15min":2.8479999999999999},"min":{"1min":0.36699999999999999,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":10.153,"5min":224.441,"15min":224.441},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.7400000000000002,"5min":3.056,"15min":2.9239999999999999},"min":{"1min":0.32900000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.6600000000000001,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.41899999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:24:30 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.4409999999999998,"15min":2.3010000000000002},"min":{"1min":0.48999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9659999999999993,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":0.436},{"interface":"front","average":{"1min":3.0550000000000002,"5min":2.6230000000000002,"15min":2.5510000000000002},"min":{"1min":0.35699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.135,"5min":10.336,"15min":10.336},"last":0.47099999999999997}]}]},{"osd":6,"up_from":304,"seq":1305670058012,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":397084,"kb_used_data":322900,"kb_used_omap":13,"kb_used_meta":74162,"kb_avail":93348068,"statfs":{"total":95995035648,"available":95588421632,"internally_reserved":0,"allocated":330649600,"data_stored":329656216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":75942243},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":16.183,"5min":4.4199999999999999,"15min":2.46},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":184.53200000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":8.5370000000000008,"5min":4.0090000000000003,"15min":3.2549999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.227999999999994,"5min":86.227999999999994,"15min":86.227999999999994},"last":2.2250000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":9.2279999999999998,"5min":4.0389999999999997,"15min":3.1739999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":86.534000000000006,"5min":86.534000000000006,"15min":86.534000000000006},"last":0.47499999999999998},{"interface":"front","average":{"1min":8.5199999999999996,"5min":3.9929999999999999,"15min":3.2389999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":86.144000000000005,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.36899999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.762,"15min":1.6040000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":2.294},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.99,"15min":1.8140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.1910000000000007,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":0.69999999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.7109999999999999,"15min":2.423},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":41.261000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":1.986},{"interface":"front","average":{"1min":5.1559999999999997,"5min":2.8479999999999999,"15min":2.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":41.408000000000001},"last":2.4380000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.1839999999999999,"15min":1.02},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":1.2310000000000001},{"interface":"front","average":{"1min":2.754,"5min":2.0070000000000001,"15min":1.883},"min":{"1min":0.39800000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.627000000000001,"5min":13.627000000000001,"15min":13.627000000000001},"last":2.262}]},{"osd":5,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.2749999999999999,"15min":1.903},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.74,"5min":13.74,"15min":13.74},"last":1.4279999999999999},{"interface":"front","average":{"1min":3.194,"5min":1.8260000000000001,"15min":1.5980000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":15.868,"5min":15.868,"15min":15.868},"last":1.1990000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.752,"15min":1.5169999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":12.023999999999999,"5min":12.023999999999999,"15min":12.023999999999999},"last":4.133},{"interface":"front","average":{"1min":3.5259999999999998,"5min":2.1629999999999998,"15min":1.9359999999999999},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":12.641999999999999},"last":0.66600000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015313,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1302300,"kb_used_data":837588,"kb_used_omap":6,"kb_used_meta":464697,"kb_avail":92442852,"statfs":{"total":95995035648,"available":94661480448,"internally_reserved":0,"allocated":857690112,"data_stored":856134753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":475849803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":3.3540000000000001,"15min":3.3820000000000001},"min":{"1min":0.23300000000000001,"5min":0.191,"15min":0.191},"max":{"1min":12.298,"5min":60.817999999999998,"15min":60.817999999999998},"last":2.081},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.1550000000000002,"15min":4.165},"min":{"1min":0.36499999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.779,"5min":63.594999999999999,"15min":63.594999999999999},"last":0.42099999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.073,"5min":2.3530000000000002,"15min":4.085},"min":{"1min":0.34699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.9619999999999997,"5min":10.329000000000001,"15min":109.375},"last":4.8200000000000003},{"interface":"front","average":{"1min":1.8580000000000001,"5min":2.2789999999999999,"15min":3.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":12.259,"5min":13.314,"15min":105.16},"last":2.1379999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":4.399,"15min":7.6849999999999996},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":12.218999999999999,"5min":64.540999999999997,"15min":211.405},"last":4.6989999999999998},{"interface":"front","average":{"1min":2.4279999999999999,"5min":4.601,"15min":8.3529999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.551,"5min":67.465000000000003,"15min":210.988},"last":2.367}]},{"osd":4,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.0430000000000001,"5min":3.375,"15min":3.3170000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.379,"5min":80.570999999999998,"15min":80.570999999999998},"last":13.112},{"interface":"front","average":{"1min":2.016,"5min":3.5099999999999998,"15min":3.4220000000000002},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":12.279999999999999,"5min":80.75,"15min":80.75},"last":4.7859999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":9.7769999999999992,"5min":4.0979999999999999,"15min":3.4649999999999999},"min":{"1min":0.33900000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":115.401,"5min":115.401,"15min":115.401},"last":4.9000000000000004},{"interface":"front","average":{"1min":9.9770000000000003,"5min":3.9180000000000001,"15min":3.3359999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":115.169,"5min":115.169,"15min":115.169},"last":13.068}]},{"osd":6,"last update":"Sat Sep 23 15:24:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":3.335,"15min":3.1589999999999998},"min":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":19.553999999999998,"5min":19.553999999999998,"15min":19.553999999999998},"last":4.8609999999999998},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.2869999999999999,"15min":2.3340000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9770000000000003,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":4.8399999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":5.2889999999999997,"5min":2.6520000000000001,"15min":2.948},"min":{"1min":0.55500000000000005,"5min":0.371,"15min":0.371},"max":{"1min":19.702999999999999,"5min":19.702999999999999,"15min":19.702999999999999},"last":4.8920000000000003},{"interface":"front","average":{"1min":3.9550000000000001,"5min":2.4849999999999999,"15min":2.6800000000000002},"min":{"1min":0.5,"5min":0.377,"15min":0.377},"max":{"1min":20.722000000000001,"5min":20.722000000000001,"15min":20.722000000000001},"last":4.7489999999999997}]}]},{"osd":0,"up_from":95,"seq":408021893362,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1469444,"kb_used_data":882812,"kb_used_omap":8,"kb_used_meta":586615,"kb_avail":92275708,"statfs":{"total":95995035648,"available":94490324992,"internally_reserved":0,"allocated":903999488,"data_stored":902577722,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":600694356},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.8519999999999999,"15min":3.468},"min":{"1min":0.28399999999999997,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":17.542000000000002,"5min":48.442999999999998,"15min":64.412999999999997},"last":8.4939999999999998},{"interface":"front","average":{"1min":3.3239999999999998,"5min":2.7250000000000001,"15min":3.387},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":12.058999999999999,"5min":51.304000000000002,"15min":51.304000000000002},"last":6.9080000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":2.4020000000000001,"15min":4.3609999999999998},"min":{"1min":0.23300000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":13.022,"5min":30.510999999999999,"15min":122.20099999999999},"last":1.04},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.7170000000000001,"15min":4.2359999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":9.2050000000000001,"5min":30.396999999999998,"15min":122.071},"last":8.8979999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:23:48 2023","interfaces":[{"interface":"back","average":{"1min":12.212999999999999,"5min":5.2119999999999997,"15min":5.7519999999999998},"min":{"1min":0.39100000000000001,"5min":0.25900000000000001,"15min":0.20000000000000001},"max":{"1min":197.78899999999999,"5min":197.78899999999999,"15min":197.78899999999999},"last":3.2330000000000001},{"interface":"front","average":{"1min":12.901999999999999,"5min":5.6210000000000004,"15min":6.226},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.252},"max":{"1min":200.58799999999999,"5min":200.58799999999999,"15min":200.58799999999999},"last":9.6839999999999993}]},{"osd":4,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.2330000000000001,"5min":2.177,"15min":2.278},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.28000000000000003},"max":{"1min":15.038,"5min":15.358000000000001,"15min":31.341999999999999},"last":6.9409999999999998},{"interface":"front","average":{"1min":2.327,"5min":1.873,"15min":2.2429999999999999},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":8.2629999999999999,"5min":9.8249999999999993,"15min":99.168999999999997},"last":0.93100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3799999999999999,"5min":2.1880000000000002,"15min":2.456},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.28599999999999998},"max":{"1min":10.618,"5min":10.768000000000001,"15min":109.369},"last":8.0850000000000009},{"interface":"front","average":{"1min":3.7959999999999998,"5min":2.3879999999999999,"15min":2.528},"min":{"1min":0.39100000000000001,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":15.363,"5min":15.436999999999999,"15min":109.387},"last":7.4260000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.887,"15min":2.835},"min":{"1min":0.45900000000000002,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":7.0229999999999997},{"interface":"front","average":{"1min":3.4630000000000001,"5min":3.3809999999999998,"15min":3.367},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":15.829000000000001,"5min":17.385000000000002,"15min":17.385000000000002},"last":0.96499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2779999999999996,"5min":2.4159999999999999,"15min":2.6219999999999999},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":17.712,"5min":17.712,"15min":99.549999999999997},"last":8.1310000000000002},{"interface":"front","average":{"1min":3.2629999999999999,"5min":2.556,"15min":2.2610000000000001},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.706,"5min":15.407999999999999,"15min":31.225999999999999},"last":6.9880000000000004}]}]},{"osd":2,"up_from":105,"seq":450971566309,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1562292,"kb_used_data":965484,"kb_used_omap":11,"kb_used_meta":596788,"kb_avail":92182860,"statfs":{"total":95995035648,"available":94395248640,"internally_reserved":0,"allocated":988655616,"data_stored":987243472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":611111696},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":13.015000000000001,"5min":7.2009999999999996,"15min":4.9459999999999997},"min":{"1min":0.214,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":88.622,"5min":88.622,"15min":88.622},"last":1.484},{"interface":"front","average":{"1min":12.861000000000001,"5min":7.21,"15min":4.6269999999999998},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":88.649000000000001,"5min":88.649000000000001,"15min":88.649000000000001},"last":1.5189999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":7.0439999999999996,"5min":4.4900000000000002,"15min":4.7210000000000001},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":71.819000000000003,"5min":71.819000000000003,"15min":84.641000000000005},"last":3.4790000000000001},{"interface":"front","average":{"1min":7.0810000000000004,"5min":4.1989999999999998,"15min":3.6110000000000002},"min":{"1min":0.317,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":71.805000000000007,"5min":71.805000000000007,"15min":84.751999999999995},"last":4.7670000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":8.5559999999999992,"5min":6.1820000000000004,"15min":5.298},"min":{"1min":0.32800000000000001,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":77.031999999999996,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.24399999999999999},{"interface":"front","average":{"1min":8.2710000000000008,"5min":6.5289999999999999,"15min":5.3239999999999998},"min":{"1min":0.313,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":76.742000000000004,"5min":76.742000000000004,"15min":86.158000000000001},"last":3.2109999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.6020000000000001,"15min":2.3969999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":16.712,"5min":16.712,"15min":84.608000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.579,"15min":2.9980000000000002},"min":{"1min":0.376,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":6.3789999999999996,"5min":7.7880000000000003,"15min":87.019999999999996},"last":8.0519999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.839,"15min":2.6480000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.6980000000000004,"5min":8.9909999999999997,"15min":84.498000000000005},"last":3.2669999999999999},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.7210000000000001,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.5099999999999998,"5min":10.808999999999999,"15min":84.664000000000001},"last":3.512}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":2.3290000000000002,"15min":2.2989999999999999},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.4600000000000009,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":3.3109999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.8340000000000001,"15min":1.978},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.6909999999999998,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.9970000000000001,"15min":2.593},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":9.3859999999999992,"5min":10.227,"15min":84.596000000000004},"last":3.375},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.9550000000000001,"15min":2.4910000000000001},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":16.074999999999999,"5min":16.074999999999999,"15min":84.783000000000001},"last":2.8809999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402783,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1275964,"kb_used_data":661620,"kb_used_omap":3,"kb_used_meta":614332,"kb_avail":92469188,"statfs":{"total":95995035648,"available":94688448512,"internally_reserved":0,"allocated":677498880,"data_stored":676079807,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":629076137},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":8.5269999999999992,"5min":3.7440000000000002,"15min":3.6949999999999998},"min":{"1min":0.20599999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":108.318,"5min":108.318,"15min":108.318},"last":0.68500000000000005},{"interface":"front","average":{"1min":9.3879999999999999,"5min":4.1669999999999998,"15min":3.9740000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":108.889,"5min":108.889,"15min":108.889},"last":4.9980000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.8179999999999996,"15min":4.8339999999999996},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":9.2390000000000008,"5min":126.875,"15min":126.875},"last":4.6550000000000002},{"interface":"front","average":{"1min":2.1400000000000001,"5min":5.5970000000000004,"15min":5.0519999999999996},"min":{"1min":0.28599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.805999999999999,"5min":128.249,"15min":128.249},"last":0.55700000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.2349999999999999,"15min":4.7539999999999996},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":12.210000000000001,"5min":17.199999999999999,"15min":109.077},"last":0.27800000000000002},{"interface":"front","average":{"1min":3.5670000000000002,"5min":3.3410000000000002,"15min":5.7969999999999997},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.763,"5min":36.582000000000001,"15min":119.72799999999999},"last":4.9749999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":2.1429999999999998,"15min":2.2669999999999999},"min":{"1min":0.35299999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.417999999999999,"5min":36.551000000000002,"15min":54.051000000000002},"last":4.9219999999999997},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.5660000000000001,"15min":2.121},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":8.0050000000000008,"5min":12.500999999999999,"15min":52.228999999999999},"last":4.758}]},{"osd":5,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":3.7330000000000001,"15min":2.9249999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":11.372999999999999,"5min":148.619,"15min":148.619},"last":4.8940000000000001},{"interface":"front","average":{"1min":1.909,"5min":2.9780000000000002,"15min":2.7789999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":6.8520000000000003,"5min":148.56999999999999,"15min":148.56999999999999},"last":4.702}]},{"osd":6,"last update":"Sat Sep 23 15:24:28 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.226,"15min":2.2250000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5460000000000003,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":4.7290000000000001},{"interface":"front","average":{"1min":2.0459999999999998,"5min":3.3300000000000001,"15min":3.544},"min":{"1min":0.42099999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":6.7699999999999996,"5min":36.517000000000003,"15min":36.517000000000003},"last":4.9450000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.9279999999999999,"15min":2.681},"min":{"1min":0.41799999999999998,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":4.6699999999999999,"5min":145.25,"15min":145.25},"last":4.8289999999999997},{"interface":"front","average":{"1min":1.877,"5min":3.121,"15min":2.4990000000000001},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":7.1829999999999998,"5min":145.00899999999999,"15min":145.00899999999999},"last":2.4470000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986548,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":812384,"kb_used_data":478104,"kb_used_omap":0,"kb_used_meta":334272,"kb_avail":92932768,"statfs":{"total":95995035648,"available":95163154432,"internally_reserved":0,"allocated":489578496,"data_stored":488581073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":342294528},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":5.2699999999999996,"5min":5.1219999999999999,"15min":4.2460000000000004},"min":{"1min":0.28299999999999997,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":53.771000000000001,"5min":162.75999999999999,"15min":162.75999999999999},"last":2.8999999999999999},{"interface":"front","average":{"1min":4.0990000000000002,"5min":4.8369999999999997,"15min":4.1959999999999997},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":53.145000000000003,"5min":162.34800000000001,"15min":162.34800000000001},"last":10.923999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":5.1509999999999998,"5min":4.1280000000000001,"15min":3.4590000000000001},"min":{"1min":0.254,"5min":0.249,"15min":0.245},"max":{"1min":60.122,"5min":113.11199999999999,"15min":113.11199999999999},"last":6.7160000000000002},{"interface":"front","average":{"1min":5.5640000000000001,"5min":4.1980000000000004,"15min":3.8839999999999999},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":60.802,"5min":113.02,"15min":113.02},"last":1.3300000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":3.3599999999999999,"15min":4.3040000000000003},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":4.8010000000000002,"5min":61.646000000000001,"15min":148.08000000000001},"last":6.2759999999999998},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.536,"15min":4.5309999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.25700000000000001},"max":{"1min":6.1980000000000004,"5min":64.840000000000003,"15min":147.88900000000001},"last":6.6820000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":6.3780000000000001,"15min":5.9969999999999999},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":6.5060000000000002,"5min":156.46000000000001,"15min":163.97499999999999},"last":2.9319999999999999},{"interface":"front","average":{"1min":2.1360000000000001,"5min":6.5069999999999997,"15min":5.9400000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27800000000000002},"max":{"1min":8.3829999999999991,"5min":156.74299999999999,"15min":164.447},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.069,"15min":1.6279999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":6.4089999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":2.9649999999999999},{"interface":"front","average":{"1min":1.458,"5min":1.2789999999999999,"15min":1.573},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":5.7199999999999998,"5min":14.449999999999999,"15min":88.602999999999994},"last":2.3149999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.446,"15min":1.446},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.8010000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.53500000000000003},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.121,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.5380000000000003,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":3.5059999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:23:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6990000000000001,"5min":0.92500000000000004,"15min":1.1890000000000001},"min":{"1min":0.42599999999999999,"5min":0.31,"15min":0.28199999999999997},"max":{"1min":5.7599999999999998,"5min":6.0730000000000004,"15min":18.23},"last":3.5430000000000001},{"interface":"front","average":{"1min":1.819,"5min":1.034,"15min":1.286},"min":{"1min":0.379,"5min":0.32700000000000001,"15min":0.314},"max":{"1min":4.2229999999999999,"5min":5.3520000000000003,"15min":14.653},"last":3.0310000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724911,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1152984,"kb_used_data":765264,"kb_used_omap":0,"kb_used_meta":387712,"kb_avail":92592168,"statfs":{"total":95995035648,"available":94814380032,"internally_reserved":0,"allocated":783630336,"data_stored":782637366,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":397017088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":4.1900000000000004,"15min":3.3079999999999998},"min":{"1min":0.251,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.178999999999998,"5min":94.372,"15min":124.31},"last":1.579},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.0720000000000001,"15min":3.3769999999999998},"min":{"1min":0.44400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.106000000000002,"5min":94.396000000000001,"15min":124.374},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":10.42,"5min":4.4420000000000002,"15min":3.1709999999999998},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":86.900999999999996,"5min":86.900999999999996,"15min":124.38800000000001},"last":1.3049999999999999},{"interface":"front","average":{"1min":5.2599999999999998,"5min":3.6110000000000002,"15min":3.0910000000000002},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":40.939,"5min":66.825000000000003,"15min":124.29000000000001},"last":0.58099999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":2.444,"15min":3.7770000000000001},"min":{"1min":0.35899999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":11.754,"5min":74.019000000000005,"15min":193.00999999999999},"last":0.77300000000000002},{"interface":"front","average":{"1min":2.8900000000000001,"5min":2.5489999999999999,"15min":3.8239999999999998},"min":{"1min":0.37,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":10.542999999999999,"5min":73.712000000000003,"15min":193.15700000000001},"last":0.72999999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":16.594999999999999,"5min":7.7089999999999996,"15min":6.0309999999999997},"min":{"1min":0.36399999999999999,"5min":0.29199999999999998,"15min":0.27500000000000002},"max":{"1min":266.70299999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":0.84699999999999998},{"interface":"front","average":{"1min":20.937000000000001,"5min":8.5090000000000003,"15min":6.0449999999999999},"min":{"1min":0.318,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":266.80399999999997,"5min":266.80399999999997,"15min":266.80399999999997},"last":1.28}]},{"osd":4,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":1.292,"15min":1.9470000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":10.749000000000001,"5min":10.749000000000001,"15min":138.68600000000001},"last":1.246},{"interface":"front","average":{"1min":3.222,"5min":1.3009999999999999,"15min":2.0449999999999999},"min":{"1min":0.41699999999999998,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":10.516,"5min":10.516,"15min":138.49600000000001},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.4390000000000001,"15min":2.411},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":10.577999999999999,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.75800000000000001},{"interface":"front","average":{"1min":3.5249999999999999,"5min":2.633,"15min":2.484},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.257999999999999,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.874}]},{"osd":7,"last update":"Sat Sep 23 15:23:47 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.131,"15min":1.407},"min":{"1min":0.40200000000000002,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":12.065,"5min":12.065,"15min":15.301},"last":1.6220000000000001},{"interface":"front","average":{"1min":2.9809999999999999,"5min":1.0920000000000001,"15min":1.351},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":7.3810000000000002,"5min":7.3810000000000002,"15min":15.762},"last":1.5489999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":884932608,"data_stored":884932608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":841351168,"data_stored":841351168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":982388736,"data_stored":982384664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":660340736,"data_stored":660340736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":476053504,"data_stored":476053504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":762646528,"data_stored":762642456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":328204288,"data_stored":328204288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":682823680,"data_stored":682819608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:24:45.466 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:45.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:45.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:45.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:45.600 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:45.648 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:24:45.648 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:24:45.677 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:45.778 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:45.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:45.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:45.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:45.974 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:46.183 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:46.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:46.245 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:24:46.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:24:46.303 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:24:46.341 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:24:46.392 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:24:46.394 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:46.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:46 smithi017 ceph-mon[215646]: pgmap v969: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 21532/66315 objects degraded (32.469%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:46.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:46 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1024396163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:46.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:46 smithi017 ceph-mon[222878]: pgmap v969: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 21532/66315 objects degraded (32.469%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:46.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:46 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1024396163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:46.455 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:24:46.470 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:46.513 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:46.557 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:46.578 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:24:46.586 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:46.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:46.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:24:46.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:46 smithi027 ceph-mon[195233]: pgmap v969: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.4 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 642 op/s; 21532/66315 objects degraded (32.469%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:46.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:46 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1024396163' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.612 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:24:46.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:46.747 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:46.822 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:46.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:46.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:46.982 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:47.102 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:47.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:47.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:47.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:47.283 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21532/66315 objects degraded (32.469%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:47.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21532/66315 objects degraded (32.469%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:47.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21532/66315 objects degraded (32.469%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:48.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:48 smithi017 ceph-mon[215646]: pgmap v970: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 21532/62427 objects degraded (34.491%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:48.339 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:48] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-09-23T15:24:48.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:24:48.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:48 smithi027 ceph-mon[195233]: pgmap v970: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 21532/62427 objects degraded (34.491%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:48.651 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:48.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:48.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:48.659 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:48 smithi017 ceph-mon[222878]: pgmap v970: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.3 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 624 op/s; 21532/62427 objects degraded (34.491%); 2.4 MiB/s, 91 objects/s recovering 2023-09-23T15:24:48.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:48.811 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:48.886 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:48.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:48.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:48.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:49.063 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:49.174 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:49.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:49.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:49.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:49.344 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:49.387 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:49.441 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:49.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:49.546 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:49.594 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:49.616 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:49.663 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:49.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:49.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:49.772 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:24:49.806 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:49.875 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:24:49.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:49.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:24:49.983 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:50.046 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:50.140 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:50.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:50.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:50.251 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:50.309 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:50.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[215646]: pgmap v971: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 21349/60072 objects degraded (35.539%); 2.3 MiB/s, 90 objects/s recovering 2023-09-23T15:24:50.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/14006857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.384 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.391 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:50.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:50.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:50.410 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:24:50.497 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:50.525 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:50 smithi027 ceph-mon[195233]: pgmap v971: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 21349/60072 objects degraded (35.539%); 2.3 MiB/s, 90 objects/s recovering 2023-09-23T15:24:50.526 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/14006857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.526 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:50 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.526 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:50.571 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:50.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:50.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:50.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:50.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[222878]: pgmap v971: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.2 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 616 op/s; 21349/60072 objects degraded (35.539%); 2.3 MiB/s, 90 objects/s recovering 2023-09-23T15:24:50.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/14006857' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:50 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:24:50.713 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:50.796 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:50.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:50.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:50.911 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:24:50.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:24:50.960 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:51.064 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:24:51.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:51.075 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:24:51.175 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:51.222 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:51.311 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:51.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:51.418 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:24:51.444 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:51.518 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:24:51.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:51.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:24:51.625 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[215646]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[215646]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[215646]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[222878]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[222878]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:51.675 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:51 smithi017 ceph-mon[222878]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T15:24:51.679 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:51 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:24:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:51 smithi027 ceph-mon[195233]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:51 smithi027 ceph-mon[195233]: failed to encode map e330 with expected crc 2023-09-23T15:24:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:24:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:51 smithi027 ceph-mon[195233]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T15:24:52.393 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:52 smithi017 ceph-mon[215646]: pgmap v973: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 21029/52713 objects degraded (39.893%); 2.3 MiB/s, 95 objects/s recovering 2023-09-23T15:24:52.393 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21029/52713 objects degraded (39.893%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:52.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:52 smithi017 ceph-mon[222878]: pgmap v973: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 21029/52713 objects degraded (39.893%); 2.3 MiB/s, 95 objects/s recovering 2023-09-23T15:24:52.394 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21029/52713 objects degraded (39.893%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:52.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:52 smithi027 ceph-mon[195233]: pgmap v973: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.1 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 628 op/s; 21029/52713 objects degraded (39.893%); 2.3 MiB/s, 95 objects/s recovering 2023-09-23T15:24:52.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21029/52713 objects degraded (39.893%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:54.130 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:54.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:54.241 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:54.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:54.361 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:54.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:54 smithi017 ceph-mon[215646]: pgmap v974: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 20836/50202 objects degraded (41.504%); 2.2 MiB/s, 95 objects/s recovering 2023-09-23T15:24:54.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:54 smithi017 ceph-mon[222878]: pgmap v974: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 20836/50202 objects degraded (41.504%); 2.2 MiB/s, 95 objects/s recovering 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:166462 8.9 8:93b47e53:::benchmark_data_smithi017_485823_object76325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.775737+0000", 2023-09-23T15:24:54.782 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0059694250000000004, 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060026339999999997, 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166462 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.783 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775737+0000", 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775737+0000", 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.784 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775735+0000", 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775741+0000", 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9089999999999997e-06 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775750+0000", 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1120000000000002e-06 2023-09-23T15:24:54.785 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775756+0000", 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6910000000000002e-06 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775921+0000", 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016493899999999999 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:24:54.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.776065+0000", 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014435899999999999 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.776200+0000", 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013479799999999999 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.787 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:166464 8.9 8:97afc077:::benchmark_data_smithi017_485823_object76327:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.775925+0000", 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0057814479999999998, 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059117090000000002, 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166464 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775925+0000", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775925+0000", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775924+0000", 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:24:54.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775930+0000", 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9120000000000003e-06 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775942+0000", 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.189e-05 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.775949+0000", 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6820000000000003e-06 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780855+0000", 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049056710000000003 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781007+0000", 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015188499999999999 2023-09-23T15:24:54.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781155+0000", 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014819 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45809.0:166462 8.9 e330/305)", 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.780624+0000", 2023-09-23T15:24:54.792 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.001082574, 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012576130000000001, 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780624+0000", 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780624+0000", 2023-09-23T15:24:54.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780623+0000", 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780633+0000", 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0471e-05 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780643+0000", 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5489999999999995e-06 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.780650+0000", 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3350000000000001e-06 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:166465 8.9 8.d2c62689 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.781466+0000", 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.000240189, 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046667399999999998, 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166465 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.796 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781466+0000", 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781466+0000", 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781465+0000", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781472+0000", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3919999999999997e-06 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781721+0000", 2023-09-23T15:24:54.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00024897800000000001 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781730+0000", 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9670000000000007e-06 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:166467 8.9 8.4d751b19 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e330)", 2023-09-23T15:24:54.799 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.781738+0000", 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9999681, 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023080900000000001, 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 166467 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781738+0000", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781738+0000", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781737+0000", 2023-09-23T15:24:54.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781742+0000", 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6779999999999999e-06 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781750+0000", 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7020000000000001e-06 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781755+0000", 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3879999999999999e-06 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45809.0:166462 8.9 e330/305)", 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:54.781779+0000", 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.999927, 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022542300000000001, 2023-09-23T15:24:54.803 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781779+0000", 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781779+0000", 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:24:54.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781778+0000", 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781784+0000", 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3820000000000003e-06 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781796+0000", 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1929e-05 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:54.781803+0000", 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5310000000000004e-06 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:24:54.806 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:24:54.807 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 6 2023-09-23T15:24:54.807 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:24:54.828 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:54.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:54.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:54 smithi027 ceph-mon[195233]: pgmap v974: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 1.0 GiB data, 7.3 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 20836/50202 objects degraded (41.504%); 2.2 MiB/s, 95 objects/s recovering 2023-09-23T15:24:54.943 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:24:55.041 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:55.436 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:24:55.446 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:24:55.708 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:24:55.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:55.717 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:24:55.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:55.892 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:55.920 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:24:56.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:56 smithi017 ceph-mon[215646]: pgmap v975: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 986 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 20516/47211 objects degraded (43.456%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:56.417 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:56 smithi017 ceph-mon[215646]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:56.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:56 smithi017 ceph-mon[222878]: pgmap v975: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 986 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 20516/47211 objects degraded (43.456%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:56.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:56 smithi017 ceph-mon[222878]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:56.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:56 smithi027 ceph-mon[195233]: pgmap v975: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 986 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 636 op/s; 20516/47211 objects degraded (43.456%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:56.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:56 smithi027 ceph-mon[195233]: from='client.46067 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:24:57.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 20516/47211 objects degraded (43.456%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:57.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 20516/47211 objects degraded (43.456%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:57.765 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:57.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:57.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:24:57.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 20516/47211 objects degraded (43.456%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:24:57.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:24:57.981 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:58.427 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:24:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:24:58] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-09-23T15:24:58.448 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:24:58.651 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:24:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:58.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:24:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:58 smithi017 ceph-mon[222878]: pgmap v976: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 895 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 20516/42816 objects degraded (47.917%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:24:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:58.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:58 smithi017 ceph-mon[215646]: pgmap v976: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 895 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 20516/42816 objects degraded (47.917%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:58.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:24:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:58.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:58.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:58 smithi027 ceph-mon[195233]: pgmap v976: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 895 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 20516/42816 objects degraded (47.917%); 2.2 MiB/s, 101 objects/s recovering 2023-09-23T15:24:58.792 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:24:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:24:58.796 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:58.807 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":975,"stamp":"2023-09-23T15:24:54.754782+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1034283377,"num_objects":15737,"num_object_clones":0,"num_object_copies":47211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15737,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":165490,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15474,"num_bytes_recovered":776113525,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113474,"ondisk_log_size":113474,"up":327,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":342,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7367648,"kb_used_data":4188512,"kb_used_omap":44,"kb_used_meta":3178835,"kb_avail":742593568,"statfs":{"total":767960285184,"available":760415813632,"internally_reserved":0,"allocated":4289036288,"data_stored":4279188247,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3255127995},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,4,4,5,5,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-417333248,"num_objects":-6368,"num_object_clones":0,"num_object_copies":-19104,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1016,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6368,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6368,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1016,"num_bytes_recovered":22937600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003230"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760002+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:24:27.760002+0000","last_peered":"2023-09-23T15:24:27.760002+0000","last_clean":"2023-09-23T15:24:27.760002+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:24:27.760002+0000","last_undegraded":"2023-09-23T15:24:27.760002+0000","last_fullsized":"2023-09-23T15:24:27.760002+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":286,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293537+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:24:40.293537+0000","last_peered":"2023-09-23T15:24:40.293537+0000","last_clean":"2023-09-23T15:24:40.293537+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:24:40.293537+0000","last_undegraded":"2023-09-23T15:24:40.293537+0000","last_fullsized":"2023-09-23T15:24:40.293537+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":235,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380207+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:24:50.380207+0000","last_peered":"2023-09-23T15:24:50.380207+0000","last_clean":"2023-09-23T15:24:50.380207+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:24:50.380207+0000","last_undegraded":"2023-09-23T15:24:50.380207+0000","last_fullsized":"2023-09-23T15:24:50.380207+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":309,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:44.233361+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:24:44.233361+0000","last_peered":"2023-09-23T15:24:44.233361+0000","last_clean":"2023-09-23T15:24:44.233361+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:24:44.233361+0000","last_undegraded":"2023-09-23T15:24:44.233361+0000","last_fullsized":"2023-09-23T15:24:44.233361+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":331,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.296127+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:24:40.296127+0000","last_peered":"2023-09-23T15:24:40.296127+0000","last_clean":"2023-09-23T15:24:40.296127+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:24:40.296127+0000","last_undegraded":"2023-09-23T15:24:40.296127+0000","last_fullsized":"2023-09-23T15:24:40.296127+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":280,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.262112+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:24:40.262112+0000","last_peered":"2023-09-23T15:24:40.262112+0000","last_clean":"2023-09-23T15:24:40.262112+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:24:40.262112+0000","last_undegraded":"2023-09-23T15:24:40.262112+0000","last_fullsized":"2023-09-23T15:24:40.262112+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":278,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.270921+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:24:40.270921+0000","last_peered":"2023-09-23T15:24:40.270921+0000","last_clean":"2023-09-23T15:24:40.270921+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:24:40.270921+0000","last_undegraded":"2023-09-23T15:24:40.270921+0000","last_fullsized":"2023-09-23T15:24:40.270921+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":328,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.296240+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:24:40.296240+0000","last_peered":"2023-09-23T15:24:40.296240+0000","last_clean":"2023-09-23T15:24:40.296240+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:24:40.296240+0000","last_undegraded":"2023-09-23T15:24:40.296240+0000","last_fullsized":"2023-09-23T15:24:40.296240+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":688,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.283196+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:24:40.283196+0000","last_peered":"2023-09-23T15:24:40.283196+0000","last_clean":"2023-09-23T15:24:40.283196+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:24:40.283196+0000","last_undegraded":"2023-09-23T15:24:40.283196+0000","last_fullsized":"2023-09-23T15:24:40.283196+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":329,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761818+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:24:27.761818+0000","last_peered":"2023-09-23T15:24:27.761818+0000","last_clean":"2023-09-23T15:24:27.761818+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:24:27.761818+0000","last_undegraded":"2023-09-23T15:24:27.761818+0000","last_fullsized":"2023-09-23T15:24:27.761818+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":282,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.292996+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:24:40.292996+0000","last_peered":"2023-09-23T15:24:40.292996+0000","last_clean":"2023-09-23T15:24:40.292996+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:24:40.292996+0000","last_undegraded":"2023-09-23T15:24:40.292996+0000","last_fullsized":"2023-09-23T15:24:40.292996+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":74,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263614+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:24:40.263614+0000","last_peered":"2023-09-23T15:24:40.263614+0000","last_clean":"2023-09-23T15:24:40.263614+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:24:40.263614+0000","last_undegraded":"2023-09-23T15:24:40.263614+0000","last_fullsized":"2023-09-23T15:24:40.263614+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":221,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.285156+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:24:40.285156+0000","last_peered":"2023-09-23T15:24:40.285156+0000","last_clean":"2023-09-23T15:24:40.285156+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:24:40.285156+0000","last_undegraded":"2023-09-23T15:24:40.285156+0000","last_fullsized":"2023-09-23T15:24:40.285156+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":63,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760452+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:24:27.760452+0000","last_peered":"2023-09-23T15:24:27.760452+0000","last_clean":"2023-09-23T15:24:27.760452+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:24:27.760452+0000","last_undegraded":"2023-09-23T15:24:27.760452+0000","last_fullsized":"2023-09-23T15:24:27.760452+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761176+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:24:27.761176+0000","last_peered":"2023-09-23T15:24:27.761176+0000","last_clean":"2023-09-23T15:24:27.761176+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:24:27.761176+0000","last_undegraded":"2023-09-23T15:24:27.761176+0000","last_fullsized":"2023-09-23T15:24:27.761176+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":639,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.274768+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:24:40.274768+0000","last_peered":"2023-09-23T15:24:40.274768+0000","last_clean":"2023-09-23T15:24:40.274768+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:24:40.274768+0000","last_undegraded":"2023-09-23T15:24:40.274768+0000","last_fullsized":"2023-09-23T15:24:40.274768+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.267791+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:24:40.267791+0000","last_peered":"2023-09-23T15:24:40.267791+0000","last_clean":"2023-09-23T15:24:40.267791+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:24:40.267791+0000","last_undegraded":"2023-09-23T15:24:40.267791+0000","last_fullsized":"2023-09-23T15:24:40.267791+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":333,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293888+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:24:40.293888+0000","last_peered":"2023-09-23T15:24:40.293888+0000","last_clean":"2023-09-23T15:24:40.293888+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:24:40.293888+0000","last_undegraded":"2023-09-23T15:24:40.293888+0000","last_fullsized":"2023-09-23T15:24:40.293888+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":687,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761651+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:24:27.761651+0000","last_peered":"2023-09-23T15:24:27.761651+0000","last_clean":"2023-09-23T15:24:27.761651+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:24:27.761651+0000","last_undegraded":"2023-09-23T15:24:27.761651+0000","last_fullsized":"2023-09-23T15:24:27.761651+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":307,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.273934+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:24:40.273934+0000","last_peered":"2023-09-23T15:24:40.273934+0000","last_clean":"2023-09-23T15:24:40.273934+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:24:40.273934+0000","last_undegraded":"2023-09-23T15:24:40.273934+0000","last_fullsized":"2023-09-23T15:24:40.273934+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":314,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762223+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:24:27.762223+0000","last_peered":"2023-09-23T15:24:27.762223+0000","last_clean":"2023-09-23T15:24:27.762223+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:24:27.762223+0000","last_undegraded":"2023-09-23T15:24:27.762223+0000","last_fullsized":"2023-09-23T15:24:27.762223+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":620,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.285918+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:24:40.285918+0000","last_peered":"2023-09-23T15:24:40.285918+0000","last_clean":"2023-09-23T15:24:40.285918+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:24:40.285918+0000","last_undegraded":"2023-09-23T15:24:40.285918+0000","last_fullsized":"2023-09-23T15:24:40.285918+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":78,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263267+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:24:40.263267+0000","last_peered":"2023-09-23T15:24:40.263267+0000","last_clean":"2023-09-23T15:24:40.263267+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:24:40.263267+0000","last_undegraded":"2023-09-23T15:24:40.263267+0000","last_fullsized":"2023-09-23T15:24:40.263267+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":296,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.274524+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:24:40.274524+0000","last_peered":"2023-09-23T15:24:40.274524+0000","last_clean":"2023-09-23T15:24:40.274524+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:24:40.274524+0000","last_undegraded":"2023-09-23T15:24:40.274524+0000","last_fullsized":"2023-09-23T15:24:40.274524+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":645,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762103+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:24:27.762103+0000","last_peered":"2023-09-23T15:24:27.762103+0000","last_clean":"2023-09-23T15:24:27.762103+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:24:27.762103+0000","last_undegraded":"2023-09-23T15:24:27.762103+0000","last_fullsized":"2023-09-23T15:24:27.762103+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":266,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.261740+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:24:40.261740+0000","last_peered":"2023-09-23T15:24:40.261740+0000","last_clean":"2023-09-23T15:24:40.261740+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:24:40.261740+0000","last_undegraded":"2023-09-23T15:24:40.261740+0000","last_fullsized":"2023-09-23T15:24:40.261740+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":333,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.294326+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:24:40.294326+0000","last_peered":"2023-09-23T15:24:40.294326+0000","last_clean":"2023-09-23T15:24:40.294326+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:24:40.294326+0000","last_undegraded":"2023-09-23T15:24:40.294326+0000","last_fullsized":"2023-09-23T15:24:40.294326+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"329'10346","reported_seq":16902,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:49.746850+0000","last_change":"2023-09-23T15:24:27.826512+0000","last_active":"2023-09-23T15:24:49.746850+0000","last_peered":"2023-09-23T15:24:49.746850+0000","last_clean":"2023-09-23T15:24:49.746850+0000","last_became_active":"2023-09-23T15:24:27.764262+0000","last_became_peered":"2023-09-23T15:24:27.764262+0000","last_unstale":"2023-09-23T15:24:49.746850+0000","last_undegraded":"2023-09-23T15:24:49.746850+0000","last_fullsized":"2023-09-23T15:24:49.746850+0000","mapping_epoch":327,"log_start":"294'2500","ondisk_log_start":"294'2500","created":284,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7846,"log_dups_size":0,"ondisk_log_size":7846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:39.059424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69992448,"num_objects":1068,"num_object_clones":0,"num_object_copies":3204,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1068,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10346,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":87425024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":592,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263855+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:24:40.263855+0000","last_peered":"2023-09-23T15:24:40.263855+0000","last_clean":"2023-09-23T15:24:40.263855+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:24:40.263855+0000","last_undegraded":"2023-09-23T15:24:40.263855+0000","last_fullsized":"2023-09-23T15:24:40.263855+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":70,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.268843+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:24:40.268843+0000","last_peered":"2023-09-23T15:24:40.268843+0000","last_clean":"2023-09-23T15:24:40.268843+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:24:40.268843+0000","last_undegraded":"2023-09-23T15:24:40.268843+0000","last_fullsized":"2023-09-23T15:24:40.268843+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":321,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761480+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:24:27.761480+0000","last_peered":"2023-09-23T15:24:27.761480+0000","last_clean":"2023-09-23T15:24:27.761480+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:24:27.761480+0000","last_undegraded":"2023-09-23T15:24:27.761480+0000","last_fullsized":"2023-09-23T15:24:27.761480+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"330'10198","reported_seq":19462,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:54.373710+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:24:54.373710+0000","last_peered":"2023-09-23T15:24:54.373710+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:24:54.373710+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"300'3300","ondisk_log_start":"300'3300","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6898,"log_dups_size":0,"ondisk_log_size":6898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":57409536,"num_objects":876,"num_object_clones":0,"num_object_copies":2628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1484,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10198,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1","4"],"object_location_counts":[{"shards":"4,7","objects":-608}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":650,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293817+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:24:40.293817+0000","last_peered":"2023-09-23T15:24:40.293817+0000","last_clean":"2023-09-23T15:24:40.293817+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:24:40.293817+0000","last_undegraded":"2023-09-23T15:24:40.293817+0000","last_fullsized":"2023-09-23T15:24:40.293817+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":70,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263927+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:24:40.263927+0000","last_peered":"2023-09-23T15:24:40.263927+0000","last_clean":"2023-09-23T15:24:40.263927+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:24:40.263927+0000","last_undegraded":"2023-09-23T15:24:40.263927+0000","last_fullsized":"2023-09-23T15:24:40.263927+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760057+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:24:27.760057+0000","last_peered":"2023-09-23T15:24:27.760057+0000","last_clean":"2023-09-23T15:24:27.760057+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:24:27.760057+0000","last_undegraded":"2023-09-23T15:24:27.760057+0000","last_fullsized":"2023-09-23T15:24:27.760057+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"330'10356","reported_seq":10431,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:54.208028+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:24:54.208028+0000","last_peered":"2023-09-23T15:24:54.208028+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:24:54.208028+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"301'3800","ondisk_log_start":"301'3800","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6556,"log_dups_size":0,"ondisk_log_size":6556,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56229888,"num_objects":858,"num_object_clones":0,"num_object_copies":2574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10356,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":-3772}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":537,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.262276+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:24:40.262276+0000","last_peered":"2023-09-23T15:24:40.262276+0000","last_clean":"2023-09-23T15:24:40.262276+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:24:40.262276+0000","last_undegraded":"2023-09-23T15:24:40.262276+0000","last_fullsized":"2023-09-23T15:24:40.262276+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":70,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263513+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:24:40.263513+0000","last_peered":"2023-09-23T15:24:40.263513+0000","last_clean":"2023-09-23T15:24:40.263513+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:24:40.263513+0000","last_undegraded":"2023-09-23T15:24:40.263513+0000","last_fullsized":"2023-09-23T15:24:40.263513+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":293,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.261919+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:24:40.261919+0000","last_peered":"2023-09-23T15:24:40.261919+0000","last_clean":"2023-09-23T15:24:40.261919+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:24:40.261919+0000","last_undegraded":"2023-09-23T15:24:40.261919+0000","last_fullsized":"2023-09-23T15:24:40.261919+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"330'10337","reported_seq":12358,"reported_epoch":330,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:52.037811+0000","last_change":"2023-09-23T15:24:25.988237+0000","last_active":"2023-09-23T15:24:52.037811+0000","last_peered":"2023-09-23T15:24:52.037811+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:24:52.037811+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"301'3800","ondisk_log_start":"301'3800","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6537,"log_dups_size":0,"ondisk_log_size":6537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63897600,"num_objects":975,"num_object_clones":0,"num_object_copies":2925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3676,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":975,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10337,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":980,"num_bytes_recovered":62521344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-2701}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":544,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761264+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:24:27.761264+0000","last_peered":"2023-09-23T15:24:27.761264+0000","last_clean":"2023-09-23T15:24:27.761264+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:24:27.761264+0000","last_undegraded":"2023-09-23T15:24:27.761264+0000","last_fullsized":"2023-09-23T15:24:27.761264+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":305,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762117+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:24:27.762117+0000","last_peered":"2023-09-23T15:24:27.762117+0000","last_clean":"2023-09-23T15:24:27.762117+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:24:27.762117+0000","last_undegraded":"2023-09-23T15:24:27.762117+0000","last_fullsized":"2023-09-23T15:24:27.762117+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":312,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.377142+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:24:50.377142+0000","last_peered":"2023-09-23T15:24:50.377142+0000","last_clean":"2023-09-23T15:24:50.377142+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:24:50.377142+0000","last_undegraded":"2023-09-23T15:24:50.377142+0000","last_fullsized":"2023-09-23T15:24:50.377142+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"330'10260","reported_seq":10316,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.512211+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:24:50.512211+0000","last_peered":"2023-09-23T15:24:50.512211+0000","last_clean":"2023-09-23T15:24:50.512211+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:24:50.512211+0000","last_undegraded":"2023-09-23T15:24:50.512211+0000","last_fullsized":"2023-09-23T15:24:50.512211+0000","mapping_epoch":284,"log_start":"300'3700","ondisk_log_start":"300'3700","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10260,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":566,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263492+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:24:40.263492+0000","last_peered":"2023-09-23T15:24:40.263492+0000","last_clean":"2023-09-23T15:24:40.263492+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:24:40.263492+0000","last_undegraded":"2023-09-23T15:24:40.263492+0000","last_fullsized":"2023-09-23T15:24:40.263492+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":323,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293250+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:24:40.293250+0000","last_peered":"2023-09-23T15:24:40.293250+0000","last_clean":"2023-09-23T15:24:40.293250+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:24:40.293250+0000","last_undegraded":"2023-09-23T15:24:40.293250+0000","last_fullsized":"2023-09-23T15:24:40.293250+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760399+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:24:27.760399+0000","last_peered":"2023-09-23T15:24:27.760399+0000","last_clean":"2023-09-23T15:24:27.760399+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:24:27.760399+0000","last_undegraded":"2023-09-23T15:24:27.760399+0000","last_fullsized":"2023-09-23T15:24:27.760399+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"330'10532","reported_seq":10587,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:52.021921+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:24:52.021921+0000","last_peered":"2023-09-23T15:24:52.021921+0000","last_clean":"2023-09-23T15:24:52.021921+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:24:52.021921+0000","last_undegraded":"2023-09-23T15:24:52.021921+0000","last_fullsized":"2023-09-23T15:24:52.021921+0000","mapping_epoch":284,"log_start":"301'4000","ondisk_log_start":"301'4000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66191360,"num_objects":1010,"num_object_clones":0,"num_object_copies":3030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10532,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":633,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761672+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:24:27.761672+0000","last_peered":"2023-09-23T15:24:27.761672+0000","last_clean":"2023-09-23T15:24:27.761672+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:24:27.761672+0000","last_undegraded":"2023-09-23T15:24:27.761672+0000","last_fullsized":"2023-09-23T15:24:27.761672+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":277,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.262005+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:24:40.262005+0000","last_peered":"2023-09-23T15:24:40.262005+0000","last_clean":"2023-09-23T15:24:40.262005+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:24:40.262005+0000","last_undegraded":"2023-09-23T15:24:40.262005+0000","last_fullsized":"2023-09-23T15:24:40.262005+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760736+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:24:27.760736+0000","last_peered":"2023-09-23T15:24:27.760736+0000","last_clean":"2023-09-23T15:24:27.760736+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:24:27.760736+0000","last_undegraded":"2023-09-23T15:24:27.760736+0000","last_fullsized":"2023-09-23T15:24:27.760736+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"329'10369","reported_seq":16921,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:49.732658+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:24:49.732658+0000","last_peered":"2023-09-23T15:24:49.732658+0000","last_clean":"2023-09-23T15:24:49.732658+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:24:49.732658+0000","last_undegraded":"2023-09-23T15:24:49.732658+0000","last_fullsized":"2023-09-23T15:24:49.732658+0000","mapping_epoch":310,"log_start":"300'3800","ondisk_log_start":"300'3800","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6569,"log_dups_size":0,"ondisk_log_size":6569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10369,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":586,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.285836+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:24:40.285836+0000","last_peered":"2023-09-23T15:24:40.285836+0000","last_clean":"2023-09-23T15:24:40.285836+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:24:40.285836+0000","last_undegraded":"2023-09-23T15:24:40.285836+0000","last_fullsized":"2023-09-23T15:24:40.285836+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760844+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:24:27.760844+0000","last_peered":"2023-09-23T15:24:27.760844+0000","last_clean":"2023-09-23T15:24:27.760844+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:24:27.760844+0000","last_undegraded":"2023-09-23T15:24:27.760844+0000","last_fullsized":"2023-09-23T15:24:27.760844+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761199+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:24:27.761199+0000","last_peered":"2023-09-23T15:24:27.761199+0000","last_clean":"2023-09-23T15:24:27.761199+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:24:27.761199+0000","last_undegraded":"2023-09-23T15:24:27.761199+0000","last_fullsized":"2023-09-23T15:24:27.761199+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"330'10284","reported_seq":10345,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:52.021295+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:24:52.021295+0000","last_peered":"2023-09-23T15:24:52.021295+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:24:52.021295+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"300'3700","ondisk_log_start":"300'3700","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6584,"log_dups_size":0,"ondisk_log_size":6584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66453504,"num_objects":1014,"num_object_clones":0,"num_object_copies":3042,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1014,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10284,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-3604}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":645,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761915+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:24:27.761915+0000","last_peered":"2023-09-23T15:24:27.761915+0000","last_clean":"2023-09-23T15:24:27.761915+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:24:27.761915+0000","last_undegraded":"2023-09-23T15:24:27.761915+0000","last_fullsized":"2023-09-23T15:24:27.761915+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":344,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.295994+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:24:40.295994+0000","last_peered":"2023-09-23T15:24:40.295994+0000","last_clean":"2023-09-23T15:24:40.295994+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:24:40.295994+0000","last_undegraded":"2023-09-23T15:24:40.295994+0000","last_fullsized":"2023-09-23T15:24:40.295994+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":238,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.288320+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:24:40.288320+0000","last_peered":"2023-09-23T15:24:40.288320+0000","last_clean":"2023-09-23T15:24:40.288320+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:24:40.288320+0000","last_undegraded":"2023-09-23T15:24:40.288320+0000","last_fullsized":"2023-09-23T15:24:40.288320+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"330'10229","reported_seq":13685,"reported_epoch":330,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:54.233663+0000","last_change":"2023-09-23T15:24:25.991143+0000","last_active":"2023-09-23T15:24:54.233663+0000","last_peered":"2023-09-23T15:24:54.233663+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:24:54.233663+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"301'3700","ondisk_log_start":"301'3700","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6529,"log_dups_size":0,"ondisk_log_size":6529,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55377920,"num_objects":845,"num_object_clones":0,"num_object_copies":2535,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1402,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":845,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10229,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1691,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":-557}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":527,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.262216+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:24:40.262216+0000","last_peered":"2023-09-23T15:24:40.262216+0000","last_clean":"2023-09-23T15:24:40.262216+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:24:40.262216+0000","last_undegraded":"2023-09-23T15:24:40.262216+0000","last_fullsized":"2023-09-23T15:24:40.262216+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":529,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.286140+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:24:40.286140+0000","last_peered":"2023-09-23T15:24:40.286140+0000","last_clean":"2023-09-23T15:24:40.286140+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:24:40.286140+0000","last_undegraded":"2023-09-23T15:24:40.286140+0000","last_fullsized":"2023-09-23T15:24:40.286140+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":305,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762111+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:24:27.762111+0000","last_peered":"2023-09-23T15:24:27.762111+0000","last_clean":"2023-09-23T15:24:27.762111+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:24:27.762111+0000","last_undegraded":"2023-09-23T15:24:27.762111+0000","last_fullsized":"2023-09-23T15:24:27.762111+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":274,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.283774+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:24:40.283774+0000","last_peered":"2023-09-23T15:24:40.283774+0000","last_clean":"2023-09-23T15:24:40.283774+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:24:40.283774+0000","last_undegraded":"2023-09-23T15:24:40.283774+0000","last_fullsized":"2023-09-23T15:24:40.283774+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"329'10151","reported_seq":10214,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:49.913980+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:24:49.913980+0000","last_peered":"2023-09-23T15:24:49.913980+0000","last_clean":"2023-09-23T15:24:49.913980+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:24:49.913980+0000","last_undegraded":"2023-09-23T15:24:49.913980+0000","last_fullsized":"2023-09-23T15:24:49.913980+0000","mapping_epoch":287,"log_start":"294'2400","ondisk_log_start":"294'2400","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7751,"log_dups_size":0,"ondisk_log_size":7751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70320128,"num_objects":1073,"num_object_clones":0,"num_object_copies":3219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10151,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":626,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761620+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:24:27.761620+0000","last_peered":"2023-09-23T15:24:27.761620+0000","last_clean":"2023-09-23T15:24:27.761620+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:24:27.761620+0000","last_undegraded":"2023-09-23T15:24:27.761620+0000","last_fullsized":"2023-09-23T15:24:27.761620+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":864,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762234+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:24:27.762234+0000","last_peered":"2023-09-23T15:24:27.762234+0000","last_clean":"2023-09-23T15:24:27.762234+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:24:27.762234+0000","last_undegraded":"2023-09-23T15:24:27.762234+0000","last_fullsized":"2023-09-23T15:24:27.762234+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761120+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:24:27.761120+0000","last_peered":"2023-09-23T15:24:27.761120+0000","last_clean":"2023-09-23T15:24:27.761120+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:24:27.761120+0000","last_undegraded":"2023-09-23T15:24:27.761120+0000","last_fullsized":"2023-09-23T15:24:27.761120+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":226,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.285560+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:24:40.285560+0000","last_peered":"2023-09-23T15:24:40.285560+0000","last_clean":"2023-09-23T15:24:40.285560+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:24:40.285560+0000","last_undegraded":"2023-09-23T15:24:40.285560+0000","last_fullsized":"2023-09-23T15:24:40.285560+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"330'10097","reported_seq":16389,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.513137+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:24:50.513137+0000","last_peered":"2023-09-23T15:24:50.513137+0000","last_clean":"2023-09-23T15:24:50.513137+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:24:50.513137+0000","last_undegraded":"2023-09-23T15:24:50.513137+0000","last_fullsized":"2023-09-23T15:24:50.513137+0000","mapping_epoch":320,"log_start":"300'3500","ondisk_log_start":"300'3500","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6597,"log_dups_size":0,"ondisk_log_size":6597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66387968,"num_objects":1013,"num_object_clones":0,"num_object_copies":3039,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1013,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10097,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":685,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263610+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:24:40.263610+0000","last_peered":"2023-09-23T15:24:40.263610+0000","last_clean":"2023-09-23T15:24:40.263610+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:24:40.263610+0000","last_undegraded":"2023-09-23T15:24:40.263610+0000","last_fullsized":"2023-09-23T15:24:40.263610+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":947,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.280899+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:24:40.280899+0000","last_peered":"2023-09-23T15:24:40.280899+0000","last_clean":"2023-09-23T15:24:40.280899+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:24:40.280899+0000","last_undegraded":"2023-09-23T15:24:40.280899+0000","last_fullsized":"2023-09-23T15:24:40.280899+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":222,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:47.041494+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:24:47.041494+0000","last_peered":"2023-09-23T15:24:47.041494+0000","last_clean":"2023-09-23T15:24:47.041494+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:24:47.041494+0000","last_undegraded":"2023-09-23T15:24:47.041494+0000","last_fullsized":"2023-09-23T15:24:47.041494+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":230,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.288260+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:24:40.288260+0000","last_peered":"2023-09-23T15:24:40.288260+0000","last_clean":"2023-09-23T15:24:40.288260+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:24:40.288260+0000","last_undegraded":"2023-09-23T15:24:40.288260+0000","last_fullsized":"2023-09-23T15:24:40.288260+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"330'10502","reported_seq":10559,"reported_epoch":330,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:24:54.174480+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:24:54.174480+0000","last_peered":"2023-09-23T15:24:54.174480+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:24:54.174480+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"301'3900","ondisk_log_start":"301'3900","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6602,"log_dups_size":0,"ondisk_log_size":6602,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":56754176,"num_objects":866,"num_object_clones":0,"num_object_copies":2598,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":866,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10502,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-3840}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":658,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761975+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:24:27.761975+0000","last_peered":"2023-09-23T15:24:27.761975+0000","last_clean":"2023-09-23T15:24:27.761975+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:24:27.761975+0000","last_undegraded":"2023-09-23T15:24:27.761975+0000","last_fullsized":"2023-09-23T15:24:27.761975+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":649,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263526+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:24:40.263526+0000","last_peered":"2023-09-23T15:24:40.263526+0000","last_clean":"2023-09-23T15:24:40.263526+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:24:40.263526+0000","last_undegraded":"2023-09-23T15:24:40.263526+0000","last_fullsized":"2023-09-23T15:24:40.263526+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":263,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.292922+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:24:40.292922+0000","last_peered":"2023-09-23T15:24:40.292922+0000","last_clean":"2023-09-23T15:24:40.292922+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:24:40.292922+0000","last_undegraded":"2023-09-23T15:24:40.292922+0000","last_fullsized":"2023-09-23T15:24:40.292922+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":281,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.261833+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:24:40.261833+0000","last_peered":"2023-09-23T15:24:40.261833+0000","last_clean":"2023-09-23T15:24:40.261833+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:24:40.261833+0000","last_undegraded":"2023-09-23T15:24:40.261833+0000","last_fullsized":"2023-09-23T15:24:40.261833+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"330'10178","reported_seq":10234,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:52.027339+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:24:52.027339+0000","last_peered":"2023-09-23T15:24:52.027339+0000","last_clean":"2023-09-23T15:24:52.027339+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:24:52.027339+0000","last_undegraded":"2023-09-23T15:24:52.027339+0000","last_fullsized":"2023-09-23T15:24:52.027339+0000","mapping_epoch":284,"log_start":"300'3600","ondisk_log_start":"300'3600","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10178,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":636,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.285736+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:24:40.285736+0000","last_peered":"2023-09-23T15:24:40.285736+0000","last_clean":"2023-09-23T15:24:40.285736+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:24:40.285736+0000","last_undegraded":"2023-09-23T15:24:40.285736+0000","last_fullsized":"2023-09-23T15:24:40.285736+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":74,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263401+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:24:40.263401+0000","last_peered":"2023-09-23T15:24:40.263401+0000","last_clean":"2023-09-23T15:24:40.263401+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:24:40.263401+0000","last_undegraded":"2023-09-23T15:24:40.263401+0000","last_fullsized":"2023-09-23T15:24:40.263401+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":297,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.274688+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:24:40.274688+0000","last_peered":"2023-09-23T15:24:40.274688+0000","last_clean":"2023-09-23T15:24:40.274688+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:24:40.274688+0000","last_undegraded":"2023-09-23T15:24:40.274688+0000","last_fullsized":"2023-09-23T15:24:40.274688+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":309,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.274210+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:24:40.274210+0000","last_peered":"2023-09-23T15:24:40.274210+0000","last_clean":"2023-09-23T15:24:40.274210+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:24:40.274210+0000","last_undegraded":"2023-09-23T15:24:40.274210+0000","last_fullsized":"2023-09-23T15:24:40.274210+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"330'10451","reported_seq":10507,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.652020+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:24:50.652020+0000","last_peered":"2023-09-23T15:24:50.652020+0000","last_clean":"2023-09-23T15:24:50.652020+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:24:50.652020+0000","last_undegraded":"2023-09-23T15:24:50.652020+0000","last_fullsized":"2023-09-23T15:24:50.652020+0000","mapping_epoch":284,"log_start":"301'3900","ondisk_log_start":"301'3900","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6551,"log_dups_size":0,"ondisk_log_size":6551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10451,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":608,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.261804+0000","last_change":"2023-09-23T15:24:24.592597+0000","last_active":"2023-09-23T15:24:40.261804+0000","last_peered":"2023-09-23T15:24:40.261804+0000","last_clean":"2023-09-23T15:24:40.261804+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:24:40.261804+0000","last_undegraded":"2023-09-23T15:24:40.261804+0000","last_fullsized":"2023-09-23T15:24:40.261804+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1043,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.263776+0000","last_change":"2023-09-23T15:24:25.990916+0000","last_active":"2023-09-23T15:24:40.263776+0000","last_peered":"2023-09-23T15:24:40.263776+0000","last_clean":"2023-09-23T15:24:40.263776+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:24:40.263776+0000","last_undegraded":"2023-09-23T15:24:40.263776+0000","last_fullsized":"2023-09-23T15:24:40.263776+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":206,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.286060+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:24:40.286060+0000","last_peered":"2023-09-23T15:24:40.286060+0000","last_clean":"2023-09-23T15:24:40.286060+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:24:40.286060+0000","last_undegraded":"2023-09-23T15:24:40.286060+0000","last_fullsized":"2023-09-23T15:24:40.286060+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":293,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.271425+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:24:40.271425+0000","last_peered":"2023-09-23T15:24:40.271425+0000","last_clean":"2023-09-23T15:24:40.271425+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:24:40.271425+0000","last_undegraded":"2023-09-23T15:24:40.271425+0000","last_fullsized":"2023-09-23T15:24:40.271425+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"330'10200","reported_seq":10265,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:54.189040+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:24:54.189040+0000","last_peered":"2023-09-23T15:24:54.189040+0000","last_clean":"2023-09-23T15:24:54.189040+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:24:54.189040+0000","last_undegraded":"2023-09-23T15:24:54.189040+0000","last_fullsized":"2023-09-23T15:24:54.189040+0000","mapping_epoch":287,"log_start":"300'3600","ondisk_log_start":"300'3600","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6600,"log_dups_size":0,"ondisk_log_size":6600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":60293120,"num_objects":920,"num_object_clones":0,"num_object_copies":2760,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":920,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10200,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":637,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760696+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:24:27.760696+0000","last_peered":"2023-09-23T15:24:27.760696+0000","last_clean":"2023-09-23T15:24:27.760696+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:24:27.760696+0000","last_undegraded":"2023-09-23T15:24:27.760696+0000","last_fullsized":"2023-09-23T15:24:27.760696+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":86,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.375644+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:24:50.375644+0000","last_peered":"2023-09-23T15:24:50.375644+0000","last_clean":"2023-09-23T15:24:50.375644+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:24:50.375644+0000","last_undegraded":"2023-09-23T15:24:50.375644+0000","last_fullsized":"2023-09-23T15:24:50.375644+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":883,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.286272+0000","last_change":"2023-09-23T15:24:25.979031+0000","last_active":"2023-09-23T15:24:40.286272+0000","last_peered":"2023-09-23T15:24:40.286272+0000","last_clean":"2023-09-23T15:24:40.286272+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:24:40.286272+0000","last_undegraded":"2023-09-23T15:24:40.286272+0000","last_fullsized":"2023-09-23T15:24:40.286272+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":324,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.296284+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:24:40.296284+0000","last_peered":"2023-09-23T15:24:40.296284+0000","last_clean":"2023-09-23T15:24:40.296284+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:24:40.296284+0000","last_undegraded":"2023-09-23T15:24:40.296284+0000","last_fullsized":"2023-09-23T15:24:40.296284+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"329'10071","reported_seq":10132,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:49.869053+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:24:49.869053+0000","last_peered":"2023-09-23T15:24:49.869053+0000","last_clean":"2023-09-23T15:24:49.869053+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:24:49.869053+0000","last_undegraded":"2023-09-23T15:24:49.869053+0000","last_fullsized":"2023-09-23T15:24:49.869053+0000","mapping_epoch":287,"log_start":"294'2300","ondisk_log_start":"294'2300","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7771,"log_dups_size":0,"ondisk_log_size":7771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633176,"num_objects":1033,"num_object_clones":0,"num_object_copies":3099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1033,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10071,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":654,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293433+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:24:40.293433+0000","last_peered":"2023-09-23T15:24:40.293433+0000","last_clean":"2023-09-23T15:24:40.293433+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:24:40.293433+0000","last_undegraded":"2023-09-23T15:24:40.293433+0000","last_fullsized":"2023-09-23T15:24:40.293433+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":766,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.262066+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:24:40.262066+0000","last_peered":"2023-09-23T15:24:40.262066+0000","last_clean":"2023-09-23T15:24:40.262066+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:24:40.262066+0000","last_undegraded":"2023-09-23T15:24:40.262066+0000","last_fullsized":"2023-09-23T15:24:40.262066+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":336,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.274306+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:24:40.274306+0000","last_peered":"2023-09-23T15:24:40.274306+0000","last_clean":"2023-09-23T15:24:40.274306+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:24:40.274306+0000","last_undegraded":"2023-09-23T15:24:40.274306+0000","last_fullsized":"2023-09-23T15:24:40.274306+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760040+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:24:27.760040+0000","last_peered":"2023-09-23T15:24:27.760040+0000","last_clean":"2023-09-23T15:24:27.760040+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:24:27.760040+0000","last_undegraded":"2023-09-23T15:24:27.760040+0000","last_fullsized":"2023-09-23T15:24:27.760040+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":313,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.271344+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:24:40.271344+0000","last_peered":"2023-09-23T15:24:40.271344+0000","last_clean":"2023-09-23T15:24:40.271344+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:24:40.271344+0000","last_undegraded":"2023-09-23T15:24:40.271344+0000","last_fullsized":"2023-09-23T15:24:40.271344+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":333,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.283350+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:24:40.283350+0000","last_peered":"2023-09-23T15:24:40.283350+0000","last_clean":"2023-09-23T15:24:40.283350+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:24:40.283350+0000","last_undegraded":"2023-09-23T15:24:40.283350+0000","last_fullsized":"2023-09-23T15:24:40.283350+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":342,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762226+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:24:27.762226+0000","last_peered":"2023-09-23T15:24:27.762226+0000","last_clean":"2023-09-23T15:24:27.762226+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:24:27.762226+0000","last_undegraded":"2023-09-23T15:24:27.762226+0000","last_fullsized":"2023-09-23T15:24:27.762226+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.761214+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:24:27.761214+0000","last_peered":"2023-09-23T15:24:27.761214+0000","last_clean":"2023-09-23T15:24:27.761214+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:24:27.761214+0000","last_undegraded":"2023-09-23T15:24:27.761214+0000","last_fullsized":"2023-09-23T15:24:27.761214+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":322,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.762025+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:24:27.762025+0000","last_peered":"2023-09-23T15:24:27.762025+0000","last_clean":"2023-09-23T15:24:27.762025+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:24:27.762025+0000","last_undegraded":"2023-09-23T15:24:27.762025+0000","last_fullsized":"2023-09-23T15:24:27.762025+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":42,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.760156+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:24:27.760156+0000","last_peered":"2023-09-23T15:24:27.760156+0000","last_clean":"2023-09-23T15:24:27.760156+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:24:27.760156+0000","last_undegraded":"2023-09-23T15:24:27.760156+0000","last_fullsized":"2023-09-23T15:24:27.760156+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":321,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.293764+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:24:40.293764+0000","last_peered":"2023-09-23T15:24:40.293764+0000","last_clean":"2023-09-23T15:24:40.293764+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:24:40.293764+0000","last_undegraded":"2023-09-23T15:24:40.293764+0000","last_fullsized":"2023-09-23T15:24:40.293764+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":319,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.292924+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:24:40.292924+0000","last_peered":"2023-09-23T15:24:40.292924+0000","last_clean":"2023-09-23T15:24:40.292924+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:24:40.292924+0000","last_undegraded":"2023-09-23T15:24:40.292924+0000","last_fullsized":"2023-09-23T15:24:40.292924+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":282,"reported_epoch":329,"state":"active+clean","last_fresh":"2023-09-23T15:24:40.270992+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:24:40.270992+0000","last_peered":"2023-09-23T15:24:40.270992+0000","last_clean":"2023-09-23T15:24:40.270992+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:24:40.270992+0000","last_undegraded":"2023-09-23T15:24:40.270992+0000","last_fullsized":"2023-09-23T15:24:40.270992+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":56,"reported_epoch":328,"state":"active+clean","last_fresh":"2023-09-23T15:24:27.759885+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:24:27.759885+0000","last_peered":"2023-09-23T15:24:27.759885+0000","last_clean":"2023-09-23T15:24:27.759885+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:24:27.759885+0000","last_undegraded":"2023-09-23T15:24:27.759885+0000","last_fullsized":"2023-09-23T15:24:27.759885+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1029439512,"num_objects":15709,"num_object_clones":0,"num_object_copies":47127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20516,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":15709,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":164561,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15329,"num_bytes_recovered":767033344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4170985472,"data_stored":4170973256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109061,"ondisk_log_size":109061,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":112,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201642,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":632512,"kb_used_data":489208,"kb_used_omap":0,"kb_used_meta":143296,"kb_avail":93112640,"statfs":{"total":95995035648,"available":95347343360,"internally_reserved":0,"allocated":500948992,"data_stored":499935679,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":146735104},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":6.6959999999999997,"15min":3.9710000000000001},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":12.096,"5min":226.226,"15min":226.226},"last":4.2430000000000003},{"interface":"front","average":{"1min":4.1159999999999997,"5min":7.9660000000000002,"15min":4.6340000000000003},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":13.196,"5min":226.619,"15min":226.619},"last":0.27100000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.573,"5min":1.425,"15min":2.056},"min":{"1min":0.29999999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":5.9589999999999996,"5min":8.4290000000000003,"15min":9.0459999999999994},"last":3.8599999999999999},{"interface":"front","average":{"1min":3.2010000000000001,"5min":1.79,"15min":2.5499999999999998},"min":{"1min":0.434,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.865,"5min":11.865,"15min":16.065999999999999},"last":3.7000000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.7450000000000001,"15min":3.327},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":7.7889999999999997,"5min":90.227999999999994,"15min":101.807},"last":4.2080000000000002},{"interface":"front","average":{"1min":4.2880000000000003,"5min":3.2400000000000002,"15min":4.0030000000000001},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.127000000000001,"5min":90.197999999999993,"15min":101.84099999999999},"last":3.1749999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:24:03 2023","interfaces":[{"interface":"back","average":{"1min":4.8319999999999999,"5min":4.5869999999999997,"15min":3.9220000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":41.588999999999999,"5min":146.166,"15min":146.166},"last":3.7989999999999999},{"interface":"front","average":{"1min":5.4000000000000004,"5min":4.8520000000000003,"15min":3.98},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.32400000000000001},"max":{"1min":41.542000000000002,"5min":146.18299999999999,"15min":146.18299999999999},"last":9.3870000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0219999999999998,"5min":1.696,"15min":2.7069999999999999},"min":{"1min":0.316,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.3550000000000004,"5min":16.771999999999998,"15min":150.119},"last":4.726},{"interface":"front","average":{"1min":3.456,"5min":1.5269999999999999,"15min":2.8620000000000001},"min":{"1min":0.315,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.023,"5min":18.023,"15min":150.596},"last":9.0649999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":12.602,"5min":5.2270000000000003,"15min":3.5640000000000001},"min":{"1min":0.35599999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":216.85900000000001,"5min":224.441,"15min":224.441},"last":0.36599999999999999},{"interface":"front","average":{"1min":13.93,"5min":5.742,"15min":3.6669999999999998},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":216.43799999999999,"5min":224.28700000000001,"15min":224.28700000000001},"last":4.7549999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:30 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.4409999999999998,"15min":2.3010000000000002},"min":{"1min":0.48999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9659999999999993,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":1.4970000000000001},{"interface":"front","average":{"1min":3.0550000000000002,"5min":2.6230000000000002,"15min":2.5510000000000002},"min":{"1min":0.35699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.135,"5min":10.336,"15min":10.336},"last":4.7839999999999998}]}]},{"osd":6,"up_from":304,"seq":1305670058015,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":373300,"kb_used_data":294252,"kb_used_omap":13,"kb_used_meta":79026,"kb_avail":93371852,"statfs":{"total":95995035648,"available":95612776448,"internally_reserved":0,"allocated":301314048,"data_stored":300312872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":80922979},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":16.183,"5min":4.4199999999999999,"15min":2.46},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":184.53200000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":0.23999999999999999},{"interface":"front","average":{"1min":8.5370000000000008,"5min":4.0090000000000003,"15min":3.2549999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.227999999999994,"5min":86.227999999999994,"15min":86.227999999999994},"last":0.997}]},{"osd":1,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":9.2279999999999998,"5min":4.0389999999999997,"15min":3.1739999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":86.534000000000006,"5min":86.534000000000006,"15min":86.534000000000006},"last":1.2210000000000001},{"interface":"front","average":{"1min":8.5199999999999996,"5min":3.9929999999999999,"15min":3.2389999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":86.144000000000005,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.27200000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.762,"15min":1.6040000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":1.111},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.99,"15min":1.8140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.1910000000000007,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":1.1970000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.7109999999999999,"15min":2.423},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":41.261000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":0.40200000000000002},{"interface":"front","average":{"1min":5.1559999999999997,"5min":2.8479999999999999,"15min":2.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":41.408000000000001},"last":0.60599999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.1839999999999999,"15min":1.02},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.754,"5min":2.0070000000000001,"15min":1.883},"min":{"1min":0.39800000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.627000000000001,"5min":13.627000000000001,"15min":13.627000000000001},"last":0.64700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.2749999999999999,"15min":1.903},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.74,"5min":13.74,"15min":13.74},"last":1.1559999999999999},{"interface":"front","average":{"1min":3.194,"5min":1.8260000000000001,"15min":1.5980000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":15.868,"5min":15.868,"15min":15.868},"last":0.43099999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.752,"15min":1.5169999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":12.023999999999999,"5min":12.023999999999999,"15min":12.023999999999999},"last":0.97099999999999997},{"interface":"front","average":{"1min":3.5259999999999998,"5min":2.1629999999999998,"15min":1.9359999999999999},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":12.641999999999999},"last":4.1989999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015315,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1112488,"kb_used_data":655968,"kb_used_omap":6,"kb_used_meta":456505,"kb_avail":92632664,"statfs":{"total":95995035648,"available":94855847936,"internally_reserved":0,"allocated":671711232,"data_stored":670151977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":467461195},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":1.964,"5min":3.3540000000000001,"15min":3.3820000000000001},"min":{"1min":0.23300000000000001,"5min":0.191,"15min":0.191},"max":{"1min":12.298,"5min":60.817999999999998,"15min":60.817999999999998},"last":3.3799999999999999},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.1550000000000002,"15min":4.165},"min":{"1min":0.36499999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.779,"5min":63.594999999999999,"15min":63.594999999999999},"last":3.5529999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.073,"5min":2.3530000000000002,"15min":4.085},"min":{"1min":0.34699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":9.9619999999999997,"5min":10.329000000000001,"15min":109.375},"last":9.0869999999999997},{"interface":"front","average":{"1min":1.8580000000000001,"5min":2.2789999999999999,"15min":3.5640000000000001},"min":{"1min":0.26700000000000002,"5min":0.219,"15min":0.219},"max":{"1min":12.259,"5min":13.314,"15min":105.16},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":2.1280000000000001,"5min":4.399,"15min":7.6849999999999996},"min":{"1min":0.34699999999999998,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":12.218999999999999,"5min":64.540999999999997,"15min":211.405},"last":4.5410000000000004},{"interface":"front","average":{"1min":2.4279999999999999,"5min":4.601,"15min":8.3529999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":10.551,"5min":67.465000000000003,"15min":210.988},"last":4.5960000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":3.0430000000000001,"5min":3.375,"15min":3.3170000000000002},"min":{"1min":0.33300000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.379,"5min":80.570999999999998,"15min":80.570999999999998},"last":9.1370000000000005},{"interface":"front","average":{"1min":2.016,"5min":3.5099999999999998,"15min":3.4220000000000002},"min":{"1min":0.40500000000000003,"5min":0.32500000000000001,"15min":0.308},"max":{"1min":12.279999999999999,"5min":80.75,"15min":80.75},"last":0.58899999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":9.7769999999999992,"5min":4.0979999999999999,"15min":3.4649999999999999},"min":{"1min":0.33900000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":115.401,"5min":115.401,"15min":115.401},"last":3.0680000000000001},{"interface":"front","average":{"1min":9.9770000000000003,"5min":3.9180000000000001,"15min":3.3359999999999999},"min":{"1min":0.33700000000000002,"5min":0.311,"15min":0.311},"max":{"1min":115.169,"5min":115.169,"15min":115.169},"last":9.173}]},{"osd":6,"last update":"Sat Sep 23 15:24:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":3.335,"15min":3.1589999999999998},"min":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":19.553999999999998,"5min":19.553999999999998,"15min":19.553999999999998},"last":9.2050000000000001},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.2869999999999999,"15min":2.3340000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9770000000000003,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":3.1110000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:24:08 2023","interfaces":[{"interface":"back","average":{"1min":5.2889999999999997,"5min":2.6520000000000001,"15min":2.948},"min":{"1min":0.55500000000000005,"5min":0.371,"15min":0.371},"max":{"1min":19.702999999999999,"5min":19.702999999999999,"15min":19.702999999999999},"last":9.234},{"interface":"front","average":{"1min":3.9550000000000001,"5min":2.4849999999999999,"15min":2.6800000000000002},"min":{"1min":0.5,"5min":0.377,"15min":0.377},"max":{"1min":20.722000000000001,"5min":20.722000000000001,"15min":20.722000000000001},"last":0.55400000000000005}]}]},{"osd":0,"up_from":95,"seq":408021893365,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1230108,"kb_used_data":647572,"kb_used_omap":8,"kb_used_meta":582519,"kb_avail":92515044,"statfs":{"total":95995035648,"available":94735405056,"internally_reserved":0,"allocated":663113728,"data_stored":661684170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":596500052},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:24:01 2023","interfaces":[{"interface":"back","average":{"1min":2.75,"5min":2.8519999999999999,"15min":3.468},"min":{"1min":0.28399999999999997,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":17.542000000000002,"5min":48.442999999999998,"15min":64.412999999999997},"last":0.439},{"interface":"front","average":{"1min":3.3239999999999998,"5min":2.7250000000000001,"15min":3.387},"min":{"1min":0.24399999999999999,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":12.058999999999999,"5min":51.304000000000002,"15min":51.304000000000002},"last":0.56699999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":2.4020000000000001,"15min":4.3609999999999998},"min":{"1min":0.23300000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":13.022,"5min":30.510999999999999,"15min":122.20099999999999},"last":2.972},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.7170000000000001,"15min":4.2359999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":9.2050000000000001,"5min":30.396999999999998,"15min":122.071},"last":0.68300000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3140000000000001,"5min":4.8300000000000001,"15min":5.8819999999999997},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.20000000000000001},"max":{"1min":41.380000000000003,"5min":197.78899999999999,"15min":197.78899999999999},"last":3.0659999999999998},{"interface":"front","average":{"1min":4.8079999999999998,"5min":5.2050000000000001,"15min":6.3630000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.252},"max":{"1min":41.465000000000003,"5min":200.58799999999999,"15min":200.58799999999999},"last":0.878}]},{"osd":4,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.2330000000000001,"5min":2.177,"15min":2.278},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.28000000000000003},"max":{"1min":15.038,"5min":15.358000000000001,"15min":31.341999999999999},"last":0.61599999999999999},{"interface":"front","average":{"1min":2.327,"5min":1.873,"15min":2.2429999999999999},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":8.2629999999999999,"5min":9.8249999999999993,"15min":99.168999999999997},"last":8.5830000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3799999999999999,"5min":2.1880000000000002,"15min":2.456},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.28599999999999998},"max":{"1min":10.618,"5min":10.768000000000001,"15min":109.369},"last":0.372},{"interface":"front","average":{"1min":3.7959999999999998,"5min":2.3879999999999999,"15min":2.528},"min":{"1min":0.39100000000000001,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":15.363,"5min":15.436999999999999,"15min":109.387},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.887,"15min":2.835},"min":{"1min":0.45900000000000002,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":4.5640000000000001},{"interface":"front","average":{"1min":3.4630000000000001,"5min":3.3809999999999998,"15min":3.367},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":15.829000000000001,"5min":17.385000000000002,"15min":17.385000000000002},"last":3.04}]},{"osd":7,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2779999999999996,"5min":2.4159999999999999,"15min":2.6219999999999999},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":17.712,"5min":17.712,"15min":99.549999999999997},"last":5.4790000000000001},{"interface":"front","average":{"1min":3.2629999999999999,"5min":2.556,"15min":2.2610000000000001},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.706,"5min":15.407999999999999,"15min":31.225999999999999},"last":4.6070000000000002}]}]},{"osd":2,"up_from":105,"seq":450971566312,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1293328,"kb_used_data":700616,"kb_used_omap":11,"kb_used_meta":592692,"kb_avail":92451824,"statfs":{"total":95995035648,"available":94670667776,"internally_reserved":0,"allocated":717430784,"data_stored":716007199,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":606917392},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":13.015000000000001,"5min":7.2009999999999996,"15min":4.9459999999999997},"min":{"1min":0.214,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":88.622,"5min":88.622,"15min":88.622},"last":0.34999999999999998},{"interface":"front","average":{"1min":12.861000000000001,"5min":7.21,"15min":4.6269999999999998},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":88.649000000000001,"5min":88.649000000000001,"15min":88.649000000000001},"last":1.4259999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":7.0439999999999996,"5min":4.4900000000000002,"15min":4.7210000000000001},"min":{"1min":0.214,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":71.819000000000003,"5min":71.819000000000003,"15min":84.641000000000005},"last":6.5529999999999999},{"interface":"front","average":{"1min":7.0810000000000004,"5min":4.1989999999999998,"15min":3.6110000000000002},"min":{"1min":0.317,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":71.805000000000007,"5min":71.805000000000007,"15min":84.751999999999995},"last":4.4320000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:24:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":5.5350000000000001,"15min":5.2850000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":8.2970000000000006,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.55500000000000005},{"interface":"front","average":{"1min":2.5190000000000001,"5min":5.9119999999999999,"15min":5.3650000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":10.935,"5min":76.742000000000004,"15min":86.158000000000001},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.6020000000000001,"15min":2.3969999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":16.712,"5min":16.712,"15min":84.608000000000004},"last":1.47},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.579,"15min":2.9980000000000002},"min":{"1min":0.376,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":6.3789999999999996,"5min":7.7880000000000003,"15min":87.019999999999996},"last":1.2470000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.839,"15min":2.6480000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.6980000000000004,"5min":8.9909999999999997,"15min":84.498000000000005},"last":0.49099999999999999},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.7210000000000001,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.5099999999999998,"5min":10.808999999999999,"15min":84.664000000000001},"last":1.5049999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":2.3290000000000002,"15min":2.2989999999999999},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.4600000000000009,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.8340000000000001,"15min":1.978},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.6909999999999998,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.9970000000000001,"15min":2.593},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":9.3859999999999992,"5min":10.227,"15min":84.596000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.9550000000000001,"15min":2.4910000000000001},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":16.074999999999999,"5min":16.074999999999999,"15min":84.783000000000001},"last":1.532}]}]},{"osd":3,"up_from":110,"seq":472446402785,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1095368,"kb_used_data":489088,"kb_used_omap":3,"kb_used_meta":606140,"kb_avail":92649784,"statfs":{"total":95995035648,"available":94873378816,"internally_reserved":0,"allocated":500826112,"data_stored":499403143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":620687529},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":8.5269999999999992,"5min":3.7440000000000002,"15min":3.6949999999999998},"min":{"1min":0.20599999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":108.318,"5min":108.318,"15min":108.318},"last":1.417},{"interface":"front","average":{"1min":9.3879999999999999,"5min":4.1669999999999998,"15min":3.9740000000000002},"min":{"1min":0.25600000000000001,"5min":0.184,"15min":0.17599999999999999},"max":{"1min":108.889,"5min":108.889,"15min":108.889},"last":3.7210000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":4.8179999999999996,"15min":4.8339999999999996},"min":{"1min":0.24099999999999999,"5min":0.214,"15min":0.214},"max":{"1min":9.2390000000000008,"5min":126.875,"15min":126.875},"last":2.823},{"interface":"front","average":{"1min":2.1400000000000001,"5min":5.5970000000000004,"15min":5.0519999999999996},"min":{"1min":0.28599999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":10.805999999999999,"5min":128.249,"15min":128.249},"last":3.0579999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.5640000000000001,"5min":2.2349999999999999,"15min":4.7539999999999996},"min":{"1min":0.27000000000000002,"5min":0.26300000000000001,"15min":0.22},"max":{"1min":12.210000000000001,"5min":17.199999999999999,"15min":109.077},"last":0.93899999999999995},{"interface":"front","average":{"1min":3.5670000000000002,"5min":3.3410000000000002,"15min":5.7969999999999997},"min":{"1min":0.34100000000000003,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":14.763,"5min":36.582000000000001,"15min":119.72799999999999},"last":2.25}]},{"osd":4,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":2.2080000000000002,"5min":2.1429999999999998,"15min":2.2669999999999999},"min":{"1min":0.35299999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":11.417999999999999,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.5660000000000001,"15min":2.121},"min":{"1min":0.35799999999999998,"5min":0.32100000000000001,"15min":0.26400000000000001},"max":{"1min":8.0050000000000008,"5min":12.500999999999999,"15min":52.228999999999999},"last":5.9130000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":3.1840000000000002,"5min":3.7330000000000001,"15min":2.9249999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.25600000000000001},"max":{"1min":11.372999999999999,"5min":148.619,"15min":148.619},"last":0.435},{"interface":"front","average":{"1min":1.909,"5min":2.9780000000000002,"15min":2.7789999999999999},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.315},"max":{"1min":6.8520000000000003,"5min":148.56999999999999,"15min":148.56999999999999},"last":2.8479999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:28 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.226,"15min":2.2250000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5460000000000003,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":6.2359999999999998},{"interface":"front","average":{"1min":2.0459999999999998,"5min":3.3300000000000001,"15min":3.544},"min":{"1min":0.42099999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":6.7699999999999996,"5min":36.517000000000003,"15min":36.517000000000003},"last":0.46100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:23:52 2023","interfaces":[{"interface":"back","average":{"1min":1.369,"5min":2.9279999999999999,"15min":2.681},"min":{"1min":0.41799999999999998,"5min":0.38200000000000001,"15min":0.28299999999999997},"max":{"1min":4.6699999999999999,"5min":145.25,"15min":145.25},"last":6.298},{"interface":"front","average":{"1min":1.877,"5min":3.121,"15min":2.4990000000000001},"min":{"1min":0.40699999999999997,"5min":0.36899999999999999,"15min":0.34799999999999998},"max":{"1min":7.1829999999999998,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.97099999999999997}]}]},{"osd":4,"up_from":31,"seq":133143986550,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":685036,"kb_used_data":352292,"kb_used_omap":0,"kb_used_meta":332672,"kb_avail":93060116,"statfs":{"total":95995035648,"available":95293558784,"internally_reserved":0,"allocated":360747008,"data_stored":359745689,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":340656128},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":5.2699999999999996,"5min":5.1219999999999999,"15min":4.2460000000000004},"min":{"1min":0.28299999999999997,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":53.771000000000001,"5min":162.75999999999999,"15min":162.75999999999999},"last":0.751},{"interface":"front","average":{"1min":4.0990000000000002,"5min":4.8369999999999997,"15min":4.1959999999999997},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":53.145000000000003,"5min":162.34800000000001,"15min":162.34800000000001},"last":4.2450000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":5.1509999999999998,"5min":4.1280000000000001,"15min":3.4590000000000001},"min":{"1min":0.254,"5min":0.249,"15min":0.245},"max":{"1min":60.122,"5min":113.11199999999999,"15min":113.11199999999999},"last":0.95099999999999996},{"interface":"front","average":{"1min":5.5640000000000001,"5min":4.1980000000000004,"15min":3.8839999999999999},"min":{"1min":0.28299999999999997,"5min":0.252,"15min":0.252},"max":{"1min":60.802,"5min":113.02,"15min":113.02},"last":2.1930000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":3.3599999999999999,"15min":4.3040000000000003},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":4.8010000000000002,"5min":61.646000000000001,"15min":148.08000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.536,"15min":4.5309999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.25700000000000001},"max":{"1min":6.1980000000000004,"5min":64.840000000000003,"15min":147.88900000000001},"last":4.2060000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:23:50 2023","interfaces":[{"interface":"back","average":{"1min":2.234,"5min":6.3780000000000001,"15min":5.9969999999999999},"min":{"1min":0.36799999999999999,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":6.5060000000000002,"5min":156.46000000000001,"15min":163.97499999999999},"last":0.874},{"interface":"front","average":{"1min":2.1360000000000001,"5min":6.5069999999999997,"15min":5.9400000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27800000000000002},"max":{"1min":8.3829999999999991,"5min":156.74299999999999,"15min":164.447},"last":6.649}]},{"osd":5,"last update":"Sat Sep 23 15:24:02 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.069,"15min":1.6279999999999999},"min":{"1min":0.251,"5min":0.251,"15min":0.23499999999999999},"max":{"1min":6.4089999999999998,"5min":6.4409999999999998,"15min":87.912999999999997},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.458,"5min":1.2789999999999999,"15min":1.573},"min":{"1min":0.28599999999999998,"5min":0.26700000000000002,"15min":0.252},"max":{"1min":5.7199999999999998,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.40200000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.446,"15min":1.446},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.8010000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.121,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.5380000000000003,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.177,"15min":1.0740000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.2999999999999998,"5min":6.0730000000000004,"15min":18.23},"last":0.51800000000000002},{"interface":"front","average":{"1min":1.895,"5min":1.2709999999999999,"15min":1.1890000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.3239999999999998,"5min":6.3239999999999998,"15min":14.044},"last":0.443}]}]},{"osd":5,"up_from":39,"seq":167503724913,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":945508,"kb_used_data":559516,"kb_used_omap":0,"kb_used_meta":385984,"kb_avail":92799644,"statfs":{"total":95995035648,"available":95026835456,"internally_reserved":0,"allocated":572944384,"data_stored":571947518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":395247616},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":4.1900000000000004,"15min":3.3079999999999998},"min":{"1min":0.251,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.178999999999998,"5min":94.372,"15min":124.31},"last":0.38700000000000001},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.0720000000000001,"15min":3.3769999999999998},"min":{"1min":0.44400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.106000000000002,"5min":94.396000000000001,"15min":124.374},"last":2.2549999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":10.42,"5min":4.4420000000000002,"15min":3.1709999999999998},"min":{"1min":0.27300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":86.900999999999996,"5min":86.900999999999996,"15min":124.38800000000001},"last":8.9320000000000004},{"interface":"front","average":{"1min":5.2599999999999998,"5min":3.6110000000000002,"15min":3.0910000000000002},"min":{"1min":0.32100000000000001,"5min":0.27400000000000002,"15min":0.25900000000000001},"max":{"1min":40.939,"5min":66.825000000000003,"15min":124.29000000000001},"last":4.2590000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":2.444,"15min":3.7770000000000001},"min":{"1min":0.35899999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":11.754,"5min":74.019000000000005,"15min":193.00999999999999},"last":2.3610000000000002},{"interface":"front","average":{"1min":2.8900000000000001,"5min":2.5489999999999999,"15min":3.8239999999999998},"min":{"1min":0.37,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":10.542999999999999,"5min":73.712000000000003,"15min":193.15700000000001},"last":4.3369999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.6630000000000003,"5min":5.2350000000000003,"15min":6.2359999999999998},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.27500000000000002},"max":{"1min":55.354999999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":4.3979999999999997},{"interface":"front","average":{"1min":5.9699999999999998,"5min":6.1470000000000002,"15min":6.3369999999999997},"min":{"1min":0.40400000000000003,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":55.417000000000002,"5min":266.80399999999997,"15min":266.80399999999997},"last":2.9359999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:24:07 2023","interfaces":[{"interface":"back","average":{"1min":3.169,"5min":1.292,"15min":1.9470000000000001},"min":{"1min":0.36399999999999999,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":10.749000000000001,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":3.222,"5min":1.3009999999999999,"15min":2.0449999999999999},"min":{"1min":0.41699999999999998,"5min":0.27800000000000002,"15min":0.26000000000000001},"max":{"1min":10.516,"5min":10.516,"15min":138.49600000000001},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.4390000000000001,"15min":2.411},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":10.577999999999999,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.90000000000000002},{"interface":"front","average":{"1min":3.5249999999999999,"5min":2.633,"15min":2.484},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.257999999999999,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.54600000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0419999999999998,"5min":1.8260000000000001,"15min":1.478},"min":{"1min":0.40500000000000003,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":15.849,"5min":15.849,"15min":15.849},"last":0.40500000000000003},{"interface":"front","average":{"1min":3.7360000000000002,"5min":1.7090000000000001,"15min":1.45},"min":{"1min":0.435,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":16.134,"5min":16.134,"15min":16.134},"last":0.58499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":644808704,"data_stored":644808704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":652935168,"data_stored":652935168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":712118272,"data_stored":712114200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":483852288,"data_stored":483852288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":344391680,"data_stored":344391680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":556011520,"data_stored":556007448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":286064640,"data_stored":286064640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":490803200,"data_stored":490799128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:24:58.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T15:24:58.855 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:58.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:58.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:58.933 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:24:58.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:59.022 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:59.112 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:59.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:59.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:59.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:24:59.289 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:59.370 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:24:59.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:59.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:24:59.481 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:24:59.586 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:24:59.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:24:59.657 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:24:59.707 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:24:59.757 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:24:59.837 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:24:59.884 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:24:59.929 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:24:59.974 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:24:59.974 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:24:59.975 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.976 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:24:59.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:24:59.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:00.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:00.155 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:00.387 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:00.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:00.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:00.550 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:00 smithi017 ceph-mon[215646]: pgmap v977: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 842 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 20277/40263 objects degraded (50.361%); 1.0 MiB/s, 107 objects/s recovering 2023-09-23T15:25:00.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/936798232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:00.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:00 smithi017 ceph-mon[222878]: pgmap v977: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 842 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 20277/40263 objects degraded (50.361%); 1.0 MiB/s, 107 objects/s recovering 2023-09-23T15:25:00.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/936798232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:00.854 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:00 smithi027 ceph-mon[195233]: pgmap v977: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 842 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 660 op/s; 20277/40263 objects degraded (50.361%); 1.0 MiB/s, 107 objects/s recovering 2023-09-23T15:25:00.855 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/936798232' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:02.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:02.141 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:02.243 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:02.477 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:02.611 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:02.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:02.628 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:02.628 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:02.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:02.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:02.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:02 smithi017 ceph-mon[215646]: pgmap v978: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 686 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 19955/32802 objects degraded (60.835%); 1.0 MiB/s, 103 objects/s recovering 2023-09-23T15:25:02.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:02 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 19955/32802 objects degraded (60.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:02.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:02 smithi017 ceph-mon[222878]: pgmap v978: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 686 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 19955/32802 objects degraded (60.835%); 1.0 MiB/s, 103 objects/s recovering 2023-09-23T15:25:02.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:02 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 19955/32802 objects degraded (60.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:02.834 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:02.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:02 smithi027 ceph-mon[195233]: pgmap v978: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 686 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 703 op/s; 19955/32802 objects degraded (60.835%); 1.0 MiB/s, 103 objects/s recovering 2023-09-23T15:25:02.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:02 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 19955/32802 objects degraded (60.835%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:03.013 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:03.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:03.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:03.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:03.204 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:04.514 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:25:04.543 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:04.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:04.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:04.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:25:04.744 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:04.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:04 smithi027 ceph-mon[195233]: pgmap v979: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 634 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 19688/30300 objects degraded (64.977%); 890 KiB/s, 111 objects/s recovering 2023-09-23T15:25:04.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3549707889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:04.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:04.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[215646]: pgmap v979: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 634 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 19688/30300 objects degraded (64.977%); 890 KiB/s, 111 objects/s recovering 2023-09-23T15:25:04.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3549707889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:04.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:04.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[222878]: pgmap v979: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 634 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 19688/30300 objects degraded (64.977%); 890 KiB/s, 111 objects/s recovering 2023-09-23T15:25:04.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3549707889' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:04.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[215646]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:05.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:05 smithi017 ceph-mon[222878]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: failed to encode map e331 with expected crc 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:05.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:05 smithi027 ceph-mon[195233]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T15:25:06.685 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:06.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:06.691 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:06 smithi017 ceph-mon[215646]: pgmap v981: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 568 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 19378/27126 objects degraded (71.437%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:06.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:06 smithi017 ceph-mon[222878]: pgmap v981: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 568 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 19378/27126 objects degraded (71.437%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:06.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:06.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:06.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:06 smithi027 ceph-mon[195233]: pgmap v981: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 568 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 669 op/s; 19378/27126 objects degraded (71.437%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:06.852 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:06.964 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:06.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:07.077 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:07.127 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:07.202 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:07.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:07.207 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:07.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:07.394 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:07.532 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 19378/27126 objects degraded (71.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:07.604 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:07.632 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:07.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:07.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:07.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 19378/27126 objects degraded (71.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 19378/27126 objects degraded (71.437%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:07.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:07.780 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:07.791 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:25:07.791 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:25:07.859 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:07.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:07.872 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:07.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:08.072 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:08] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-09-23T15:25:08.073 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:08.397 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:25:08.398 INFO:teuthology.orchestra.run.smithi017.stdout:{"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 19378/22509 objects degraded (86.090%), 6 pgs degraded, 6 pgs undersized","count":12},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1316,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":331,"num_osds":8,"num_up_osds":8,"osd_up_since":1695482545,"num_in_osds":8,"osd_in_since":1695482516,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":103},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":2}],"num_pgs":109,"num_pools":5,"num_objects":7503,"data_bytes":494659953,"bytes_used":5977030656,"bytes_avail":761983254528,"bytes_total":767960285184,"degraded_objects":19378,"degraded_total":22509,"degraded_ratio":0.86090008441068022,"recovering_objects_per_sec":113,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":1138,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":676},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi017.ygjhom","status":"up:active","gid":14676}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.17:8443/","prometheus":"http://172.21.15.17:9283/"}},"servicemap":{"epoch":25,"modified":"2023-09-23T15:23:24.711679+0000","services":{}},"progress_events":{"0db75cd7-9cf4-44ec-92f9-1c8436eb450e":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"a90e8e5c-cd46-427f-990c-d4f2d8f69255":{"message":"Global Recovery Event (3m)\n [==========================..] (remaining: 13s)","progress":0.94495415687561035,"add_to_ceph_s":true}}} 2023-09-23T15:25:08.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:08 smithi017 ceph-mon[215646]: pgmap v982: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 472 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 19378/22509 objects degraded (86.090%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:08.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3170340937' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:25:08.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:08 smithi017 ceph-mon[222878]: pgmap v982: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 472 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 19378/22509 objects degraded (86.090%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:08.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3170340937' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:25:08.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:08 smithi027 ceph-mon[195233]: pgmap v982: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 472 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 676 op/s; 19378/22509 objects degraded (86.090%); 0 B/s, 113 objects/s recovering 2023-09-23T15:25:08.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3170340937' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T15:25:09.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:25:09.563 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:09 smithi017 ceph-mon[215646]: pgmap v983: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 416 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 19123/19827 objects degraded (96.449%); 0 B/s, 115 objects/s recovering 2023-09-23T15:25:09.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:09.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:09 smithi027 ceph-mon[195233]: pgmap v983: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 416 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 19123/19827 objects degraded (96.449%); 0 B/s, 115 objects/s recovering 2023-09-23T15:25:09.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:09.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:09.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:09 smithi017 ceph-mon[222878]: pgmap v983: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 416 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 680 op/s; 19123/19827 objects degraded (96.449%); 0 B/s, 115 objects/s recovering 2023-09-23T15:25:09.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:10.042 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:10.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:25:10.643 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T15:25:10.644 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:25:10.756 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:10.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:10.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:10.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:10.921 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:11.000 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:11.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:11.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:11.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:11.171 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:11.514 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:11.520 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:11.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:11.724 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:11.739 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:11.787 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:11.830 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:11 smithi017 ceph-mon[222878]: pgmap v984: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 261 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 18765/12402 objects degraded (151.306%); 0 B/s, 118 objects/s recovering 2023-09-23T15:25:11.831 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:11 smithi017 ceph-mon[215646]: pgmap v984: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 261 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 18765/12402 objects degraded (151.306%); 0 B/s, 118 objects/s recovering 2023-09-23T15:25:11.837 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:12.081 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:12.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:12.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:12.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:11 smithi027 ceph-mon[195233]: pgmap v984: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 261 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 679 op/s; 18765/12402 objects degraded (151.306%); 0 B/s, 118 objects/s recovering 2023-09-23T15:25:12.195 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:12.292 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:12.503 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:25:12.554 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:12.562 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:25:12.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:12.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:12.595 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:25:12.637 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:25:12.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:12.684 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:25:12.738 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:25:12.773 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:12.782 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:25:12.854 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:25:12.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.876 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:25:12.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 18765/12402 objects degraded (151.306%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:12.907 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1136921117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:12.908 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 18765/12402 objects degraded (151.306%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:12.908 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1136921117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 18765/12402 objects degraded (151.306%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1136921117' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[215646]: pgmap v985: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 207 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 18511/9804 objects degraded (188.811%); 0 B/s, 117 objects/s recovering 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[222878]: pgmap v985: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 207 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 18511/9804 objects degraded (188.811%); 0 B/s, 117 objects/s recovering 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:14.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:14 smithi027 ceph-mon[195233]: pgmap v985: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 207 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 682 op/s; 18511/9804 objects degraded (188.811%); 0 B/s, 117 objects/s recovering 2023-09-23T15:25:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:15.173 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 90134 objects 2023-09-23T15:25:15.174 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :143.747 2023-09-23T15:25:16.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: pgmap v986: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 18122/6642 objects degraded (272.840%); 0 B/s, 122 objects/s recovering 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[215646]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: pgmap v986: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 18122/6642 objects degraded (272.840%); 0 B/s, 122 objects/s recovering 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T15:25:16.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:16 smithi017 ceph-mon[222878]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: pgmap v986: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 141 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 664 op/s; 18122/6642 objects degraded (272.840%); 0 B/s, 122 objects/s recovering 2023-09-23T15:25:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T15:25:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: failed to encode map e332 with expected crc 2023-09-23T15:25:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:16 smithi027 ceph-mon[195233]: failed to encode map e332 with expected crc 2023-09-23T15:25:17.205 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:17.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:17.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:17.309 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:17.337 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:17.383 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:17.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:17.488 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:17 smithi027 ceph-mon[195233]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:17 smithi027 ceph-mon[195233]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:17 smithi027 ceph-mon[195233]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T15:25:17.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[215646]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[215646]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[215646]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T15:25:17.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[222878]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[222878]: failed to encode map e333 with expected crc 2023-09-23T15:25:17.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:17 smithi017 ceph-mon[222878]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T15:25:17.621 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:17.961 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:17.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:18.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:18.092 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:18.137 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:18.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:18.142 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:18.242 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:18.258 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:18 smithi017 ceph-mon[215646]: pgmap v989: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 53 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 18122/2400 objects degraded (755.083%); 0 B/s, 125 objects/s recovering 2023-09-23T15:25:18.258 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:18 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 18122/2400 objects degraded (755.083%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:18.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:18 smithi017 ceph-mon[222878]: pgmap v989: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 53 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 18122/2400 objects degraded (755.083%); 0 B/s, 125 objects/s recovering 2023-09-23T15:25:18.258 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:18 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 18122/2400 objects degraded (755.083%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:18.259 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:18] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T15:25:18.278 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:18 smithi027 ceph-mon[195233]: pgmap v989: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 53 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 725 op/s; 18122/2400 objects degraded (755.083%); 0 B/s, 125 objects/s recovering 2023-09-23T15:25:18.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:18 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 18122/2400 objects degraded (755.083%), 6 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:18.581 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:18.585 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:18.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:152397 7.9 7:9f18f3c0:::benchmark_data_smithi017_422824_object58527:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:56.198451+0000", 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 322.37270334599998, 2023-09-23T15:25:18.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91843015699999997, 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 152397 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198451+0000", 2023-09-23T15:25:18.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198451+0000", 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198450+0000", 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198453+0000", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0199999999999999e-06 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198460+0000", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5540000000000003e-06 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.198465+0000", 2023-09-23T15:25:18.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4179999999999996e-06 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.211044+0000", 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012578529 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:56.211054+0000", 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0274e-05 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.111924+0000", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.900869904 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.112049+0000", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012552700000000001 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.112646+0000", 2023-09-23T15:25:18.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00059622600000000005 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.116690+0000", 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040439600000000001 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.116745+0000", 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5689999999999997e-05 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.116757+0000", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.188e-05 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.116810+0000", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2986999999999999e-05 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:57.116881+0000", 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0619999999999998e-05 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.616 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161416 7.4 7:23fd1d4d:::benchmark_data_smithi017_422824_object67546:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.148630266, 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2737346510000001, 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161416 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422522+0000", 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422531+0000", 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.473e-06 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422551+0000", 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0579000000000001e-05 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422561+0000", 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4590000000000004e-06 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422648+0000", 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7039000000000005e-05 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422805+0000", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157534 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422938+0000", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013334499999999999 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.426632+0000", 2023-09-23T15:25:18.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036934110000000002 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.446635+0000", 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020002678999999999 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.446646+0000", 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1630999999999999e-05 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689234+0000", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2425875829999999 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689345+0000", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111606 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696168+0000", 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068226500000000004 2023-09-23T15:25:18.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696258+0000", 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0245000000000001e-05 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161431 7.4 7:211b85c0:::benchmark_data_smithi017_422824_object67561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:25:18.624 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.13243118499997, 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2578971299999999, 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161431 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:25:18.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438721+0000", 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438728+0000", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2910000000000004e-06 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438740+0000", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2554e-05 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438749+0000", 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6279999999999994e-06 2023-09-23T15:25:18.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438939+0000", 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018970900000000001 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.439081+0000", 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014192400000000001 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2023-09-23T15:25:18.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.439213+0000", 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132261 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.447521+0000", 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083078590000000008 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.447530+0000", 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4390000000000006e-06 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689610+0000", 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2420803679999999 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689711+0000", 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010036500000000001 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696563+0000", 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068518750000000003 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696620+0000", 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.715e-05 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161437 7.4 7:2481e772:::benchmark_data_smithi017_422824_object67567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:25:18.631 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.12086191899999, 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2465200679999999, 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161437 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450295+0000", 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0510000000000001e-06 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450301+0000", 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4320000000000004e-06 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450309+0000", 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7349999999999996e-06 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450363+0000", 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4568999999999999e-05 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450373+0000", 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689746+0000", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2393721659999999 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689844+0000", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7947000000000003e-05 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689939+0000", 2023-09-23T15:25:18.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5031999999999994e-05 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.695831+0000", 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058926730000000002 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696129+0000", 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029764899999999998 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696672+0000", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054306700000000005 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696721+0000", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9468000000000001e-05 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696812+0000", 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0595999999999998e-05 2023-09-23T15:25:18.638 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161441 7.4 7:2b51fbee:::benchmark_data_smithi017_422824_object67571:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.11021259900002, 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2360482610000001, 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.639 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161441 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:25:18.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460940+0000", 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460947+0000", 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460965+0000", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7609e-05 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460973+0000", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4910000000000005e-06 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.461022+0000", 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9020000000000002e-05 2023-09-23T15:25:18.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.461032+0000", 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8660000000000001e-06 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692245+0000", 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2312128790000001 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692324+0000", 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9361000000000001e-05 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692410+0000", 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5955e-05 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.695969+0000", 2023-09-23T15:25:18.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003558912 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696133+0000", 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000163463 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696869+0000", 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073619500000000004 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696912+0000", 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3534e-05 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696990+0000", 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7205000000000004e-05 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.646 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161456 7.4 7:2e1a1183:::benchmark_data_smithi017_422824_object67586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.09228355900001, 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2247869389999999, 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161456 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478869+0000", 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478876+0000", 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.109e-06 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478888+0000", 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2215000000000001e-05 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478895+0000", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3579999999999999e-06 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478940+0000", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5683999999999998e-05 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478951+0000", 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0312000000000001e-05 2023-09-23T15:25:18.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694102+0000", 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2151512950000001 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694202+0000", 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100055 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694274+0000", 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2175000000000006e-05 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696135+0000", 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018612909999999999 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696295+0000", 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015990700000000001 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703512+0000", 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072162390000000002 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703565+0000", 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3285000000000002e-05 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703657+0000", 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2373000000000005e-05 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161460 7.4 7:2562121b:::benchmark_data_smithi017_422824_object67590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 305.08483752799998, 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.226105368, 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.654 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161460 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486314+0000", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486321+0000", 2023-09-23T15:25:18.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8519999999999998e-06 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486333+0000", 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2021000000000001e-05 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486341+0000", 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7870000000000002e-06 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486430+0000", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8721999999999995e-05 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486440+0000", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8989999999999996e-06 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697044+0000", 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.210603841 2023-09-23T15:25:18.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697146+0000", 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102636 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697263+0000", 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011694299999999999 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706589+0000", 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0093257740000000002 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706638+0000", 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9157999999999999e-05 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712250+0000", 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056120750000000002 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712305+0000", 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5049000000000001e-05 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712422+0000", 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116574 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161519 7.4 7:220f2d51:::benchmark_data_smithi017_422824_object67649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.98550040100002, 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1269551660000001, 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161519 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.662 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585653+0000", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585656+0000", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1020000000000001e-06 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585663+0000", 2023-09-23T15:25:18.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4749999999999998e-06 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585668+0000", 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6969999999999998e-06 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585721+0000", 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3004999999999998e-05 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585731+0000", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7520000000000008e-06 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700079+0000", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.114347703 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700199+0000", 2023-09-23T15:25:18.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120475 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700331+0000", 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013203800000000001 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706596+0000", 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062643619999999999 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706677+0000", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1079000000000001e-05 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712483+0000", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058062499999999998 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712525+0000", 2023-09-23T15:25:18.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1771999999999999e-05 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712609+0000", 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3947000000000001e-05 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161524 7.4 7:2a8e46ee:::benchmark_data_smithi017_422824_object67654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:25:18.669 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.97438717900002, 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.116073053, 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161524 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596766+0000", 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596771+0000", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.335e-06 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596777+0000", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9299999999999997e-06 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596783+0000", 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:25:18.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596883+0000", 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9469999999999995e-05 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596893+0000", 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703705+0000", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1068115599999999 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703843+0000", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013830099999999999 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703988+0000", 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000145492 2023-09-23T15:25:18.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712149+0000", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081608110000000005 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712160+0000", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1076e-05 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712690+0000", 2023-09-23T15:25:18.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052939600000000001 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712745+0000", 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5758999999999999e-05 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712840+0000", 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4439999999999997e-05 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.676 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161528 7.4 7:2096f0c5:::benchmark_data_smithi017_422824_object67658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.965582531, 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.114153634, 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161528 2023-09-23T15:25:18.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605575+0000", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9169999999999999e-06 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605582+0000", 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8940000000000004e-06 2023-09-23T15:25:18.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605588+0000", 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.908e-06 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605644+0000", 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6863000000000001e-05 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605654+0000", 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5279999999999992e-06 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706705+0000", 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1010514419999999 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706832+0000", 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012622100000000001 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706973+0000", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014145499999999999 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712190+0000", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052171300000000004 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712441+0000", 2023-09-23T15:25:18.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025033599999999998 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719593+0000", 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071525429999999999 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719634+0000", 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0735999999999998e-05 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719725+0000", 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1137999999999994e-05 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161573 7.4 7:2a9df9eb:::benchmark_data_smithi017_422824_object67703:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.709099+0000", 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.86205503899998, 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.010777236, 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.684 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161573 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709099+0000", 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709099+0000", 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709097+0000", 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709107+0000", 2023-09-23T15:25:18.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0165e-05 2023-09-23T15:25:18.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709129+0000", 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1552999999999998e-05 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709142+0000", 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3220999999999999e-05 2023-09-23T15:25:18.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709201+0000", 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8721000000000002e-05 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.709215+0000", 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4399999999999999e-05 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.709696+0000", 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.00048074 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.709789+0000", 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2899000000000006e-05 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.709862+0000", 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3400999999999997e-05 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712622+0000", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027595279999999998 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719401+0000", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067791429999999996 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719760+0000", 2023-09-23T15:25:18.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00035954200000000002 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719796+0000", 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5725000000000001e-05 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719876+0000", 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0159000000000003e-05 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161606 7.4 7:2acff49d:::benchmark_data_smithi017_422824_object67736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.773298+0000", 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.79785583, 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94672692199999997, 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161606 2023-09-23T15:25:18.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773298+0000", 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773298+0000", 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773296+0000", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773305+0000", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6600000000000001e-06 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773328+0000", 2023-09-23T15:25:18.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3003999999999999e-05 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773339+0000", 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0764000000000001e-05 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773415+0000", 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5771000000000003e-05 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.773425+0000", 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0297e-05 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712895+0000", 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.93947016000000005 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.713035+0000", 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140419 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.713165+0000", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013000899999999999 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719443+0000", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006277113 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719456+0000", 2023-09-23T15:25:18.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3339999999999999e-05 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719913+0000", 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00045656499999999998 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719946+0000", 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3892e-05 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.720025+0000", 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8591999999999996e-05 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161612 7.4 7:26f4a9ce:::benchmark_data_smithi017_422824_object67742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.784331+0000", 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.78682267599999, 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.94524195600000005, 2023-09-23T15:25:18.701 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161612 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784331+0000", 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784331+0000", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784329+0000", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784337+0000", 2023-09-23T15:25:18.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4569999999999999e-06 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784353+0000", 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6543999999999999e-05 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784362+0000", 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8540000000000003e-06 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784893+0000", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00053063999999999997 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.784902+0000", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6209999999999993e-06 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.716190+0000", 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.93128811 2023-09-23T15:25:18.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.716309+0000", 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011893 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.716447+0000", 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013836100000000001 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719562+0000", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031149810000000002 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729353+0000", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0097905430000000005 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729468+0000", 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000114997 2023-09-23T15:25:18.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729515+0000", 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7596000000000003e-05 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729573+0000", 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8082999999999999e-05 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.708 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161625 7.4 7:2e08d1db:::benchmark_data_smithi017_422824_object67755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.807467+0000", 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.76368682700002, 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92232260799999999, 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161625 2023-09-23T15:25:18.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807467+0000", 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807467+0000", 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807465+0000", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807472+0000", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.515e-06 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807487+0000", 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4785e-05 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807498+0000", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0369e-05 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807555+0000", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7689000000000003e-05 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807565+0000", 2023-09-23T15:25:18.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8919999999999995e-06 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.720051+0000", 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91248618299999995 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.720162+0000", 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011045899999999999 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.720282+0000", 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120265 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729360+0000", 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0090781500000000001 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729423+0000", 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2774000000000006e-05 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729646+0000", 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022265699999999999 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729698+0000", 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1879999999999998e-05 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729790+0000", 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2085999999999999e-05 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161626 7.4 7:292fe5f4:::benchmark_data_smithi017_422824_object67756:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.807530+0000", 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 304.76362374299998, 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92251957699999998, 2023-09-23T15:25:18.716 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161626 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807530+0000", 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.717 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807530+0000", 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807529+0000", 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.718 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807534+0000", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1729999999999997e-06 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807543+0000", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4909999999999994e-06 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807552+0000", 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0960000000000007e-06 2023-09-23T15:25:18.719 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807598+0000", 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6034e-05 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.807608+0000", 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3060000000000003e-06 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.720 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.723505+0000", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91589744299999998 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.723628+0000", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012332000000000001 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.723756+0000", 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012752600000000001 2023-09-23T15:25:18.721 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729587+0000", 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00583162 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729860+0000", 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027291899999999999 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.722 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729915+0000", 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4873999999999997e-05 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.729959+0000", 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4032000000000001e-05 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.730050+0000", 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0443000000000001e-05 2023-09-23T15:25:18.723 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50588 8.e 8:73911130:::benchmark_data_smithi017_485823_object50587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.36755574399999, 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.107695917, 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.724 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50588 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:25:18.725 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203597+0000", 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203626+0000", 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9794999999999998e-05 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.726 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203639+0000", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2792999999999999e-05 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203648+0000", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0340000000000002e-06 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203778+0000", 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129462 2023-09-23T15:25:18.727 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203789+0000", 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1683e-05 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123801+0000", 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92001208899999998 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.728 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123813+0000", 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1676e-05 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301009+0000", 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.177195422 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301190+0000", 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018111599999999999 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.729 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301356+0000", 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166479 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307313+0000", 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059565829999999997 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.730 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307350+0000", 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6908e-05 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309727+0000", 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023774629999999998 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309803+0000", 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6243999999999997e-05 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.731 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311294+0000", 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014907830000000001 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50599 8.e 8:730ce43d:::benchmark_data_smithi017_485823_object50598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.35932924599999, 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1058259499999998, 2023-09-23T15:25:18.732 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50599 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.733 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211823+0000", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211849+0000", 2023-09-23T15:25:18.734 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6114000000000001e-05 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211866+0000", 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6600999999999999e-05 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211874+0000", 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3229999999999998e-06 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.735 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211940+0000", 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6039999999999998e-05 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211958+0000", 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7448e-05 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123836+0000", 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91187878 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.736 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123844+0000", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9489999999999996e-06 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.304978+0000", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1811335549999999 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305174+0000", 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196164 2023-09-23T15:25:18.737 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305380+0000", 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000206092 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311368+0000", 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059874079999999996 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.738 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311447+0000", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9140000000000005e-05 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315621+0000", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041739070000000001 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315691+0000", 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0588999999999998e-05 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.739 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317651+0000", 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001959391 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50609 8.e 8:797f5f67:::benchmark_data_smithi017_485823_object50608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.34925805899999, 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.097560632, 2023-09-23T15:25:18.740 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50609 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.741 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221894+0000", 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.742 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221919+0000", 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5066999999999999e-05 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221946+0000", 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7331e-05 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221959+0000", 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.269e-05 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.743 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222024+0000", 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4824000000000001e-05 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222038+0000", 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3788000000000001e-05 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.744 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123868+0000", 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90182994900000002 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123876+0000", 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2979999999999992e-06 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307446+0000", 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183570029 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.745 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307604+0000", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015765000000000001 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307772+0000", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168114 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311434+0000", 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036626810000000001 2023-09-23T15:25:18.746 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311453+0000", 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9004e-05 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317944+0000", 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064906199999999999 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.747 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317998+0000", 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4296999999999999e-05 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319456+0000", 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014581900000000001 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.748 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50596 8.2 8:4072d885:::benchmark_data_smithi017_485823_object50595:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 201.45020770100001, 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1658852479999999, 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50596 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.749 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120944+0000", 2023-09-23T15:25:18.750 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120979+0000", 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4545000000000002e-05 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121002+0000", 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2994e-05 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.751 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121014+0000", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.219e-05 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121135+0000", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012048399999999999 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121143+0000", 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2023-09-23T15:25:18.752 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123756+0000", 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026131420000000002 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123766+0000", 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0566e-05 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.753 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275228+0000", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1514619580000001 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275358+0000", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129383 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275499+0000", 2023-09-23T15:25:18.754 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141419 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279452+0000", 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039528280000000002 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279468+0000", 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5583e-05 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.755 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285296+0000", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005828415 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285337+0000", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1216e-05 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.286831+0000", 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014941799999999999 2023-09-23T15:25:18.756 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50597 8.2 8:4463d1ca:::benchmark_data_smithi017_485823_object50596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 201.450120591, 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.173769987, 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:18.757 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50597 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:25:18.758 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121059+0000", 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6764999999999999e-05 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.759 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121068+0000", 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6510000000000007e-06 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121075+0000", 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6290000000000004e-06 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.760 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121210+0000", 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135697 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121216+0000", 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2360000000000004e-06 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126405+0000", 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051892259999999999 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.761 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126415+0000", 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0358e-05 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.277883+0000", 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.151468277 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:18.762 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278066+0000", 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018207400000000001 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278266+0000", 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020090099999999999 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279562+0000", 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001295825 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.763 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.281149+0000", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00158647 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293587+0000", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012437981000000001 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293648+0000", 2023-09-23T15:25:18.764 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0943000000000002e-05 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.294803+0000", 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011556100000000001 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:18.765 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:25:18.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:18.800 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:19.208 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:25:19.261 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T15:25:19.262 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T15:25:19.592 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:25:19.594 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:25:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:25:19.618 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:25:19.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:25:19.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:25:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: pgmap v990: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 20 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 17338/822 objects degraded (2109.246%); 0 B/s, 178 objects/s recovering 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[215646]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: pgmap v990: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 20 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 17338/822 objects degraded (2109.246%); 0 B/s, 178 objects/s recovering 2023-09-23T15:25:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:20.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:25:20.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:20 smithi017 ceph-mon[222878]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T15:25:20.532 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:20.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: pgmap v990: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 20 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 17338/822 objects degraded (2109.246%); 0 B/s, 178 objects/s recovering 2023-09-23T15:25:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: failed to encode map e334 with expected crc 2023-09-23T15:25:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1266756322' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:25:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:20 smithi027 ceph-mon[195233]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T15:25:20.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:20.665 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:21.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:25:21.318 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:21.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:21.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:21.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:21.451 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:21.471 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:21 smithi017 ceph-mon[215646]: from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:21.471 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:21 smithi017 ceph-mon[222878]: from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:21.498 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:21.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:21 smithi027 ceph-mon[195233]: from='client.36159 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:21.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:21.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:21.631 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:21.677 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:21.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:21.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:21.786 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:21.816 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:22.180 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":991,"stamp":"2023-09-23T15:25:19.203251+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":20900209,"num_objects":274,"num_object_clones":0,"num_object_copies":822,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":274,"num_whiteouts":0,"num_read":6040,"num_read_kb":13296,"num_write":180953,"num_write_kb":5781522,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18652,"num_bytes_recovered":776113525,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113937,"ondisk_log_size":113937,"up":327,"acting":321,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":342,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4428928,"kb_used_data":1177664,"kb_used_omap":44,"kb_used_meta":3251027,"kb_avail":745532288,"statfs":{"total":767960285184,"available":763425062912,"internally_reserved":0,"allocated":1205927936,"data_stored":1196025548,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3329052603},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-196214784,"num_objects":-2994,"num_object_clones":0,"num_object_copies":-8982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1173,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-2994,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2994,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1173,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.438890"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367036+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:24:50.367036+0000","last_peered":"2023-09-23T15:24:50.367036+0000","last_clean":"2023-09-23T15:24:50.367036+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:24:50.367036+0000","last_undegraded":"2023-09-23T15:24:50.367036+0000","last_fullsized":"2023-09-23T15:24:50.367036+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":288,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.528208+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:25:04.528208+0000","last_peered":"2023-09-23T15:25:04.528208+0000","last_clean":"2023-09-23T15:25:04.528208+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:25:04.528208+0000","last_undegraded":"2023-09-23T15:25:04.528208+0000","last_fullsized":"2023-09-23T15:25:04.528208+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":238,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.197340+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:25:15.197340+0000","last_peered":"2023-09-23T15:25:15.197340+0000","last_clean":"2023-09-23T15:25:15.197340+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:25:15.197340+0000","last_undegraded":"2023-09-23T15:25:15.197340+0000","last_fullsized":"2023-09-23T15:25:15.197340+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":310,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380529+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:24:50.380529+0000","last_peered":"2023-09-23T15:24:50.380529+0000","last_clean":"2023-09-23T15:24:50.380529+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:24:50.380529+0000","last_undegraded":"2023-09-23T15:24:50.380529+0000","last_fullsized":"2023-09-23T15:24:50.380529+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":333,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527718+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:25:04.527718+0000","last_peered":"2023-09-23T15:25:04.527718+0000","last_clean":"2023-09-23T15:25:04.527718+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:25:04.527718+0000","last_undegraded":"2023-09-23T15:25:04.527718+0000","last_fullsized":"2023-09-23T15:25:04.527718+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":281,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.371427+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:24:50.371427+0000","last_peered":"2023-09-23T15:24:50.371427+0000","last_clean":"2023-09-23T15:24:50.371427+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:24:50.371427+0000","last_undegraded":"2023-09-23T15:24:50.371427+0000","last_fullsized":"2023-09-23T15:24:50.371427+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":279,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.382484+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:24:50.382484+0000","last_peered":"2023-09-23T15:24:50.382484+0000","last_clean":"2023-09-23T15:24:50.382484+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:24:50.382484+0000","last_undegraded":"2023-09-23T15:24:50.382484+0000","last_fullsized":"2023-09-23T15:24:50.382484+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":331,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527795+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:25:04.527795+0000","last_peered":"2023-09-23T15:25:04.527795+0000","last_clean":"2023-09-23T15:25:04.527795+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:25:04.527795+0000","last_undegraded":"2023-09-23T15:25:04.527795+0000","last_fullsized":"2023-09-23T15:25:04.527795+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":689,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.375982+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:24:50.375982+0000","last_peered":"2023-09-23T15:24:50.375982+0000","last_clean":"2023-09-23T15:24:50.375982+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:24:50.375982+0000","last_undegraded":"2023-09-23T15:24:50.375982+0000","last_fullsized":"2023-09-23T15:24:50.375982+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":331,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388100+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:24:50.388100+0000","last_peered":"2023-09-23T15:24:50.388100+0000","last_clean":"2023-09-23T15:24:50.388100+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:24:50.388100+0000","last_undegraded":"2023-09-23T15:24:50.388100+0000","last_fullsized":"2023-09-23T15:24:50.388100+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":285,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527645+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:25:04.527645+0000","last_peered":"2023-09-23T15:25:04.527645+0000","last_clean":"2023-09-23T15:25:04.527645+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:25:04.527645+0000","last_undegraded":"2023-09-23T15:25:04.527645+0000","last_fullsized":"2023-09-23T15:25:04.527645+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":75,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380223+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:24:50.380223+0000","last_peered":"2023-09-23T15:24:50.380223+0000","last_clean":"2023-09-23T15:24:50.380223+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:24:50.380223+0000","last_undegraded":"2023-09-23T15:24:50.380223+0000","last_fullsized":"2023-09-23T15:24:50.380223+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":224,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.197213+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:25:15.197213+0000","last_peered":"2023-09-23T15:25:15.197213+0000","last_clean":"2023-09-23T15:25:15.197213+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:25:15.197213+0000","last_undegraded":"2023-09-23T15:25:15.197213+0000","last_fullsized":"2023-09-23T15:25:15.197213+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":65,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368459+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:24:50.368459+0000","last_peered":"2023-09-23T15:24:50.368459+0000","last_clean":"2023-09-23T15:24:50.368459+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:24:50.368459+0000","last_undegraded":"2023-09-23T15:24:50.368459+0000","last_fullsized":"2023-09-23T15:24:50.368459+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367706+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:24:50.367706+0000","last_peered":"2023-09-23T15:24:50.367706+0000","last_clean":"2023-09-23T15:24:50.367706+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:24:50.367706+0000","last_undegraded":"2023-09-23T15:24:50.367706+0000","last_fullsized":"2023-09-23T15:24:50.367706+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":640,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380447+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:24:50.380447+0000","last_peered":"2023-09-23T15:24:50.380447+0000","last_clean":"2023-09-23T15:24:50.380447+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:24:50.380447+0000","last_undegraded":"2023-09-23T15:24:50.380447+0000","last_fullsized":"2023-09-23T15:24:50.380447+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":79,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379389+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:24:50.379389+0000","last_peered":"2023-09-23T15:24:50.379389+0000","last_clean":"2023-09-23T15:24:50.379389+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:24:50.379389+0000","last_undegraded":"2023-09-23T15:24:50.379389+0000","last_fullsized":"2023-09-23T15:24:50.379389+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":336,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527396+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:25:04.527396+0000","last_peered":"2023-09-23T15:25:04.527396+0000","last_clean":"2023-09-23T15:25:04.527396+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:25:04.527396+0000","last_undegraded":"2023-09-23T15:25:04.527396+0000","last_fullsized":"2023-09-23T15:25:04.527396+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":689,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.391006+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:24:50.391006+0000","last_peered":"2023-09-23T15:24:50.391006+0000","last_clean":"2023-09-23T15:24:50.391006+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:24:50.391006+0000","last_undegraded":"2023-09-23T15:24:50.391006+0000","last_fullsized":"2023-09-23T15:24:50.391006+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":308,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.377107+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:24:50.377107+0000","last_peered":"2023-09-23T15:24:50.377107+0000","last_clean":"2023-09-23T15:24:50.377107+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:24:50.377107+0000","last_undegraded":"2023-09-23T15:24:50.377107+0000","last_fullsized":"2023-09-23T15:24:50.377107+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":316,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388790+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:24:50.388790+0000","last_peered":"2023-09-23T15:24:50.388790+0000","last_clean":"2023-09-23T15:24:50.388790+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:24:50.388790+0000","last_undegraded":"2023-09-23T15:24:50.388790+0000","last_fullsized":"2023-09-23T15:24:50.388790+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":623,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.199344+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:25:15.199344+0000","last_peered":"2023-09-23T15:25:15.199344+0000","last_clean":"2023-09-23T15:25:15.199344+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:25:15.199344+0000","last_undegraded":"2023-09-23T15:25:15.199344+0000","last_fullsized":"2023-09-23T15:25:15.199344+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":79,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.378958+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:24:50.378958+0000","last_peered":"2023-09-23T15:24:50.378958+0000","last_clean":"2023-09-23T15:24:50.378958+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:24:50.378958+0000","last_undegraded":"2023-09-23T15:24:50.378958+0000","last_fullsized":"2023-09-23T15:24:50.378958+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":297,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380595+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:24:50.380595+0000","last_peered":"2023-09-23T15:24:50.380595+0000","last_clean":"2023-09-23T15:24:50.380595+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:24:50.380595+0000","last_undegraded":"2023-09-23T15:24:50.380595+0000","last_fullsized":"2023-09-23T15:24:50.380595+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":647,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388122+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:24:50.388122+0000","last_peered":"2023-09-23T15:24:50.388122+0000","last_clean":"2023-09-23T15:24:50.388122+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:24:50.388122+0000","last_undegraded":"2023-09-23T15:24:50.388122+0000","last_fullsized":"2023-09-23T15:24:50.388122+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":267,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368582+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:24:50.368582+0000","last_peered":"2023-09-23T15:24:50.368582+0000","last_clean":"2023-09-23T15:24:50.368582+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:24:50.368582+0000","last_undegraded":"2023-09-23T15:24:50.368582+0000","last_fullsized":"2023-09-23T15:24:50.368582+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":335,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.528488+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:25:04.528488+0000","last_peered":"2023-09-23T15:25:04.528488+0000","last_clean":"2023-09-23T15:25:04.528488+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:25:04.528488+0000","last_undegraded":"2023-09-23T15:25:04.528488+0000","last_fullsized":"2023-09-23T15:25:04.528488+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.4","version":"331'11398","reported_seq":17956,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:14.883107+0000","last_change":"2023-09-23T15:24:27.826512+0000","last_active":"2023-09-23T15:25:14.883107+0000","last_peered":"2023-09-23T15:25:14.883107+0000","last_clean":"2023-09-23T15:25:14.883107+0000","last_became_active":"2023-09-23T15:24:27.764262+0000","last_became_peered":"2023-09-23T15:24:27.764262+0000","last_unstale":"2023-09-23T15:25:14.883107+0000","last_undegraded":"2023-09-23T15:25:14.883107+0000","last_fullsized":"2023-09-23T15:25:14.883107+0000","mapping_epoch":327,"log_start":"300'3500","ondisk_log_start":"300'3500","created":284,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7898,"log_dups_size":0,"ondisk_log_size":7898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:39.059424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1048576,"num_objects":16,"num_object_clones":0,"num_object_copies":48,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":16,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11398,"num_write_kb":365248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":87425024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":593,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380343+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:24:50.380343+0000","last_peered":"2023-09-23T15:24:50.380343+0000","last_clean":"2023-09-23T15:24:50.380343+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:24:50.380343+0000","last_undegraded":"2023-09-23T15:24:50.380343+0000","last_fullsized":"2023-09-23T15:24:50.380343+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":71,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380023+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:24:50.380023+0000","last_peered":"2023-09-23T15:24:50.380023+0000","last_clean":"2023-09-23T15:24:50.380023+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:24:50.380023+0000","last_undegraded":"2023-09-23T15:24:50.380023+0000","last_fullsized":"2023-09-23T15:24:50.380023+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":323,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.390794+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:24:50.390794+0000","last_peered":"2023-09-23T15:24:50.390794+0000","last_clean":"2023-09-23T15:24:50.390794+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:24:50.390794+0000","last_undegraded":"2023-09-23T15:24:50.390794+0000","last_fullsized":"2023-09-23T15:24:50.390794+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"331'11053","reported_seq":20318,"reported_epoch":331,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:14.360748+0000","last_change":"2023-09-23T15:22:27.563481+0000","last_active":"2023-09-23T15:25:14.360748+0000","last_peered":"2023-09-23T15:25:14.360748+0000","last_clean":"2023-09-23T15:21:56.129057+0000","last_became_active":"2023-09-23T15:22:27.546944+0000","last_became_peered":"2023-09-23T15:22:27.546944+0000","last_unstale":"2023-09-23T15:25:14.360748+0000","last_undegraded":"2023-09-23T15:22:27.429573+0000","last_fullsized":"2023-09-23T15:22:27.290610+0000","mapping_epoch":305,"log_start":"301'4200","ondisk_log_start":"301'4200","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6853,"log_dups_size":0,"ondisk_log_size":6853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:12.596152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1484,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11053,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[7,4],"avail_no_missing":["7","0","1","4"],"object_location_counts":[{"shards":"4,7","objects":-1463}],"blocked_by":[],"up_primary":6,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":652,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527309+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:25:04.527309+0000","last_peered":"2023-09-23T15:25:04.527309+0000","last_clean":"2023-09-23T15:25:04.527309+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:25:04.527309+0000","last_undegraded":"2023-09-23T15:25:04.527309+0000","last_fullsized":"2023-09-23T15:25:04.527309+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":71,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379500+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:24:50.379500+0000","last_peered":"2023-09-23T15:24:50.379500+0000","last_clean":"2023-09-23T15:24:50.379500+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:24:50.379500+0000","last_undegraded":"2023-09-23T15:24:50.379500+0000","last_fullsized":"2023-09-23T15:24:50.379500+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367104+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:24:50.367104+0000","last_peered":"2023-09-23T15:24:50.367104+0000","last_clean":"2023-09-23T15:24:50.367104+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:24:50.367104+0000","last_undegraded":"2023-09-23T15:24:50.367104+0000","last_fullsized":"2023-09-23T15:24:50.367104+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"331'11175","reported_seq":11251,"reported_epoch":331,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:14.260774+0000","last_change":"2023-09-23T15:22:27.584050+0000","last_active":"2023-09-23T15:25:14.260774+0000","last_peered":"2023-09-23T15:25:14.260774+0000","last_clean":"2023-09-23T15:22:25.311750+0000","last_became_active":"2023-09-23T15:22:27.575476+0000","last_became_peered":"2023-09-23T15:22:27.575476+0000","last_unstale":"2023-09-23T15:25:14.260774+0000","last_undegraded":"2023-09-23T15:22:27.407406+0000","last_fullsized":"2023-09-23T15:22:27.292618+0000","mapping_epoch":305,"log_start":"303'4600","ondisk_log_start":"303'4600","created":284,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6575,"log_dups_size":0,"ondisk_log_size":6575,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:31.957863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2555904,"num_objects":39,"num_object_clones":0,"num_object_copies":117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":39,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11175,"num_write_kb":358848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,7],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,7","objects":-4591}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":538,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.371511+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:24:50.371511+0000","last_peered":"2023-09-23T15:24:50.371511+0000","last_clean":"2023-09-23T15:24:50.371511+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:24:50.371511+0000","last_undegraded":"2023-09-23T15:24:50.371511+0000","last_fullsized":"2023-09-23T15:24:50.371511+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":71,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380174+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:24:50.380174+0000","last_peered":"2023-09-23T15:24:50.380174+0000","last_clean":"2023-09-23T15:24:50.380174+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:24:50.380174+0000","last_undegraded":"2023-09-23T15:24:50.380174+0000","last_fullsized":"2023-09-23T15:24:50.380174+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":294,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368891+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:24:50.368891+0000","last_peered":"2023-09-23T15:24:50.368891+0000","last_clean":"2023-09-23T15:24:50.368891+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:24:50.368891+0000","last_undegraded":"2023-09-23T15:24:50.368891+0000","last_fullsized":"2023-09-23T15:24:50.368891+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"331'11312","reported_seq":16935,"reported_epoch":333,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:17.088552+0000","last_change":"2023-09-23T15:24:25.988237+0000","last_active":"2023-09-23T15:25:17.088552+0000","last_peered":"2023-09-23T15:25:17.088552+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574808+0000","last_became_peered":"2023-09-23T15:22:27.574808+0000","last_unstale":"2023-09-23T15:25:17.088552+0000","last_undegraded":"2023-09-23T15:22:27.322626+0000","last_fullsized":"2023-09-23T15:22:27.290673+0000","mapping_epoch":305,"log_start":"306'4700","ondisk_log_start":"306'4700","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6612,"log_dups_size":0,"ondisk_log_size":6612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:46.767737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1877,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11312,"num_write_kb":361984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2779,"num_bytes_recovered":62521344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":-1877}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":546,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368108+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:24:50.368108+0000","last_peered":"2023-09-23T15:24:50.368108+0000","last_clean":"2023-09-23T15:24:50.368108+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:24:50.368108+0000","last_undegraded":"2023-09-23T15:24:50.368108+0000","last_fullsized":"2023-09-23T15:24:50.368108+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":307,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.389178+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:24:50.389178+0000","last_peered":"2023-09-23T15:24:50.389178+0000","last_clean":"2023-09-23T15:24:50.389178+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:24:50.389178+0000","last_undegraded":"2023-09-23T15:24:50.389178+0000","last_fullsized":"2023-09-23T15:24:50.389178+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":313,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:54.239678+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:24:54.239678+0000","last_peered":"2023-09-23T15:24:54.239678+0000","last_clean":"2023-09-23T15:24:54.239678+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:24:54.239678+0000","last_undegraded":"2023-09-23T15:24:54.239678+0000","last_fullsized":"2023-09-23T15:24:54.239678+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"331'11274","reported_seq":11333,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.417809+0000","last_change":"2023-09-23T15:21:12.223147+0000","last_active":"2023-09-23T15:25:15.417809+0000","last_peered":"2023-09-23T15:25:15.417809+0000","last_clean":"2023-09-23T15:25:15.417809+0000","last_became_active":"2023-09-23T15:21:12.222843+0000","last_became_peered":"2023-09-23T15:21:12.222843+0000","last_unstale":"2023-09-23T15:25:15.417809+0000","last_undegraded":"2023-09-23T15:25:15.417809+0000","last_fullsized":"2023-09-23T15:25:15.417809+0000","mapping_epoch":284,"log_start":"306'4700","ondisk_log_start":"306'4700","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6574,"log_dups_size":0,"ondisk_log_size":6574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:44:24.673704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11274,"num_write_kb":360768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,1],"acting":[3,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":567,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379182+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:24:50.379182+0000","last_peered":"2023-09-23T15:24:50.379182+0000","last_clean":"2023-09-23T15:24:50.379182+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:24:50.379182+0000","last_undegraded":"2023-09-23T15:24:50.379182+0000","last_fullsized":"2023-09-23T15:24:50.379182+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":325,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527840+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:25:04.527840+0000","last_peered":"2023-09-23T15:25:04.527840+0000","last_clean":"2023-09-23T15:25:04.527840+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:25:04.527840+0000","last_undegraded":"2023-09-23T15:25:04.527840+0000","last_fullsized":"2023-09-23T15:25:04.527840+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368332+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:24:50.368332+0000","last_peered":"2023-09-23T15:24:50.368332+0000","last_clean":"2023-09-23T15:24:50.368332+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:24:50.368332+0000","last_undegraded":"2023-09-23T15:24:50.368332+0000","last_fullsized":"2023-09-23T15:24:50.368332+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"331'11542","reported_seq":11601,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.197311+0000","last_change":"2023-09-23T15:21:12.276313+0000","last_active":"2023-09-23T15:25:15.197311+0000","last_peered":"2023-09-23T15:25:15.197311+0000","last_clean":"2023-09-23T15:25:15.197311+0000","last_became_active":"2023-09-23T15:21:12.276013+0000","last_became_peered":"2023-09-23T15:21:12.276013+0000","last_unstale":"2023-09-23T15:25:15.197311+0000","last_undegraded":"2023-09-23T15:25:15.197311+0000","last_fullsized":"2023-09-23T15:25:15.197311+0000","mapping_epoch":284,"log_start":"307'5000","ondisk_log_start":"307'5000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6542,"log_dups_size":0,"ondisk_log_size":6542,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.917385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11542,"num_write_kb":369344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":635,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.391075+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:24:50.391075+0000","last_peered":"2023-09-23T15:24:50.391075+0000","last_clean":"2023-09-23T15:24:50.391075+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:24:50.391075+0000","last_undegraded":"2023-09-23T15:24:50.391075+0000","last_fullsized":"2023-09-23T15:24:50.391075+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":278,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368984+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:24:50.368984+0000","last_peered":"2023-09-23T15:24:50.368984+0000","last_clean":"2023-09-23T15:24:50.368984+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:24:50.368984+0000","last_undegraded":"2023-09-23T15:24:50.368984+0000","last_fullsized":"2023-09-23T15:24:50.368984+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367828+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:24:50.367828+0000","last_peered":"2023-09-23T15:24:50.367828+0000","last_clean":"2023-09-23T15:24:50.367828+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:24:50.367828+0000","last_undegraded":"2023-09-23T15:24:50.367828+0000","last_fullsized":"2023-09-23T15:24:50.367828+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"331'11421","reported_seq":17974,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:14.784542+0000","last_change":"2023-09-23T15:22:47.515540+0000","last_active":"2023-09-23T15:25:14.784542+0000","last_peered":"2023-09-23T15:25:14.784542+0000","last_clean":"2023-09-23T15:25:14.784542+0000","last_became_active":"2023-09-23T15:22:47.508502+0000","last_became_peered":"2023-09-23T15:22:47.508502+0000","last_unstale":"2023-09-23T15:25:14.784542+0000","last_undegraded":"2023-09-23T15:25:14.784542+0000","last_fullsized":"2023-09-23T15:25:14.784542+0000","mapping_epoch":310,"log_start":"306'4800","ondisk_log_start":"306'4800","created":284,"last_epoch_clean":311,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6621,"log_dups_size":0,"ondisk_log_size":6621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:17.382446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1376256,"num_objects":21,"num_object_clones":0,"num_object_copies":63,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":21,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11421,"num_write_kb":366144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3242,"num_bytes_recovered":212467712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":589,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.199302+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:25:15.199302+0000","last_peered":"2023-09-23T15:25:15.199302+0000","last_clean":"2023-09-23T15:25:15.199302+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:25:15.199302+0000","last_undegraded":"2023-09-23T15:25:15.199302+0000","last_fullsized":"2023-09-23T15:25:15.199302+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367214+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:24:50.367214+0000","last_peered":"2023-09-23T15:24:50.367214+0000","last_clean":"2023-09-23T15:24:50.367214+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:24:50.367214+0000","last_undegraded":"2023-09-23T15:24:50.367214+0000","last_fullsized":"2023-09-23T15:24:50.367214+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368039+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:24:50.368039+0000","last_peered":"2023-09-23T15:24:50.368039+0000","last_clean":"2023-09-23T15:24:50.368039+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:24:50.368039+0000","last_undegraded":"2023-09-23T15:24:50.368039+0000","last_fullsized":"2023-09-23T15:24:50.368039+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"331'11298","reported_seq":11362,"reported_epoch":332,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:15.198235+0000","last_change":"2023-09-23T15:22:27.591942+0000","last_active":"2023-09-23T15:25:15.198235+0000","last_peered":"2023-09-23T15:25:15.198235+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.574529+0000","last_became_peered":"2023-09-23T15:22:27.574529+0000","last_unstale":"2023-09-23T15:25:15.198235+0000","last_undegraded":"2023-09-23T15:22:27.326874+0000","last_fullsized":"2023-09-23T15:22:27.297619+0000","mapping_epoch":305,"log_start":"306'4700","ondisk_log_start":"306'4700","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6598,"log_dups_size":0,"ondisk_log_size":6598,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:41.403981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4618,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11298,"num_write_kb":361536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[{"shards":"1,5","objects":-4618}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":647,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.387981+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:24:50.387981+0000","last_peered":"2023-09-23T15:24:50.387981+0000","last_clean":"2023-09-23T15:24:50.387981+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:24:50.387981+0000","last_undegraded":"2023-09-23T15:24:50.387981+0000","last_fullsized":"2023-09-23T15:24:50.387981+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":346,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527631+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:25:04.527631+0000","last_peered":"2023-09-23T15:25:04.527631+0000","last_clean":"2023-09-23T15:25:04.527631+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:25:04.527631+0000","last_undegraded":"2023-09-23T15:25:04.527631+0000","last_fullsized":"2023-09-23T15:25:04.527631+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":241,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.198420+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:25:15.198420+0000","last_peered":"2023-09-23T15:25:15.198420+0000","last_clean":"2023-09-23T15:25:15.198420+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:25:15.198420+0000","last_undegraded":"2023-09-23T15:25:15.198420+0000","last_fullsized":"2023-09-23T15:25:15.198420+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"331'11032","reported_seq":17247,"reported_epoch":331,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:14.262300+0000","last_change":"2023-09-23T15:24:25.991143+0000","last_active":"2023-09-23T15:25:14.262300+0000","last_peered":"2023-09-23T15:25:14.262300+0000","last_clean":"2023-09-23T15:21:56.117984+0000","last_became_active":"2023-09-23T15:21:58.295821+0000","last_became_peered":"2023-09-23T15:21:58.295821+0000","last_unstale":"2023-09-23T15:25:14.262300+0000","last_undegraded":"2023-09-23T15:21:58.156599+0000","last_fullsized":"2023-09-23T15:21:58.128828+0000","mapping_epoch":297,"log_start":"302'4500","ondisk_log_start":"302'4500","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6532,"log_dups_size":0,"ondisk_log_size":6532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:33:21.952353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":23,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11032,"num_write_kb":354368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3070,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2],"avail_no_missing":["0","2","7"],"object_location_counts":[{"shards":"0,2","objects":19}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":528,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.371456+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:24:50.371456+0000","last_peered":"2023-09-23T15:24:50.371456+0000","last_clean":"2023-09-23T15:24:50.371456+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:24:50.371456+0000","last_undegraded":"2023-09-23T15:24:50.371456+0000","last_fullsized":"2023-09-23T15:24:50.371456+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":533,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T15:25:16.194312+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:25:16.194312+0000","last_peered":"2023-09-23T15:25:16.194312+0000","last_clean":"2023-09-23T15:25:16.194312+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:25:16.194312+0000","last_undegraded":"2023-09-23T15:25:16.194312+0000","last_fullsized":"2023-09-23T15:25:16.194312+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":307,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388727+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:24:50.388727+0000","last_peered":"2023-09-23T15:24:50.388727+0000","last_clean":"2023-09-23T15:24:50.388727+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:24:50.388727+0000","last_undegraded":"2023-09-23T15:24:50.388727+0000","last_fullsized":"2023-09-23T15:24:50.388727+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":275,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.376254+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:24:50.376254+0000","last_peered":"2023-09-23T15:24:50.376254+0000","last_clean":"2023-09-23T15:24:50.376254+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:24:50.376254+0000","last_undegraded":"2023-09-23T15:24:50.376254+0000","last_fullsized":"2023-09-23T15:24:50.376254+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"8.8","version":"331'11213","reported_seq":11278,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:14.889357+0000","last_change":"2023-09-23T15:21:19.250003+0000","last_active":"2023-09-23T15:25:14.889357+0000","last_peered":"2023-09-23T15:25:14.889357+0000","last_clean":"2023-09-23T15:25:14.889357+0000","last_became_active":"2023-09-23T15:21:19.249631+0000","last_became_peered":"2023-09-23T15:21:19.249631+0000","last_unstale":"2023-09-23T15:25:14.889357+0000","last_undegraded":"2023-09-23T15:25:14.889357+0000","last_fullsized":"2023-09-23T15:25:14.889357+0000","mapping_epoch":287,"log_start":"299'3300","ondisk_log_start":"299'3300","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7913,"log_dups_size":0,"ondisk_log_size":7913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:39.487181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":720896,"num_objects":11,"num_object_clones":0,"num_object_copies":33,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":11,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11213,"num_write_kb":359168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":628,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.389411+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:24:50.389411+0000","last_peered":"2023-09-23T15:24:50.389411+0000","last_clean":"2023-09-23T15:24:50.389411+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:24:50.389411+0000","last_undegraded":"2023-09-23T15:24:50.389411+0000","last_fullsized":"2023-09-23T15:24:50.389411+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":866,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388272+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:24:50.388272+0000","last_peered":"2023-09-23T15:24:50.388272+0000","last_clean":"2023-09-23T15:24:50.388272+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:24:50.388272+0000","last_undegraded":"2023-09-23T15:24:50.388272+0000","last_fullsized":"2023-09-23T15:24:50.388272+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367635+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:24:50.367635+0000","last_peered":"2023-09-23T15:24:50.367635+0000","last_clean":"2023-09-23T15:24:50.367635+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:24:50.367635+0000","last_undegraded":"2023-09-23T15:24:50.367635+0000","last_fullsized":"2023-09-23T15:24:50.367635+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":230,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T15:25:16.194013+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:25:16.194013+0000","last_peered":"2023-09-23T15:25:16.194013+0000","last_clean":"2023-09-23T15:25:16.194013+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:25:16.194013+0000","last_undegraded":"2023-09-23T15:25:16.194013+0000","last_fullsized":"2023-09-23T15:25:16.194013+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"331'11110","reported_seq":17404,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.161892+0000","last_change":"2023-09-23T15:23:41.531523+0000","last_active":"2023-09-23T15:25:15.161892+0000","last_peered":"2023-09-23T15:25:15.161892+0000","last_clean":"2023-09-23T15:25:15.161892+0000","last_became_active":"2023-09-23T15:23:41.512981+0000","last_became_peered":"2023-09-23T15:23:41.512981+0000","last_unstale":"2023-09-23T15:25:15.161892+0000","last_undegraded":"2023-09-23T15:25:15.161892+0000","last_fullsized":"2023-09-23T15:25:15.161892+0000","mapping_epoch":320,"log_start":"302'4500","ondisk_log_start":"302'4500","created":284,"last_epoch_clean":321,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6610,"log_dups_size":0,"ondisk_log_size":6610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:00.153741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11110,"num_write_kb":355520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3109,"num_bytes_recovered":203751424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":686,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379733+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:24:50.379733+0000","last_peered":"2023-09-23T15:24:50.379733+0000","last_clean":"2023-09-23T15:24:50.379733+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:24:50.379733+0000","last_undegraded":"2023-09-23T15:24:50.379733+0000","last_fullsized":"2023-09-23T15:24:50.379733+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":948,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.375632+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:24:50.375632+0000","last_peered":"2023-09-23T15:24:50.375632+0000","last_clean":"2023-09-23T15:24:50.375632+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:24:50.375632+0000","last_undegraded":"2023-09-23T15:24:50.375632+0000","last_fullsized":"2023-09-23T15:24:50.375632+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":225,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.196808+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:25:15.196808+0000","last_peered":"2023-09-23T15:25:15.196808+0000","last_clean":"2023-09-23T15:25:15.196808+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:25:15.196808+0000","last_undegraded":"2023-09-23T15:25:15.196808+0000","last_fullsized":"2023-09-23T15:25:15.196808+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":233,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.198335+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:25:15.198335+0000","last_peered":"2023-09-23T15:25:15.198335+0000","last_clean":"2023-09-23T15:25:15.198335+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:25:15.198335+0000","last_undegraded":"2023-09-23T15:25:15.198335+0000","last_fullsized":"2023-09-23T15:25:15.198335+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"331'11324","reported_seq":11382,"reported_epoch":331,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:25:14.187827+0000","last_change":"2023-09-23T15:22:27.565209+0000","last_active":"2023-09-23T15:25:14.187827+0000","last_peered":"2023-09-23T15:25:14.187827+0000","last_clean":"2023-09-23T15:21:11.199407+0000","last_became_active":"2023-09-23T15:22:27.551100+0000","last_became_peered":"2023-09-23T15:22:27.551100+0000","last_unstale":"2023-09-23T15:25:14.187827+0000","last_undegraded":"2023-09-23T15:22:27.323220+0000","last_fullsized":"2023-09-23T15:22:27.288665+0000","mapping_epoch":305,"log_start":"306'4800","ondisk_log_start":"306'4800","created":284,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6524,"log_dups_size":0,"ondisk_log_size":6524,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:29.700249+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2883584,"num_objects":44,"num_object_clones":0,"num_object_copies":132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4706,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":44,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11324,"num_write_kb":363776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":-4662}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":660,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388486+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:24:50.388486+0000","last_peered":"2023-09-23T15:24:50.388486+0000","last_clean":"2023-09-23T15:24:50.388486+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:24:50.388486+0000","last_undegraded":"2023-09-23T15:24:50.388486+0000","last_fullsized":"2023-09-23T15:24:50.388486+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":650,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379074+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:24:50.379074+0000","last_peered":"2023-09-23T15:24:50.379074+0000","last_clean":"2023-09-23T15:24:50.379074+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:24:50.379074+0000","last_undegraded":"2023-09-23T15:24:50.379074+0000","last_fullsized":"2023-09-23T15:24:50.379074+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":266,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527143+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:25:04.527143+0000","last_peered":"2023-09-23T15:25:04.527143+0000","last_clean":"2023-09-23T15:25:04.527143+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:25:04.527143+0000","last_undegraded":"2023-09-23T15:25:04.527143+0000","last_fullsized":"2023-09-23T15:25:04.527143+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":282,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368918+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:24:50.368918+0000","last_peered":"2023-09-23T15:24:50.368918+0000","last_clean":"2023-09-23T15:24:50.368918+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:24:50.368918+0000","last_undegraded":"2023-09-23T15:24:50.368918+0000","last_fullsized":"2023-09-23T15:24:50.368918+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"331'11150","reported_seq":11209,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.196911+0000","last_change":"2023-09-23T15:21:12.277462+0000","last_active":"2023-09-23T15:25:15.196911+0000","last_peered":"2023-09-23T15:25:15.196911+0000","last_clean":"2023-09-23T15:25:15.196911+0000","last_became_active":"2023-09-23T15:21:12.277166+0000","last_became_peered":"2023-09-23T15:21:12.277166+0000","last_unstale":"2023-09-23T15:25:15.196911+0000","last_undegraded":"2023-09-23T15:25:15.196911+0000","last_fullsized":"2023-09-23T15:25:15.196911+0000","mapping_epoch":284,"log_start":"306'4600","ondisk_log_start":"306'4600","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:53.610247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11150,"num_write_kb":356800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":640,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.199257+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:25:15.199257+0000","last_peered":"2023-09-23T15:25:15.199257+0000","last_clean":"2023-09-23T15:25:15.199257+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:25:15.199257+0000","last_undegraded":"2023-09-23T15:25:15.199257+0000","last_fullsized":"2023-09-23T15:25:15.199257+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":75,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.379051+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:24:50.379051+0000","last_peered":"2023-09-23T15:24:50.379051+0000","last_clean":"2023-09-23T15:24:50.379051+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:24:50.379051+0000","last_undegraded":"2023-09-23T15:24:50.379051+0000","last_fullsized":"2023-09-23T15:24:50.379051+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":298,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380349+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:24:50.380349+0000","last_peered":"2023-09-23T15:24:50.380349+0000","last_clean":"2023-09-23T15:24:50.380349+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:24:50.380349+0000","last_undegraded":"2023-09-23T15:24:50.380349+0000","last_fullsized":"2023-09-23T15:24:50.380349+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":310,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380345+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:24:50.380345+0000","last_peered":"2023-09-23T15:24:50.380345+0000","last_clean":"2023-09-23T15:24:50.380345+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:24:50.380345+0000","last_undegraded":"2023-09-23T15:24:50.380345+0000","last_fullsized":"2023-09-23T15:24:50.380345+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.f","version":"331'11550","reported_seq":11608,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.165574+0000","last_change":"2023-09-23T15:21:12.276512+0000","last_active":"2023-09-23T15:25:15.165574+0000","last_peered":"2023-09-23T15:25:15.165574+0000","last_clean":"2023-09-23T15:25:15.165574+0000","last_became_active":"2023-09-23T15:21:12.276209+0000","last_became_peered":"2023-09-23T15:21:12.276209+0000","last_unstale":"2023-09-23T15:25:15.165574+0000","last_undegraded":"2023-09-23T15:25:15.165574+0000","last_fullsized":"2023-09-23T15:25:15.165574+0000","mapping_epoch":284,"log_start":"307'5000","ondisk_log_start":"307'5000","created":284,"last_epoch_clean":285,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6550,"log_dups_size":0,"ondisk_log_size":6550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:15.177838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11550,"num_write_kb":369600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,1],"acting":[3,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":609,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.368962+0000","last_change":"2023-09-23T15:24:24.592597+0000","last_active":"2023-09-23T15:24:50.368962+0000","last_peered":"2023-09-23T15:24:50.368962+0000","last_clean":"2023-09-23T15:24:50.368962+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:24:50.368962+0000","last_undegraded":"2023-09-23T15:24:50.368962+0000","last_fullsized":"2023-09-23T15:24:50.368962+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1044,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380299+0000","last_change":"2023-09-23T15:24:25.990916+0000","last_active":"2023-09-23T15:24:50.380299+0000","last_peered":"2023-09-23T15:24:50.380299+0000","last_clean":"2023-09-23T15:24:50.380299+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:24:50.380299+0000","last_undegraded":"2023-09-23T15:24:50.380299+0000","last_fullsized":"2023-09-23T15:24:50.380299+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":210,"reported_epoch":333,"state":"active+clean","last_fresh":"2023-09-23T15:25:16.194187+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:25:16.194187+0000","last_peered":"2023-09-23T15:25:16.194187+0000","last_clean":"2023-09-23T15:25:16.194187+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:25:16.194187+0000","last_undegraded":"2023-09-23T15:25:16.194187+0000","last_fullsized":"2023-09-23T15:25:16.194187+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":294,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.377381+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:24:50.377381+0000","last_peered":"2023-09-23T15:24:50.377381+0000","last_clean":"2023-09-23T15:24:50.377381+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:24:50.377381+0000","last_undegraded":"2023-09-23T15:24:50.377381+0000","last_fullsized":"2023-09-23T15:24:50.377381+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"331'11078","reported_seq":11144,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:14.248944+0000","last_change":"2023-09-23T15:21:19.258050+0000","last_active":"2023-09-23T15:25:14.248944+0000","last_peered":"2023-09-23T15:25:14.248944+0000","last_clean":"2023-09-23T15:25:14.248944+0000","last_became_active":"2023-09-23T15:21:19.257731+0000","last_became_peered":"2023-09-23T15:21:19.257731+0000","last_unstale":"2023-09-23T15:25:14.248944+0000","last_undegraded":"2023-09-23T15:25:14.248944+0000","last_fullsized":"2023-09-23T15:25:14.248944+0000","mapping_epoch":287,"log_start":"303'4500","ondisk_log_start":"303'4500","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":6578,"log_dups_size":0,"ondisk_log_size":6578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:58.662304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2752512,"num_objects":42,"num_object_clones":0,"num_object_copies":126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":42,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11078,"num_write_kb":355840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":639,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367750+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:24:50.367750+0000","last_peered":"2023-09-23T15:24:50.367750+0000","last_clean":"2023-09-23T15:24:50.367750+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:24:50.367750+0000","last_undegraded":"2023-09-23T15:24:50.367750+0000","last_fullsized":"2023-09-23T15:24:50.367750+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":87,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:54.252809+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:24:54.252809+0000","last_peered":"2023-09-23T15:24:54.252809+0000","last_clean":"2023-09-23T15:24:54.252809+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:24:54.252809+0000","last_undegraded":"2023-09-23T15:24:54.252809+0000","last_fullsized":"2023-09-23T15:24:54.252809+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":886,"reported_epoch":332,"state":"active+clean","last_fresh":"2023-09-23T15:25:15.197425+0000","last_change":"2023-09-23T15:24:25.979031+0000","last_active":"2023-09-23T15:25:15.197425+0000","last_peered":"2023-09-23T15:25:15.197425+0000","last_clean":"2023-09-23T15:25:15.197425+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:25:15.197425+0000","last_undegraded":"2023-09-23T15:25:15.197425+0000","last_fullsized":"2023-09-23T15:25:15.197425+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":326,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527873+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:25:04.527873+0000","last_peered":"2023-09-23T15:25:04.527873+0000","last_clean":"2023-09-23T15:25:04.527873+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:25:04.527873+0000","last_undegraded":"2023-09-23T15:25:04.527873+0000","last_fullsized":"2023-09-23T15:25:04.527873+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"8.d","version":"331'11094","reported_seq":11157,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:14.908922+0000","last_change":"2023-09-23T15:21:19.249484+0000","last_active":"2023-09-23T15:25:14.908922+0000","last_peered":"2023-09-23T15:25:14.908922+0000","last_clean":"2023-09-23T15:25:14.908922+0000","last_became_active":"2023-09-23T15:21:19.249141+0000","last_became_peered":"2023-09-23T15:21:19.249141+0000","last_unstale":"2023-09-23T15:25:14.908922+0000","last_undegraded":"2023-09-23T15:25:14.908922+0000","last_fullsized":"2023-09-23T15:25:14.908922+0000","mapping_epoch":287,"log_start":"295'3100","ondisk_log_start":"295'3100","created":284,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:21:11.199407+0000","last_clean_scrub_stamp":"2023-09-23T15:21:11.199407+0000","objects_scrubbed":0,"log_size":7994,"log_dups_size":0,"ondisk_log_size":7994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:56.451116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":589848,"num_objects":10,"num_object_clones":0,"num_object_copies":30,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":10,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11094,"num_write_kb":355265,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":657,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:10.670777+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:25:10.670777+0000","last_peered":"2023-09-23T15:25:10.670777+0000","last_clean":"2023-09-23T15:25:10.670777+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:25:10.670777+0000","last_undegraded":"2023-09-23T15:25:10.670777+0000","last_fullsized":"2023-09-23T15:25:10.670777+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":767,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.371358+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:24:50.371358+0000","last_peered":"2023-09-23T15:24:50.371358+0000","last_clean":"2023-09-23T15:24:50.371358+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:24:50.371358+0000","last_undegraded":"2023-09-23T15:24:50.371358+0000","last_fullsized":"2023-09-23T15:24:50.371358+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":337,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.380449+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:24:50.380449+0000","last_peered":"2023-09-23T15:24:50.380449+0000","last_clean":"2023-09-23T15:24:50.380449+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:24:50.380449+0000","last_undegraded":"2023-09-23T15:24:50.380449+0000","last_fullsized":"2023-09-23T15:24:50.380449+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367016+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:24:50.367016+0000","last_peered":"2023-09-23T15:24:50.367016+0000","last_clean":"2023-09-23T15:24:50.367016+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:24:50.367016+0000","last_undegraded":"2023-09-23T15:24:50.367016+0000","last_fullsized":"2023-09-23T15:24:50.367016+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":314,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.377298+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:24:50.377298+0000","last_peered":"2023-09-23T15:24:50.377298+0000","last_clean":"2023-09-23T15:24:50.377298+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:24:50.377298+0000","last_undegraded":"2023-09-23T15:24:50.377298+0000","last_fullsized":"2023-09-23T15:24:50.377298+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":334,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.376120+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:24:50.376120+0000","last_peered":"2023-09-23T15:24:50.376120+0000","last_clean":"2023-09-23T15:24:50.376120+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:24:50.376120+0000","last_undegraded":"2023-09-23T15:24:50.376120+0000","last_fullsized":"2023-09-23T15:24:50.376120+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":344,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.389257+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:24:50.389257+0000","last_peered":"2023-09-23T15:24:50.389257+0000","last_clean":"2023-09-23T15:24:50.389257+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:24:50.389257+0000","last_undegraded":"2023-09-23T15:24:50.389257+0000","last_fullsized":"2023-09-23T15:24:50.389257+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367773+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:24:50.367773+0000","last_peered":"2023-09-23T15:24:50.367773+0000","last_clean":"2023-09-23T15:24:50.367773+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:24:50.367773+0000","last_undegraded":"2023-09-23T15:24:50.367773+0000","last_fullsized":"2023-09-23T15:24:50.367773+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":324,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.388658+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:24:50.388658+0000","last_peered":"2023-09-23T15:24:50.388658+0000","last_clean":"2023-09-23T15:24:50.388658+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:24:50.388658+0000","last_undegraded":"2023-09-23T15:24:50.388658+0000","last_fullsized":"2023-09-23T15:24:50.388658+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":44,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.367206+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:24:50.367206+0000","last_peered":"2023-09-23T15:24:50.367206+0000","last_clean":"2023-09-23T15:24:50.367206+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:24:50.367206+0000","last_undegraded":"2023-09-23T15:24:50.367206+0000","last_fullsized":"2023-09-23T15:24:50.367206+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":323,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.528329+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:25:04.528329+0000","last_peered":"2023-09-23T15:25:04.528329+0000","last_clean":"2023-09-23T15:25:04.528329+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:25:04.528329+0000","last_undegraded":"2023-09-23T15:25:04.528329+0000","last_fullsized":"2023-09-23T15:25:04.528329+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":321,"reported_epoch":331,"state":"active+clean","last_fresh":"2023-09-23T15:25:04.527573+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:25:04.527573+0000","last_peered":"2023-09-23T15:25:04.527573+0000","last_clean":"2023-09-23T15:25:04.527573+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:25:04.527573+0000","last_undegraded":"2023-09-23T15:25:04.527573+0000","last_fullsized":"2023-09-23T15:25:04.527573+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":283,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.377300+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:24:50.377300+0000","last_peered":"2023-09-23T15:24:50.377300+0000","last_clean":"2023-09-23T15:24:50.377300+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:24:50.377300+0000","last_undegraded":"2023-09-23T15:24:50.377300+0000","last_fullsized":"2023-09-23T15:24:50.377300+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":58,"reported_epoch":330,"state":"active+clean","last_fresh":"2023-09-23T15:24:50.366886+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:24:50.366886+0000","last_peered":"2023-09-23T15:24:50.366886+0000","last_clean":"2023-09-23T15:24:50.366886+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:24:50.366886+0000","last_undegraded":"2023-09-23T15:24:50.366886+0000","last_fullsized":"2023-09-23T15:24:50.366886+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":16056344,"num_objects":246,"num_object_clones":0,"num_object_copies":738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17338,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":246,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":180024,"num_write_kb":5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18507,"num_bytes_recovered":767033344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1110257664,"data_stored":1110245448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109524,"ondisk_log_size":109524,"up":48,"acting":42,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":112,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201646,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":364428,"kb_used_data":217476,"kb_used_omap":0,"kb_used_meta":146944,"kb_avail":93380724,"statfs":{"total":95995035648,"available":95621861376,"internally_reserved":0,"allocated":222695424,"data_stored":221678215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":150470656},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":3.3740000000000001,"5min":6.6959999999999997,"15min":3.9710000000000001},"min":{"1min":0.26100000000000001,"5min":0.22700000000000001,"15min":0.22500000000000001},"max":{"1min":12.096,"5min":226.226,"15min":226.226},"last":2.1970000000000001},{"interface":"front","average":{"1min":4.1159999999999997,"5min":7.9660000000000002,"15min":4.6340000000000003},"min":{"1min":0.27100000000000002,"5min":0.26200000000000001,"15min":0.24399999999999999},"max":{"1min":13.196,"5min":226.619,"15min":226.619},"last":0.747}]},{"osd":1,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":18.012,"5min":4.9059999999999997,"15min":3.0590000000000002},"min":{"1min":0.39900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":311.16699999999997,"5min":311.16699999999997,"15min":311.16699999999997},"last":2.5049999999999999},{"interface":"front","average":{"1min":18.634,"5min":5.3079999999999998,"15min":3.5590000000000002},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":311.279,"5min":311.279,"15min":311.279},"last":3.117}]},{"osd":2,"last update":"Sat Sep 23 15:24:24 2023","interfaces":[{"interface":"back","average":{"1min":2.802,"5min":2.7450000000000001,"15min":3.327},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.24099999999999999},"max":{"1min":7.7889999999999997,"5min":90.227999999999994,"15min":101.807},"last":3.552},{"interface":"front","average":{"1min":4.2880000000000003,"5min":3.2400000000000002,"15min":4.0030000000000001},"min":{"1min":0.34999999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.127000000000001,"5min":90.197999999999993,"15min":101.84099999999999},"last":0.82799999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:25:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":4.0880000000000001,"15min":3.9089999999999998},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":9.8000000000000007,"5min":146.166,"15min":146.166},"last":0.47799999999999998},{"interface":"front","average":{"1min":3.141,"5min":4.5049999999999999,"15min":4.0499999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":9.8179999999999996,"5min":146.18299999999999,"15min":146.18299999999999},"last":0.63200000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0219999999999998,"5min":1.696,"15min":2.7069999999999999},"min":{"1min":0.316,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.3550000000000004,"5min":16.771999999999998,"15min":150.119},"last":0.76100000000000001},{"interface":"front","average":{"1min":3.456,"5min":1.5269999999999999,"15min":2.8620000000000001},"min":{"1min":0.315,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.023,"5min":18.023,"15min":150.596},"last":0.70799999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":12.602,"5min":5.2270000000000003,"15min":3.5640000000000001},"min":{"1min":0.35599999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":216.85900000000001,"5min":224.441,"15min":224.441},"last":0.51200000000000001},{"interface":"front","average":{"1min":13.93,"5min":5.742,"15min":3.6669999999999998},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":216.43799999999999,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:24:30 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.4409999999999998,"15min":2.3010000000000002},"min":{"1min":0.48999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9659999999999993,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":0.871},{"interface":"front","average":{"1min":3.0550000000000002,"5min":2.6230000000000002,"15min":2.5510000000000002},"min":{"1min":0.35699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.135,"5min":10.336,"15min":10.336},"last":0.72299999999999998}]}]},{"osd":6,"up_from":304,"seq":1305670058019,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":115328,"kb_used_data":25848,"kb_used_omap":13,"kb_used_meta":89394,"kb_avail":93629824,"statfs":{"total":95995035648,"available":95876939776,"internally_reserved":0,"allocated":26468352,"data_stored":25463280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":91539811},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":16.183,"5min":4.4199999999999999,"15min":2.46},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":184.53200000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":46.740000000000002},{"interface":"front","average":{"1min":8.5370000000000008,"5min":4.0090000000000003,"15min":3.2549999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.227999999999994,"5min":86.227999999999994,"15min":86.227999999999994},"last":46.549999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":9.2279999999999998,"5min":4.0389999999999997,"15min":3.1739999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":86.534000000000006,"5min":86.534000000000006,"15min":86.534000000000006},"last":2.145},{"interface":"front","average":{"1min":8.5199999999999996,"5min":3.9929999999999999,"15min":3.2389999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":86.144000000000005,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.439}]},{"osd":2,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.762,"15min":1.6040000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":4.7999999999999998},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.99,"15min":1.8140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.1910000000000007,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":2.2330000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.7109999999999999,"15min":2.423},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":41.261000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":0.47999999999999998},{"interface":"front","average":{"1min":5.1559999999999997,"5min":2.8479999999999999,"15min":2.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":41.408000000000001},"last":4.8470000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.1839999999999999,"15min":1.02},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.754,"5min":2.0070000000000001,"15min":1.883},"min":{"1min":0.39800000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.627000000000001,"5min":13.627000000000001,"15min":13.627000000000001},"last":4.8289999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.2749999999999999,"15min":1.903},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.74,"5min":13.74,"15min":13.74},"last":2.1819999999999999},{"interface":"front","average":{"1min":3.194,"5min":1.8260000000000001,"15min":1.5980000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":15.868,"5min":15.868,"15min":15.868},"last":0.46300000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.752,"15min":1.5169999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":12.023999999999999,"5min":12.023999999999999,"15min":12.023999999999999},"last":4.8639999999999999},{"interface":"front","average":{"1min":3.5259999999999998,"5min":2.1629999999999998,"15min":1.9359999999999999},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":12.641999999999999},"last":2.2109999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015320,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":668880,"kb_used_data":204168,"kb_used_omap":6,"kb_used_meta":464697,"kb_avail":93076272,"statfs":{"total":95995035648,"available":95310102528,"internally_reserved":0,"allocated":209068032,"data_stored":207493390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":475849803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":3.1619999999999999,"15min":3.3130000000000002},"min":{"1min":0.223,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":7.6379999999999999,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.20100000000000001},{"interface":"front","average":{"1min":3.6360000000000001,"5min":4.2119999999999997,"15min":4.0999999999999996},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":17.128,"5min":63.594999999999999,"15min":63.594999999999999},"last":0.26200000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":2.4750000000000001,"15min":4.0869999999999997},"min":{"1min":0.26500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":12.429,"5min":12.429,"15min":109.375},"last":0.376},{"interface":"front","average":{"1min":2.597,"5min":2.1059999999999999,"15min":3.456},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":7.758,"5min":12.259,"15min":105.16},"last":0.28699999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":3.8130000000000002,"15min":7.1580000000000004},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":7.6820000000000004,"5min":64.540999999999997,"15min":211.405},"last":0.36299999999999999},{"interface":"front","average":{"1min":2.6549999999999998,"5min":4.1879999999999997,"15min":7.6929999999999996},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":15.637,"5min":67.465000000000003,"15min":210.988},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":2.2999999999999998,"15min":3.3879999999999999},"min":{"1min":0.50800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.112,"5min":13.379,"15min":80.570999999999998},"last":0.34399999999999997},{"interface":"front","average":{"1min":3.343,"5min":2.1640000000000001,"15min":3.407},"min":{"1min":0.313,"5min":0.313,"15min":0.308},"max":{"1min":8.7639999999999993,"5min":12.279999999999999,"15min":80.75},"last":0.436}]},{"osd":5,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":3.9409999999999998,"15min":3.4380000000000002},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.27400000000000002},"max":{"1min":15.561,"5min":115.401,"15min":115.401},"last":0.32900000000000001},{"interface":"front","average":{"1min":4.3680000000000003,"5min":3.9159999999999999,"15min":3.4340000000000002},"min":{"1min":0.54500000000000004,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":13.068,"5min":115.169,"15min":115.169},"last":0.41499999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:24:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":3.335,"15min":3.1589999999999998},"min":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":19.553999999999998,"5min":19.553999999999998,"15min":19.553999999999998},"last":0.48799999999999999},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.2869999999999999,"15min":2.3340000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9770000000000003,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":0.45100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.9159999999999999,"5min":2.964,"15min":2.9980000000000002},"min":{"1min":0.71799999999999997,"5min":0.371,"15min":0.371},"max":{"1min":9.3469999999999995,"5min":19.702999999999999,"15min":19.702999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":2.9740000000000002,"5min":2.4329999999999998,"15min":2.6549999999999998},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.377},"max":{"1min":7.7859999999999996,"5min":20.722000000000001,"15min":20.722000000000001},"last":0.47799999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893369,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":903976,"kb_used_data":321376,"kb_used_omap":8,"kb_used_meta":582519,"kb_avail":92841176,"statfs":{"total":95995035648,"available":95069364224,"internally_reserved":0,"allocated":329089024,"data_stored":327655570,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":596500052},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:25:02 2023","interfaces":[{"interface":"back","average":{"1min":10.018000000000001,"5min":3.879,"15min":3.8500000000000001},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":97.343000000000004,"5min":97.343000000000004,"15min":97.343000000000004},"last":3.9209999999999998},{"interface":"front","average":{"1min":9.1280000000000001,"5min":3.6589999999999998,"15min":3.681},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":96.158000000000001,"5min":96.158000000000001,"15min":96.158000000000001},"last":0.29899999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":2.4020000000000001,"15min":4.3609999999999998},"min":{"1min":0.23300000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":13.022,"5min":30.510999999999999,"15min":122.20099999999999},"last":5.9009999999999998},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.7170000000000001,"15min":4.2359999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":9.2050000000000001,"5min":30.396999999999998,"15min":122.071},"last":0.60099999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3140000000000001,"5min":4.8300000000000001,"15min":5.8819999999999997},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.20000000000000001},"max":{"1min":41.380000000000003,"5min":197.78899999999999,"15min":197.78899999999999},"last":4.4850000000000003},{"interface":"front","average":{"1min":4.8079999999999998,"5min":5.2050000000000001,"15min":6.3630000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.252},"max":{"1min":41.465000000000003,"5min":200.58799999999999,"15min":200.58799999999999},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.2330000000000001,"5min":2.177,"15min":2.278},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.28000000000000003},"max":{"1min":15.038,"5min":15.358000000000001,"15min":31.341999999999999},"last":1.4410000000000001},{"interface":"front","average":{"1min":2.327,"5min":1.873,"15min":2.2429999999999999},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":8.2629999999999999,"5min":9.8249999999999993,"15min":99.168999999999997},"last":4.5819999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3799999999999999,"5min":2.1880000000000002,"15min":2.456},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.28599999999999998},"max":{"1min":10.618,"5min":10.768000000000001,"15min":109.369},"last":0.63800000000000001},{"interface":"front","average":{"1min":3.7959999999999998,"5min":2.3879999999999999,"15min":2.528},"min":{"1min":0.39100000000000001,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":15.363,"5min":15.436999999999999,"15min":109.387},"last":1.363}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.887,"15min":2.835},"min":{"1min":0.45900000000000002,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":1.3260000000000001},{"interface":"front","average":{"1min":3.4630000000000001,"5min":3.3809999999999998,"15min":3.367},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":15.829000000000001,"5min":17.385000000000002,"15min":17.385000000000002},"last":4.5279999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2779999999999996,"5min":2.4159999999999999,"15min":2.6219999999999999},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":17.712,"5min":17.712,"15min":99.549999999999997},"last":0.66700000000000004},{"interface":"front","average":{"1min":3.2629999999999999,"5min":2.556,"15min":2.2610000000000001},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.706,"5min":15.407999999999999,"15min":31.225999999999999},"last":1.474}]}]},{"osd":2,"up_from":105,"seq":450971566316,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":951708,"kb_used_data":336852,"kb_used_omap":11,"kb_used_meta":614836,"kb_avail":92793444,"statfs":{"total":95995035648,"available":95020486656,"internally_reserved":0,"allocated":344936448,"data_stored":343508967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":629592848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":13.015000000000001,"5min":7.2009999999999996,"15min":4.9459999999999997},"min":{"1min":0.214,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":88.622,"5min":88.622,"15min":88.622},"last":0.21199999999999999},{"interface":"front","average":{"1min":12.861000000000001,"5min":7.21,"15min":4.6269999999999998},"min":{"1min":0.27500000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":88.649000000000001,"5min":88.649000000000001,"15min":88.649000000000001},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":3.9209999999999998,"15min":4.577},"min":{"1min":0.21099999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.0129999999999999,"5min":71.819000000000003,"15min":84.641000000000005},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.7749999999999999,"5min":4.0910000000000002,"15min":3.6930000000000001},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.036,"5min":71.805000000000007,"15min":84.751999999999995},"last":0.24399999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:24:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":5.5350000000000001,"15min":5.2850000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":8.2970000000000006,"5min":77.031999999999996,"15min":87.108000000000004},"last":48.527000000000001},{"interface":"front","average":{"1min":2.5190000000000001,"5min":5.9119999999999999,"15min":5.3650000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":10.935,"5min":76.742000000000004,"15min":86.158000000000001},"last":48.371000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.536,"5min":1.6020000000000001,"15min":2.3969999999999998},"min":{"1min":0.32600000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":16.712,"5min":16.712,"15min":84.608000000000004},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.0600000000000001,"5min":1.579,"15min":2.9980000000000002},"min":{"1min":0.376,"5min":0.29699999999999999,"15min":0.26200000000000001},"max":{"1min":6.3789999999999996,"5min":7.7880000000000003,"15min":87.019999999999996},"last":0.28399999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1640000000000001,"5min":1.839,"15min":2.6480000000000001},"min":{"1min":0.36899999999999999,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":7.6980000000000004,"5min":8.9909999999999997,"15min":84.498000000000005},"last":0.374},{"interface":"front","average":{"1min":2.2810000000000001,"5min":1.7210000000000001,"15min":2.5409999999999999},"min":{"1min":0.40400000000000003,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":7.5099999999999998,"5min":10.808999999999999,"15min":84.664000000000001},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":2.3290000000000002,"15min":2.2989999999999999},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.4600000000000009,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.8340000000000001,"15min":1.978},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.6909999999999998,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.439}]},{"osd":7,"last update":"Sat Sep 23 15:24:18 2023","interfaces":[{"interface":"back","average":{"1min":2.1549999999999998,"5min":1.9970000000000001,"15min":2.593},"min":{"1min":0.41299999999999998,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":9.3859999999999992,"5min":10.227,"15min":84.596000000000004},"last":0.54400000000000004},{"interface":"front","average":{"1min":3.2250000000000001,"5min":1.9550000000000001,"15min":2.4910000000000001},"min":{"1min":0.42899999999999999,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":16.074999999999999,"5min":16.074999999999999,"15min":84.783000000000001},"last":0.46200000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402790,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":645344,"kb_used_data":35096,"kb_used_omap":3,"kb_used_meta":610236,"kb_avail":93099808,"statfs":{"total":95995035648,"available":95334203392,"internally_reserved":0,"allocated":35938304,"data_stored":34507543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":624881833},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":3.613,"15min":3.4329999999999998},"min":{"1min":0.23499999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":21.949000000000002,"5min":108.318,"15min":108.318},"last":0.27600000000000002},{"interface":"front","average":{"1min":3.3029999999999999,"5min":4.101,"15min":3.726},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":15.81,"5min":108.889,"15min":108.889},"last":0.56899999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:25:09 2023","interfaces":[{"interface":"back","average":{"1min":4.1760000000000002,"5min":4.7469999999999999,"15min":4.8070000000000004},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.603,"5min":126.875,"15min":126.875},"last":0.372},{"interface":"front","average":{"1min":4.8650000000000002,"5min":5.532,"15min":5.0529999999999999},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":22.006,"5min":128.249,"15min":128.249},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":2.2909999999999999,"15min":3.911},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.22},"max":{"1min":14.853,"5min":15.048999999999999,"15min":109.077},"last":0.34000000000000002},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.9609999999999999,"15min":4.8559999999999999},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":9.6489999999999991,"5min":36.582000000000001,"15min":119.72799999999999},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.925,"15min":2.2709999999999999},"min":{"1min":0.33000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9219999999999997,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.41299999999999998},{"interface":"front","average":{"1min":3.3090000000000002,"5min":1.8660000000000001,"15min":2.2160000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.115,"5min":13.115,"15min":52.228999999999999},"last":0.52100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":3.6669999999999998,"15min":2.9550000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.9260000000000002,"5min":148.619,"15min":148.619},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.972,"5min":3.383,"15min":2.9020000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.026999999999999,"5min":148.56999999999999,"15min":148.56999999999999},"last":0.48499999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:28 2023","interfaces":[{"interface":"back","average":{"1min":2.2320000000000002,"5min":2.226,"15min":2.2250000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":6.5460000000000003,"5min":9.2200000000000006,"15min":9.2200000000000006},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.0459999999999998,"5min":3.3300000000000001,"15min":3.544},"min":{"1min":0.42099999999999999,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":6.7699999999999996,"5min":36.517000000000003,"15min":36.517000000000003},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":3.25,"15min":2.7749999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":13.153,"5min":145.25,"15min":145.25},"last":0.58399999999999996},{"interface":"front","average":{"1min":2.9990000000000001,"5min":3.2879999999999998,"15min":2.613},"min":{"1min":0.39800000000000002,"5min":0.38500000000000001,"15min":0.34799999999999998},"max":{"1min":13.058999999999999,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.53700000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986555,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372356,"kb_used_data":18300,"kb_used_omap":0,"kb_used_meta":354048,"kb_avail":93372796,"statfs":{"total":95995035648,"available":95613743104,"internally_reserved":0,"allocated":18739200,"data_stored":17730089,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":362545152},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":5.2699999999999996,"5min":5.1219999999999999,"15min":4.2460000000000004},"min":{"1min":0.28299999999999997,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":53.771000000000001,"5min":162.75999999999999,"15min":162.75999999999999},"last":2.3599999999999999},{"interface":"front","average":{"1min":4.0990000000000002,"5min":4.8369999999999997,"15min":4.1959999999999997},"min":{"1min":0.27200000000000002,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":53.145000000000003,"5min":162.34800000000001,"15min":162.34800000000001},"last":4.3209999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":4.2359999999999998,"15min":3.3370000000000002},"min":{"1min":0.314,"5min":0.249,"15min":0.245},"max":{"1min":8.7249999999999996,"5min":113.11199999999999,"15min":113.11199999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.4329999999999998,"5min":4.1600000000000001,"15min":3.7069999999999999},"min":{"1min":0.38900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.0129999999999999,"5min":113.02,"15min":113.02},"last":3.1920000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":1.8600000000000001,"5min":3.3599999999999999,"15min":4.3040000000000003},"min":{"1min":0.316,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":4.8010000000000002,"5min":61.646000000000001,"15min":148.08000000000001},"last":0.40699999999999997},{"interface":"front","average":{"1min":2.4039999999999999,"5min":3.536,"15min":4.5309999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.25700000000000001},"max":{"1min":6.1980000000000004,"5min":64.840000000000003,"15min":147.88900000000001},"last":2.5529999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:24:50 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.9990000000000001,"15min":6.0659999999999998},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":8.1859999999999999,"5min":112.681,"15min":163.97499999999999},"last":2.4420000000000002},{"interface":"front","average":{"1min":2.6579999999999999,"5min":3.0289999999999999,"15min":6.0439999999999996},"min":{"1min":0.35999999999999999,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":7.1719999999999997,"5min":112.621,"15min":164.447},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 15:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.2190000000000001,"15min":1.3879999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":5.5510000000000002,"5min":6.4409999999999998,"15min":87.912999999999997},"last":2.593},{"interface":"front","average":{"1min":1.22,"5min":1.403,"15min":1.3979999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":6.8250000000000002,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.39500000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.446,"15min":1.446},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.8010000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.121,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.5380000000000003,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.177,"15min":1.0740000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.2999999999999998,"5min":6.0730000000000004,"15min":18.23},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.895,"5min":1.2709999999999999,"15min":1.1890000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.3239999999999998,"5min":6.3239999999999998,"15min":14.044},"last":2.6349999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724918,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":406908,"kb_used_data":18548,"kb_used_omap":0,"kb_used_meta":388352,"kb_avail":93338244,"statfs":{"total":95995035648,"available":95578361856,"internally_reserved":0,"allocated":18993152,"data_stored":17988494,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":397672448},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":4.0010000000000003,"5min":4.1900000000000004,"15min":3.3079999999999998},"min":{"1min":0.251,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.178999999999998,"5min":94.372,"15min":124.31},"last":0.26200000000000001},{"interface":"front","average":{"1min":4.0410000000000004,"5min":4.0720000000000001,"15min":3.3769999999999998},"min":{"1min":0.44400000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":17.106000000000002,"5min":94.396000000000001,"15min":124.374},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":3.319,"5min":4.875,"15min":3.2799999999999998},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.9320000000000004,"5min":86.900999999999996,"15min":124.38800000000001},"last":2.2149999999999999},{"interface":"front","average":{"1min":2.6429999999999998,"5min":3.867,"15min":3.1160000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":7.0650000000000004,"5min":66.825000000000003,"15min":124.29000000000001},"last":0.50800000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:24:16 2023","interfaces":[{"interface":"back","average":{"1min":3.613,"5min":2.444,"15min":3.7770000000000001},"min":{"1min":0.35899999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":11.754,"5min":74.019000000000005,"15min":193.00999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":2.8900000000000001,"5min":2.5489999999999999,"15min":3.8239999999999998},"min":{"1min":0.37,"5min":0.29299999999999998,"15min":0.254},"max":{"1min":10.542999999999999,"5min":73.712000000000003,"15min":193.15700000000001},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.6630000000000003,"5min":5.2350000000000003,"15min":6.2359999999999998},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.27500000000000002},"max":{"1min":55.354999999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":5.9699999999999998,"5min":6.1470000000000002,"15min":6.3369999999999997},"min":{"1min":0.40400000000000003,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":55.417000000000002,"5min":266.80399999999997,"15min":266.80399999999997},"last":2.1749999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.7010000000000001,"15min":2.0390000000000001},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":8.8689999999999998,"5min":10.749000000000001,"15min":138.68600000000001},"last":2.327},{"interface":"front","average":{"1min":2.621,"5min":1.714,"15min":2.089},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":8.6349999999999998,"5min":10.516,"15min":138.49600000000001},"last":4.8179999999999996}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.4390000000000001,"15min":2.411},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":10.577999999999999,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.5249999999999999,"5min":2.633,"15min":2.484},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.257999999999999,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0419999999999998,"5min":1.8260000000000001,"15min":1.478},"min":{"1min":0.40500000000000003,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":15.849,"5min":15.849,"15min":15.849},"last":4.9619999999999997},{"interface":"front","average":{"1min":3.7360000000000002,"5min":1.7090000000000001,"15min":1.45},"min":{"1min":0.435,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":16.134,"5min":16.134,"15min":16.134},"last":2.2989999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":314703872,"data_stored":314703872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":200736768,"data_stored":200736768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":338694144,"data_stored":338690072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":23789568,"data_stored":23789568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":6488064,"data_stored":6488064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":5705728,"data_stored":5701656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":8781824,"data_stored":8781824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":211357696,"data_stored":211353624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:25:22.224 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:22.243 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:22.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:22.351 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:22.383 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:22.409 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:22.434 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:22.435 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:22.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:22.438 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[215646]: pgmap v992: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 17315/84 objects degraded (20613.095%); 0 B/s, 134 objects/s recovering 2023-09-23T15:25:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:25:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 17315/84 objects degraded (20613.095%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[222878]: pgmap v992: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 17315/84 objects degraded (20613.095%); 0 B/s, 134 objects/s recovering 2023-09-23T15:25:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:25:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 17315/84 objects degraded (20613.095%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:22.538 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:22.576 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:22 smithi027 ceph-mon[195233]: pgmap v992: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 364 op/s; 17315/84 objects degraded (20613.095%); 0 B/s, 134 objects/s recovering 2023-09-23T15:25:22.576 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:25:22.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 17315/84 objects degraded (20613.095%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:22.579 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:22.681 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:22.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:22.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:22.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:22.816 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:22.868 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:22.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:22.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:22.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:23.006 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:23.058 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:23.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:23.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:23.203 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:23.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:25:23.403 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:23.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:23.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:23.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:23.540 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:23.593 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:23.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:23.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:23.727 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:24 smithi027 ceph-mon[195233]: pgmap v993: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 16008/84 objects degraded (19057.143%); 0 B/s, 278 objects/s recovering 2023-09-23T15:25:24.692 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:24 smithi017 ceph-mon[215646]: pgmap v993: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 16008/84 objects degraded (19057.143%); 0 B/s, 278 objects/s recovering 2023-09-23T15:25:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:24 smithi017 ceph-mon[222878]: pgmap v993: 109 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 178 op/s; 16008/84 objects degraded (19057.143%); 0 B/s, 278 objects/s recovering 2023-09-23T15:25:24.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:24.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:24.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:24.827 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:24.933 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:24.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:24.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:25.037 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:25.067 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:25.122 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:25.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:25.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:25.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:25:25.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:25.288 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:25.329 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[222878]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.329 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[222878]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.329 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[222878]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T15:25:25.329 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[222878]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.329 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[215646]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[215646]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[215646]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T15:25:25.330 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:25 smithi017 ceph-mon[215646]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.444 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:25.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:25.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:25.548 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:25 smithi027 ceph-mon[195233]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:25 smithi027 ceph-mon[195233]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:25 smithi027 ceph-mon[195233]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T15:25:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:25 smithi027 ceph-mon[195233]: failed to encode map e335 with expected crc 2023-09-23T15:25:25.581 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:25.637 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:25.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:25.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:25.656 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:25.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:25.764 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:25:25.764 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:25:25.765 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:25:25.765 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:25:25.775 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:25.832 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:25.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:25.840 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:25:25.861 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:25:25.884 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:25:25.903 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:25:25.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:25:25.936 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:25.947 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:25:25.965 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:25.968 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:25:25.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:25:25.992 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: pgmap v995: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 15915/84 objects degraded (18946.429%); 0 B/s, 275 objects/s recovering 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3532243071' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.287 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4089062254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: pgmap v995: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 15915/84 objects degraded (18946.429%); 0 B/s, 275 objects/s recovering 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3532243071' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.288 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4089062254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:26.422 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_3' removed 2023-09-23T15:25:26.563 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:26.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:26.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: pgmap v995: 109 pgs: 2 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 96 op/s; 15915/84 objects degraded (18946.429%); 0 B/s, 275 objects/s recovering 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: failed to encode map e336 with expected crc 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3532243071' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:25:26.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4089062254' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:26.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:26.696 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:26.741 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:26.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:26.846 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:26.892 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:26.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:26.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:26.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:27.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:27.072 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:27.320 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:27.323 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:27.423 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:27.453 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[215646]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[215646]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[215646]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:27.564 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[222878]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:25:27.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[222878]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T15:25:27.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[222878]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.565 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:27.806 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:25:27.807 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:25:27.808 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T15:25:27.808 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T15:25:27.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:25:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:27 smithi027 ceph-mon[195233]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:25:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:27 smithi027 ceph-mon[195233]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T15:25:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:27 smithi027 ceph-mon[195233]: failed to encode map e337 with expected crc 2023-09-23T15:25:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T15:25:27.850 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:27.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:27.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:27.907 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:27.956 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:27.985 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:28.038 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:28.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:28.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:28.168 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:28] "GET /metrics HTTP/1.1" 200 36398 "" "Prometheus/2.43.0" 2023-09-23T15:25:28.171 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[215646]: pgmap v998: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[215646]: failed to encode map e337 with expected crc 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized) 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[222878]: pgmap v998: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[222878]: failed to encode map e337 with expected crc 2023-09-23T15:25:28.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized) 2023-09-23T15:25:28.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:28.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:28.459 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:28.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:28.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:28.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:28.603 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:28.801 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:28.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:28.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:28 smithi027 ceph-mon[195233]: pgmap v998: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:28 smithi027 ceph-mon[195233]: failed to encode map e337 with expected crc 2023-09-23T15:25:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:28 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 15915/84 objects degraded (18946.429%), 5 pgs degraded, 5 pgs undersized) 2023-09-23T15:25:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:28.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:28.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:28.940 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:28.992 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:28.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:28.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:29.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:29.128 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:29.174 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:29.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:29.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:29.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:25:29.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:29.313 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:29.617 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:29.643 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:29.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:29.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:29.704 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:29.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:29.782 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:30.110 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:30.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:30.113 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:30.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:30.247 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:30.299 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:30.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:30.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:30.394 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:25:30.404 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:30.431 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:30.476 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:30.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:30.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:30 smithi027 ceph-mon[195233]: pgmap v999: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:30.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:30 smithi017 ceph-mon[215646]: pgmap v999: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:30.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:30.610 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:30 smithi017 ceph-mon[222878]: pgmap v999: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:30.610 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:30.613 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:31.561 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:31.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:31.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:31.665 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:31.690 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[215646]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:31 smithi017 ceph-mon[222878]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T15:25:31.692 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: failed to encode map e338 with expected crc 2023-09-23T15:25:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3249950499' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:31 smithi027 ceph-mon[195233]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T15:25:32.142 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:32.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:32.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:32.173 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:32.248 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:32.281 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:32.332 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:32.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:32.339 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:32.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:32 smithi017 ceph-mon[215646]: pgmap v1001: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:32.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:32.472 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:25:32.512 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:25:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:32 smithi027 ceph-mon[195233]: pgmap v1001: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:32.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:32 smithi017 ceph-mon[222878]: pgmap v1001: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:32.712 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:32.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:32.816 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:32.854 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:33.544 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1001,"stamp":"2023-09-23T15:25:30.769235+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4843865,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6039,"num_read_kb":13295,"num_write":929,"num_write_kb":12945,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":9080181,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4413,"ondisk_log_size":4413,"up":279,"acting":279,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":311,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4290876,"kb_used_data":1007932,"kb_used_omap":44,"kb_used_meta":3282835,"kb_avail":745670340,"statfs":{"total":767960285184,"available":763566428160,"internally_reserved":0,"allocated":1032122368,"data_stored":1022011071,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3361623995},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":57,"apply_latency_ms":57,"commit_latency_ns":57000000,"apply_latency_ns":57000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-15915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-180270,"num_write_kb":-5768577,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-19930,"num_bytes_recovered":-767033344,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001355"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227866+0000","last_change":"2023-09-23T15:21:57.139247+0000","last_active":"2023-09-23T15:25:25.227866+0000","last_peered":"2023-09-23T15:25:25.227866+0000","last_clean":"2023-09-23T15:25:25.227866+0000","last_became_active":"2023-09-23T15:21:57.138498+0000","last_became_peered":"2023-09-23T15:21:57.138498+0000","last_unstale":"2023-09-23T15:25:25.227866+0000","last_undegraded":"2023-09-23T15:25:25.227866+0000","last_fullsized":"2023-09-23T15:25:25.227866+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:49.103596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":293,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.230120+0000","last_change":"2023-09-23T15:21:57.144311+0000","last_active":"2023-09-23T15:25:25.230120+0000","last_peered":"2023-09-23T15:25:25.230120+0000","last_clean":"2023-09-23T15:25:25.230120+0000","last_became_active":"2023-09-23T15:21:57.143885+0000","last_became_peered":"2023-09-23T15:21:57.143885+0000","last_unstale":"2023-09-23T15:25:25.230120+0000","last_undegraded":"2023-09-23T15:25:25.230120+0000","last_fullsized":"2023-09-23T15:25:25.230120+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:37:07.426608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":242,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227485+0000","last_change":"2023-09-23T15:21:57.144112+0000","last_active":"2023-09-23T15:25:25.227485+0000","last_peered":"2023-09-23T15:25:25.227485+0000","last_clean":"2023-09-23T15:25:25.227485+0000","last_became_active":"2023-09-23T15:21:57.143788+0000","last_became_peered":"2023-09-23T15:21:57.143788+0000","last_unstale":"2023-09-23T15:25:25.227485+0000","last_undegraded":"2023-09-23T15:25:25.227485+0000","last_fullsized":"2023-09-23T15:25:25.227485+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:24:16.512024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":316,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244867+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:25:25.244867+0000","last_peered":"2023-09-23T15:25:25.244867+0000","last_clean":"2023-09-23T15:25:25.244867+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:25:25.244867+0000","last_undegraded":"2023-09-23T15:25:25.244867+0000","last_fullsized":"2023-09-23T15:25:25.244867+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":338,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229858+0000","last_change":"2023-09-23T15:12:41.927001+0000","last_active":"2023-09-23T15:25:25.229858+0000","last_peered":"2023-09-23T15:25:25.229858+0000","last_clean":"2023-09-23T15:25:25.229858+0000","last_became_active":"2023-09-23T15:12:41.926483+0000","last_became_peered":"2023-09-23T15:12:41.926483+0000","last_unstale":"2023-09-23T15:25:25.229858+0000","last_undegraded":"2023-09-23T15:25:25.229858+0000","last_fullsized":"2023-09-23T15:25:25.229858+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:39.659758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":287,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227429+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:25:25.227429+0000","last_peered":"2023-09-23T15:25:25.227429+0000","last_clean":"2023-09-23T15:25:25.227429+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:25:25.227429+0000","last_undegraded":"2023-09-23T15:25:25.227429+0000","last_fullsized":"2023-09-23T15:25:25.227429+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":285,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244312+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:25:25.244312+0000","last_peered":"2023-09-23T15:25:25.244312+0000","last_clean":"2023-09-23T15:25:25.244312+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:25:25.244312+0000","last_undegraded":"2023-09-23T15:25:25.244312+0000","last_fullsized":"2023-09-23T15:25:25.244312+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":336,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229920+0000","last_change":"2023-09-23T15:22:26.296431+0000","last_active":"2023-09-23T15:25:25.229920+0000","last_peered":"2023-09-23T15:25:25.229920+0000","last_clean":"2023-09-23T15:25:25.229920+0000","last_became_active":"2023-09-23T15:22:26.296106+0000","last_became_peered":"2023-09-23T15:22:26.296106+0000","last_unstale":"2023-09-23T15:25:25.229920+0000","last_undegraded":"2023-09-23T15:25:25.229920+0000","last_fullsized":"2023-09-23T15:25:25.229920+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:29.220475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":695,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.243999+0000","last_change":"2023-09-23T15:22:00.148180+0000","last_active":"2023-09-23T15:25:25.243999+0000","last_peered":"2023-09-23T15:25:25.243999+0000","last_clean":"2023-09-23T15:25:25.243999+0000","last_became_active":"2023-09-23T15:22:00.147808+0000","last_became_peered":"2023-09-23T15:22:00.147808+0000","last_unstale":"2023-09-23T15:25:25.243999+0000","last_undegraded":"2023-09-23T15:25:25.243999+0000","last_fullsized":"2023-09-23T15:25:25.243999+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:58.904879+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":337,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228834+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:25:25.228834+0000","last_peered":"2023-09-23T15:25:25.228834+0000","last_clean":"2023-09-23T15:25:25.228834+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:25:25.228834+0000","last_undegraded":"2023-09-23T15:25:25.228834+0000","last_fullsized":"2023-09-23T15:25:25.228834+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":290,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.230771+0000","last_change":"2023-09-23T15:22:27.307183+0000","last_active":"2023-09-23T15:25:25.230771+0000","last_peered":"2023-09-23T15:25:25.230771+0000","last_clean":"2023-09-23T15:25:25.230771+0000","last_became_active":"2023-09-23T15:22:27.306832+0000","last_became_peered":"2023-09-23T15:22:27.306832+0000","last_unstale":"2023-09-23T15:25:25.230771+0000","last_undegraded":"2023-09-23T15:25:25.230771+0000","last_fullsized":"2023-09-23T15:25:25.230771+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:39:51.980304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":81,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232929+0000","last_change":"2023-09-23T15:21:57.147438+0000","last_active":"2023-09-23T15:25:25.232929+0000","last_peered":"2023-09-23T15:25:25.232929+0000","last_clean":"2023-09-23T15:25:25.232929+0000","last_became_active":"2023-09-23T15:21:57.137294+0000","last_became_peered":"2023-09-23T15:21:57.137294+0000","last_unstale":"2023-09-23T15:25:25.232929+0000","last_undegraded":"2023-09-23T15:25:25.232929+0000","last_fullsized":"2023-09-23T15:25:25.232929+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:38.980224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":228,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227734+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:25:25.227734+0000","last_peered":"2023-09-23T15:25:25.227734+0000","last_clean":"2023-09-23T15:25:25.227734+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:25:25.227734+0000","last_undegraded":"2023-09-23T15:25:25.227734+0000","last_fullsized":"2023-09-23T15:25:25.227734+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":71,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227639+0000","last_change":"2023-09-23T15:22:27.318346+0000","last_active":"2023-09-23T15:25:25.227639+0000","last_peered":"2023-09-23T15:25:25.227639+0000","last_clean":"2023-09-23T15:25:25.227639+0000","last_became_active":"2023-09-23T15:22:27.317777+0000","last_became_peered":"2023-09-23T15:22:27.317777+0000","last_unstale":"2023-09-23T15:25:25.227639+0000","last_undegraded":"2023-09-23T15:25:25.227639+0000","last_fullsized":"2023-09-23T15:25:25.227639+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:26:28.243899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227518+0000","last_change":"2023-09-23T15:22:27.319262+0000","last_active":"2023-09-23T15:25:25.227518+0000","last_peered":"2023-09-23T15:25:25.227518+0000","last_clean":"2023-09-23T15:25:25.227518+0000","last_became_active":"2023-09-23T15:22:27.318000+0000","last_became_peered":"2023-09-23T15:22:27.318000+0000","last_unstale":"2023-09-23T15:25:25.227518+0000","last_undegraded":"2023-09-23T15:25:25.227518+0000","last_fullsized":"2023-09-23T15:25:25.227518+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:20:48.649793+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":646,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.243305+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:25:25.243305+0000","last_peered":"2023-09-23T15:25:25.243305+0000","last_clean":"2023-09-23T15:25:25.243305+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:25:25.243305+0000","last_undegraded":"2023-09-23T15:25:25.243305+0000","last_fullsized":"2023-09-23T15:25:25.243305+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":85,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232998+0000","last_change":"2023-09-23T15:21:58.157014+0000","last_active":"2023-09-23T15:25:25.232998+0000","last_peered":"2023-09-23T15:25:25.232998+0000","last_clean":"2023-09-23T15:25:25.232998+0000","last_became_active":"2023-09-23T15:21:58.143740+0000","last_became_peered":"2023-09-23T15:21:58.143740+0000","last_unstale":"2023-09-23T15:25:25.232998+0000","last_undegraded":"2023-09-23T15:25:25.232998+0000","last_fullsized":"2023-09-23T15:25:25.232998+0000","mapping_epoch":297,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":298,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:20.019387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":341,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231809+0000","last_change":"2023-09-23T15:21:19.256204+0000","last_active":"2023-09-23T15:25:25.231809+0000","last_peered":"2023-09-23T15:25:25.231809+0000","last_clean":"2023-09-23T15:25:25.231809+0000","last_became_active":"2023-09-23T15:21:19.255817+0000","last_became_peered":"2023-09-23T15:21:19.255817+0000","last_unstale":"2023-09-23T15:25:25.231809+0000","last_undegraded":"2023-09-23T15:25:25.231809+0000","last_fullsized":"2023-09-23T15:25:25.231809+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:38.619909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":695,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228646+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:25:25.228646+0000","last_peered":"2023-09-23T15:25:25.228646+0000","last_clean":"2023-09-23T15:25:25.228646+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:25:25.228646+0000","last_undegraded":"2023-09-23T15:25:25.228646+0000","last_fullsized":"2023-09-23T15:25:25.228646+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":314,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.240664+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:25:25.240664+0000","last_peered":"2023-09-23T15:25:25.240664+0000","last_clean":"2023-09-23T15:25:25.240664+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:25:25.240664+0000","last_undegraded":"2023-09-23T15:25:25.240664+0000","last_fullsized":"2023-09-23T15:25:25.240664+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":322,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229092+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:25:25.229092+0000","last_peered":"2023-09-23T15:25:25.229092+0000","last_clean":"2023-09-23T15:25:25.229092+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:25:25.229092+0000","last_undegraded":"2023-09-23T15:25:25.229092+0000","last_fullsized":"2023-09-23T15:25:25.229092+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":627,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229789+0000","last_change":"2023-09-23T15:22:00.279314+0000","last_active":"2023-09-23T15:25:25.229789+0000","last_peered":"2023-09-23T15:25:25.229789+0000","last_clean":"2023-09-23T15:25:25.229789+0000","last_became_active":"2023-09-23T15:22:00.279045+0000","last_became_peered":"2023-09-23T15:22:00.279045+0000","last_unstale":"2023-09-23T15:25:25.229789+0000","last_undegraded":"2023-09-23T15:25:25.229789+0000","last_fullsized":"2023-09-23T15:25:25.229789+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:46.856088+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":85,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232660+0000","last_change":"2023-09-23T15:22:27.307123+0000","last_active":"2023-09-23T15:25:25.232660+0000","last_peered":"2023-09-23T15:25:25.232660+0000","last_clean":"2023-09-23T15:25:25.232660+0000","last_became_active":"2023-09-23T15:22:27.306842+0000","last_became_peered":"2023-09-23T15:22:27.306842+0000","last_unstale":"2023-09-23T15:25:25.232660+0000","last_undegraded":"2023-09-23T15:25:25.232660+0000","last_fullsized":"2023-09-23T15:25:25.232660+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:41.344840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":303,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244948+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:25:25.244948+0000","last_peered":"2023-09-23T15:25:25.244948+0000","last_clean":"2023-09-23T15:25:25.244948+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:25:25.244948+0000","last_undegraded":"2023-09-23T15:25:25.244948+0000","last_fullsized":"2023-09-23T15:25:25.244948+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":653,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228833+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:25:25.228833+0000","last_peered":"2023-09-23T15:25:25.228833+0000","last_clean":"2023-09-23T15:25:25.228833+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:25:25.228833+0000","last_undegraded":"2023-09-23T15:25:25.228833+0000","last_fullsized":"2023-09-23T15:25:25.228833+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":273,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.226823+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:25:25.226823+0000","last_peered":"2023-09-23T15:25:25.226823+0000","last_clean":"2023-09-23T15:25:25.226823+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:25:25.226823+0000","last_undegraded":"2023-09-23T15:25:25.226823+0000","last_fullsized":"2023-09-23T15:25:25.226823+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":340,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231706+0000","last_change":"2023-09-23T15:21:19.252286+0000","last_active":"2023-09-23T15:25:25.231706+0000","last_peered":"2023-09-23T15:25:25.231706+0000","last_clean":"2023-09-23T15:25:25.231706+0000","last_became_active":"2023-09-23T15:21:19.251964+0000","last_became_peered":"2023-09-23T15:21:19.251964+0000","last_unstale":"2023-09-23T15:25:25.231706+0000","last_undegraded":"2023-09-23T15:25:25.231706+0000","last_fullsized":"2023-09-23T15:25:25.231706+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:15:06.741789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":599,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.233055+0000","last_change":"2023-09-23T15:22:00.252534+0000","last_active":"2023-09-23T15:25:25.233055+0000","last_peered":"2023-09-23T15:25:25.233055+0000","last_clean":"2023-09-23T15:25:25.233055+0000","last_became_active":"2023-09-23T15:22:00.252141+0000","last_became_peered":"2023-09-23T15:22:00.252141+0000","last_unstale":"2023-09-23T15:25:25.233055+0000","last_undegraded":"2023-09-23T15:25:25.233055+0000","last_fullsized":"2023-09-23T15:25:25.233055+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:00.962781+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":77,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.233256+0000","last_change":"2023-09-23T15:21:57.151449+0000","last_active":"2023-09-23T15:25:25.233256+0000","last_peered":"2023-09-23T15:25:25.233256+0000","last_clean":"2023-09-23T15:25:25.233256+0000","last_became_active":"2023-09-23T15:21:57.138055+0000","last_became_peered":"2023-09-23T15:21:57.138055+0000","last_unstale":"2023-09-23T15:25:25.233256+0000","last_undegraded":"2023-09-23T15:25:25.233256+0000","last_fullsized":"2023-09-23T15:25:25.233256+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:10.616693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":329,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228483+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:25:25.228483+0000","last_peered":"2023-09-23T15:25:25.228483+0000","last_clean":"2023-09-23T15:25:25.228483+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:25:25.228483+0000","last_undegraded":"2023-09-23T15:25:25.228483+0000","last_fullsized":"2023-09-23T15:25:25.228483+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":657,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231724+0000","last_change":"2023-09-23T15:22:26.296651+0000","last_active":"2023-09-23T15:25:25.231724+0000","last_peered":"2023-09-23T15:25:25.231724+0000","last_clean":"2023-09-23T15:25:25.231724+0000","last_became_active":"2023-09-23T15:22:26.296262+0000","last_became_peered":"2023-09-23T15:22:26.296262+0000","last_unstale":"2023-09-23T15:25:25.231724+0000","last_undegraded":"2023-09-23T15:25:25.231724+0000","last_fullsized":"2023-09-23T15:25:25.231724+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:46.628805+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:46.628805+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:30:26.661018+0000","scrub_duration":0.001685803,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":77,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.233447+0000","last_change":"2023-09-23T15:21:57.142587+0000","last_active":"2023-09-23T15:25:25.233447+0000","last_peered":"2023-09-23T15:25:25.233447+0000","last_clean":"2023-09-23T15:25:25.233447+0000","last_became_active":"2023-09-23T15:21:57.134717+0000","last_became_peered":"2023-09-23T15:21:57.134717+0000","last_unstale":"2023-09-23T15:25:25.233447+0000","last_undegraded":"2023-09-23T15:25:25.233447+0000","last_fullsized":"2023-09-23T15:25:25.233447+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:51.689707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227214+0000","last_change":"2023-09-23T15:21:57.140208+0000","last_active":"2023-09-23T15:25:25.227214+0000","last_peered":"2023-09-23T15:25:25.227214+0000","last_clean":"2023-09-23T15:25:25.227214+0000","last_became_active":"2023-09-23T15:21:57.139852+0000","last_became_peered":"2023-09-23T15:21:57.139852+0000","last_unstale":"2023-09-23T15:25:25.227214+0000","last_undegraded":"2023-09-23T15:25:25.227214+0000","last_fullsized":"2023-09-23T15:25:25.227214+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:11:09.928167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":544,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227575+0000","last_change":"2023-09-23T15:22:27.308266+0000","last_active":"2023-09-23T15:25:25.227575+0000","last_peered":"2023-09-23T15:25:25.227575+0000","last_clean":"2023-09-23T15:25:25.227575+0000","last_became_active":"2023-09-23T15:22:27.307894+0000","last_became_peered":"2023-09-23T15:22:27.307894+0000","last_unstale":"2023-09-23T15:25:25.227575+0000","last_undegraded":"2023-09-23T15:25:25.227575+0000","last_fullsized":"2023-09-23T15:25:25.227575+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:10.288224+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":77,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232901+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:25:25.232901+0000","last_peered":"2023-09-23T15:25:25.232901+0000","last_clean":"2023-09-23T15:25:25.232901+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:25:25.232901+0000","last_undegraded":"2023-09-23T15:25:25.232901+0000","last_fullsized":"2023-09-23T15:25:25.232901+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":300,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.226799+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:25:25.226799+0000","last_peered":"2023-09-23T15:25:25.226799+0000","last_clean":"2023-09-23T15:25:25.226799+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:25:25.226799+0000","last_undegraded":"2023-09-23T15:25:25.226799+0000","last_fullsized":"2023-09-23T15:25:25.226799+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":552,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228031+0000","last_change":"2023-09-23T15:22:00.275131+0000","last_active":"2023-09-23T15:25:25.228031+0000","last_peered":"2023-09-23T15:25:25.228031+0000","last_clean":"2023-09-23T15:25:25.228031+0000","last_became_active":"2023-09-23T15:22:00.274694+0000","last_became_peered":"2023-09-23T15:22:00.274694+0000","last_unstale":"2023-09-23T15:25:25.228031+0000","last_undegraded":"2023-09-23T15:25:25.228031+0000","last_fullsized":"2023-09-23T15:25:25.228031+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:23.409905+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:23.409905+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":313,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229264+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:25:25.229264+0000","last_peered":"2023-09-23T15:25:25.229264+0000","last_clean":"2023-09-23T15:25:25.229264+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:25:25.229264+0000","last_undegraded":"2023-09-23T15:25:25.229264+0000","last_fullsized":"2023-09-23T15:25:25.229264+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":319,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244613+0000","last_change":"2023-09-23T15:21:57.143048+0000","last_active":"2023-09-23T15:25:25.244613+0000","last_peered":"2023-09-23T15:25:25.244613+0000","last_clean":"2023-09-23T15:25:25.244613+0000","last_became_active":"2023-09-23T15:21:57.142663+0000","last_became_peered":"2023-09-23T15:21:57.142663+0000","last_unstale":"2023-09-23T15:25:25.244613+0000","last_undegraded":"2023-09-23T15:25:25.244613+0000","last_fullsized":"2023-09-23T15:25:25.244613+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:20.661093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":573,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232764+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:25:25.232764+0000","last_peered":"2023-09-23T15:25:25.232764+0000","last_clean":"2023-09-23T15:25:25.232764+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:25:25.232764+0000","last_undegraded":"2023-09-23T15:25:25.232764+0000","last_fullsized":"2023-09-23T15:25:25.232764+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":330,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231379+0000","last_change":"2023-09-23T15:21:57.153391+0000","last_active":"2023-09-23T15:25:25.231379+0000","last_peered":"2023-09-23T15:25:25.231379+0000","last_clean":"2023-09-23T15:25:25.231379+0000","last_became_active":"2023-09-23T15:21:57.141486+0000","last_became_peered":"2023-09-23T15:21:57.141486+0000","last_unstale":"2023-09-23T15:25:25.231379+0000","last_undegraded":"2023-09-23T15:25:25.231379+0000","last_fullsized":"2023-09-23T15:25:25.231379+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.121277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227562+0000","last_change":"2023-09-23T15:22:27.318677+0000","last_active":"2023-09-23T15:25:25.227562+0000","last_peered":"2023-09-23T15:25:25.227562+0000","last_clean":"2023-09-23T15:25:25.227562+0000","last_became_active":"2023-09-23T15:22:27.318055+0000","last_became_peered":"2023-09-23T15:22:27.318055+0000","last_unstale":"2023-09-23T15:25:25.227562+0000","last_undegraded":"2023-09-23T15:25:25.227562+0000","last_fullsized":"2023-09-23T15:25:25.227562+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:48.009120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":641,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228700+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:25:25.228700+0000","last_peered":"2023-09-23T15:25:25.228700+0000","last_clean":"2023-09-23T15:25:25.228700+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:25:25.228700+0000","last_undegraded":"2023-09-23T15:25:25.228700+0000","last_fullsized":"2023-09-23T15:25:25.228700+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":284,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.226902+0000","last_change":"2023-09-23T15:22:27.319953+0000","last_active":"2023-09-23T15:25:25.226902+0000","last_peered":"2023-09-23T15:25:25.226902+0000","last_clean":"2023-09-23T15:25:25.226902+0000","last_became_active":"2023-09-23T15:22:27.319625+0000","last_became_peered":"2023-09-23T15:22:27.319625+0000","last_unstale":"2023-09-23T15:25:25.226902+0000","last_undegraded":"2023-09-23T15:25:25.226902+0000","last_fullsized":"2023-09-23T15:25:25.226902+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:27.908732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227889+0000","last_change":"2023-09-23T15:21:57.136653+0000","last_active":"2023-09-23T15:25:25.227889+0000","last_peered":"2023-09-23T15:25:25.227889+0000","last_clean":"2023-09-23T15:25:25.227889+0000","last_became_active":"2023-09-23T15:21:57.135918+0000","last_became_peered":"2023-09-23T15:21:57.135918+0000","last_unstale":"2023-09-23T15:25:25.227889+0000","last_undegraded":"2023-09-23T15:25:25.227889+0000","last_fullsized":"2023-09-23T15:25:25.227889+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:41.384200+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":593,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229705+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:25:25.229705+0000","last_peered":"2023-09-23T15:25:25.229705+0000","last_clean":"2023-09-23T15:25:25.229705+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:25:25.229705+0000","last_undegraded":"2023-09-23T15:25:25.229705+0000","last_fullsized":"2023-09-23T15:25:25.229705+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227262+0000","last_change":"2023-09-23T15:21:57.139817+0000","last_active":"2023-09-23T15:25:25.227262+0000","last_peered":"2023-09-23T15:25:25.227262+0000","last_clean":"2023-09-23T15:25:25.227262+0000","last_became_active":"2023-09-23T15:21:57.132988+0000","last_became_peered":"2023-09-23T15:21:57.132988+0000","last_unstale":"2023-09-23T15:25:25.227262+0000","last_undegraded":"2023-09-23T15:25:25.227262+0000","last_fullsized":"2023-09-23T15:25:25.227262+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:14:03.171094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227977+0000","last_change":"2023-09-23T15:22:27.310325+0000","last_active":"2023-09-23T15:25:25.227977+0000","last_peered":"2023-09-23T15:25:25.227977+0000","last_clean":"2023-09-23T15:25:25.227977+0000","last_became_active":"2023-09-23T15:22:27.309727+0000","last_became_peered":"2023-09-23T15:22:27.309727+0000","last_unstale":"2023-09-23T15:25:25.227977+0000","last_undegraded":"2023-09-23T15:25:25.227977+0000","last_fullsized":"2023-09-23T15:25:25.227977+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:09.873810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":653,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228742+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:25:25.228742+0000","last_peered":"2023-09-23T15:25:25.228742+0000","last_clean":"2023-09-23T15:25:25.228742+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:25:25.228742+0000","last_undegraded":"2023-09-23T15:25:25.228742+0000","last_fullsized":"2023-09-23T15:25:25.228742+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":351,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229839+0000","last_change":"2023-09-23T15:21:19.253660+0000","last_active":"2023-09-23T15:25:25.229839+0000","last_peered":"2023-09-23T15:25:25.229839+0000","last_clean":"2023-09-23T15:25:25.229839+0000","last_became_active":"2023-09-23T15:21:19.253325+0000","last_became_peered":"2023-09-23T15:21:19.253325+0000","last_unstale":"2023-09-23T15:25:25.229839+0000","last_undegraded":"2023-09-23T15:25:25.229839+0000","last_fullsized":"2023-09-23T15:25:25.229839+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:17.550995+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":245,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228145+0000","last_change":"2023-09-23T15:21:57.137577+0000","last_active":"2023-09-23T15:25:25.228145+0000","last_peered":"2023-09-23T15:25:25.228145+0000","last_clean":"2023-09-23T15:25:25.228145+0000","last_became_active":"2023-09-23T15:21:57.137224+0000","last_became_peered":"2023-09-23T15:21:57.137224+0000","last_unstale":"2023-09-23T15:25:25.228145+0000","last_undegraded":"2023-09-23T15:25:25.228145+0000","last_fullsized":"2023-09-23T15:25:25.228145+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:14.865666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":534,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227501+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:25:25.227501+0000","last_peered":"2023-09-23T15:25:25.227501+0000","last_clean":"2023-09-23T15:25:25.227501+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:25:25.227501+0000","last_undegraded":"2023-09-23T15:25:25.227501+0000","last_fullsized":"2023-09-23T15:25:25.227501+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":537,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228003+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:25:25.228003+0000","last_peered":"2023-09-23T15:25:25.228003+0000","last_clean":"2023-09-23T15:25:25.228003+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:25:25.228003+0000","last_undegraded":"2023-09-23T15:25:25.228003+0000","last_fullsized":"2023-09-23T15:25:25.228003+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":313,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229067+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:25:25.229067+0000","last_peered":"2023-09-23T15:25:25.229067+0000","last_clean":"2023-09-23T15:25:25.229067+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:25:25.229067+0000","last_undegraded":"2023-09-23T15:25:25.229067+0000","last_fullsized":"2023-09-23T15:25:25.229067+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":281,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.242799+0000","last_change":"2023-09-23T15:21:57.138068+0000","last_active":"2023-09-23T15:25:25.242799+0000","last_peered":"2023-09-23T15:25:25.242799+0000","last_clean":"2023-09-23T15:25:25.242799+0000","last_became_active":"2023-09-23T15:21:57.137381+0000","last_became_peered":"2023-09-23T15:21:57.137381+0000","last_unstale":"2023-09-23T15:25:25.242799+0000","last_undegraded":"2023-09-23T15:25:25.242799+0000","last_fullsized":"2023-09-23T15:25:25.242799+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:01.723767+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":634,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228655+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:25:25.228655+0000","last_peered":"2023-09-23T15:25:25.228655+0000","last_clean":"2023-09-23T15:25:25.228655+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:25:25.228655+0000","last_undegraded":"2023-09-23T15:25:25.228655+0000","last_fullsized":"2023-09-23T15:25:25.228655+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":872,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228967+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:25:25.228967+0000","last_peered":"2023-09-23T15:25:25.228967+0000","last_clean":"2023-09-23T15:25:25.228967+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:25:25.228967+0000","last_undegraded":"2023-09-23T15:25:25.228967+0000","last_fullsized":"2023-09-23T15:25:25.228967+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227463+0000","last_change":"2023-09-23T15:22:27.319041+0000","last_active":"2023-09-23T15:25:25.227463+0000","last_peered":"2023-09-23T15:25:25.227463+0000","last_clean":"2023-09-23T15:25:25.227463+0000","last_became_active":"2023-09-23T15:22:27.318523+0000","last_became_peered":"2023-09-23T15:22:27.318523+0000","last_unstale":"2023-09-23T15:25:25.227463+0000","last_undegraded":"2023-09-23T15:25:25.227463+0000","last_fullsized":"2023-09-23T15:25:25.227463+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:23:56.808734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":234,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227803+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:25:25.227803+0000","last_peered":"2023-09-23T15:25:25.227803+0000","last_clean":"2023-09-23T15:25:25.227803+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:25:25.227803+0000","last_undegraded":"2023-09-23T15:25:25.227803+0000","last_fullsized":"2023-09-23T15:25:25.227803+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":692,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232842+0000","last_change":"2023-09-23T15:22:00.153965+0000","last_active":"2023-09-23T15:25:25.232842+0000","last_peered":"2023-09-23T15:25:25.232842+0000","last_clean":"2023-09-23T15:25:25.232842+0000","last_became_active":"2023-09-23T15:22:00.153580+0000","last_became_peered":"2023-09-23T15:22:00.153580+0000","last_unstale":"2023-09-23T15:25:25.232842+0000","last_undegraded":"2023-09-23T15:25:25.232842+0000","last_fullsized":"2023-09-23T15:25:25.232842+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":300,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:50:20.735494+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":954,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.242055+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:25:25.242055+0000","last_peered":"2023-09-23T15:25:25.242055+0000","last_clean":"2023-09-23T15:25:25.242055+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:25:25.242055+0000","last_undegraded":"2023-09-23T15:25:25.242055+0000","last_fullsized":"2023-09-23T15:25:25.242055+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":229,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227555+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:25:25.227555+0000","last_peered":"2023-09-23T15:25:25.227555+0000","last_clean":"2023-09-23T15:25:25.227555+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:25:25.227555+0000","last_undegraded":"2023-09-23T15:25:25.227555+0000","last_fullsized":"2023-09-23T15:25:25.227555+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":237,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228065+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:25:25.228065+0000","last_peered":"2023-09-23T15:25:25.228065+0000","last_clean":"2023-09-23T15:25:25.228065+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:25:25.228065+0000","last_undegraded":"2023-09-23T15:25:25.228065+0000","last_fullsized":"2023-09-23T15:25:25.228065+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":666,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229073+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:25:25.229073+0000","last_peered":"2023-09-23T15:25:25.229073+0000","last_clean":"2023-09-23T15:25:25.229073+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:25:25.229073+0000","last_undegraded":"2023-09-23T15:25:25.229073+0000","last_fullsized":"2023-09-23T15:25:25.229073+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":656,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232798+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:25:25.232798+0000","last_peered":"2023-09-23T15:25:25.232798+0000","last_clean":"2023-09-23T15:25:25.232798+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:25:25.232798+0000","last_undegraded":"2023-09-23T15:25:25.232798+0000","last_fullsized":"2023-09-23T15:25:25.232798+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":271,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231013+0000","last_change":"2023-09-23T15:22:26.296939+0000","last_active":"2023-09-23T15:25:25.231013+0000","last_peered":"2023-09-23T15:25:25.231013+0000","last_clean":"2023-09-23T15:25:25.231013+0000","last_became_active":"2023-09-23T15:22:26.296571+0000","last_became_peered":"2023-09-23T15:22:26.296571+0000","last_unstale":"2023-09-23T15:25:25.231013+0000","last_undegraded":"2023-09-23T15:25:25.231013+0000","last_fullsized":"2023-09-23T15:25:25.231013+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:30:05.502316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":288,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228666+0000","last_change":"2023-09-23T15:22:27.501553+0000","last_active":"2023-09-23T15:25:25.228666+0000","last_peered":"2023-09-23T15:25:25.228666+0000","last_clean":"2023-09-23T15:25:25.228666+0000","last_became_active":"2023-09-23T15:22:27.501277+0000","last_became_peered":"2023-09-23T15:22:27.501277+0000","last_unstale":"2023-09-23T15:25:25.228666+0000","last_undegraded":"2023-09-23T15:25:25.228666+0000","last_fullsized":"2023-09-23T15:25:25.228666+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:58.446929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":644,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229613+0000","last_change":"2023-09-23T15:22:27.329593+0000","last_active":"2023-09-23T15:25:25.229613+0000","last_peered":"2023-09-23T15:25:25.229613+0000","last_clean":"2023-09-23T15:25:25.229613+0000","last_became_active":"2023-09-23T15:22:27.329242+0000","last_became_peered":"2023-09-23T15:22:27.329242+0000","last_unstale":"2023-09-23T15:25:25.229613+0000","last_undegraded":"2023-09-23T15:25:25.229613+0000","last_fullsized":"2023-09-23T15:25:25.229613+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:33.241208+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":81,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.232738+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:25:25.232738+0000","last_peered":"2023-09-23T15:25:25.232738+0000","last_clean":"2023-09-23T15:25:25.232738+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:25:25.232738+0000","last_undegraded":"2023-09-23T15:25:25.232738+0000","last_fullsized":"2023-09-23T15:25:25.232738+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":305,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T15:25:26.374874+0000","last_change":"2023-09-23T15:21:57.142224+0000","last_active":"2023-09-23T15:25:26.374874+0000","last_peered":"2023-09-23T15:25:26.374874+0000","last_clean":"2023-09-23T15:25:26.374874+0000","last_became_active":"2023-09-23T15:21:57.141848+0000","last_became_peered":"2023-09-23T15:21:57.141848+0000","last_unstale":"2023-09-23T15:25:26.374874+0000","last_undegraded":"2023-09-23T15:25:26.374874+0000","last_fullsized":"2023-09-23T15:25:26.374874+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:05.458806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":316,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244681+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:25:25.244681+0000","last_peered":"2023-09-23T15:25:25.244681+0000","last_clean":"2023-09-23T15:25:25.244681+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:25:25.244681+0000","last_undegraded":"2023-09-23T15:25:25.244681+0000","last_fullsized":"2023-09-23T15:25:25.244681+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":615,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.226919+0000","last_change":"2023-09-23T15:24:24.592597+0000","last_active":"2023-09-23T15:25:25.226919+0000","last_peered":"2023-09-23T15:25:25.226919+0000","last_clean":"2023-09-23T15:25:25.226919+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:25:25.226919+0000","last_undegraded":"2023-09-23T15:25:25.226919+0000","last_fullsized":"2023-09-23T15:25:25.226919+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.409159+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.409159+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1050,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.233021+0000","last_change":"2023-09-23T15:24:25.990916+0000","last_active":"2023-09-23T15:25:25.233021+0000","last_peered":"2023-09-23T15:25:25.233021+0000","last_clean":"2023-09-23T15:25:25.233021+0000","last_became_active":"2023-09-23T15:22:27.434867+0000","last_became_peered":"2023-09-23T15:22:27.434867+0000","last_unstale":"2023-09-23T15:25:25.233021+0000","last_undegraded":"2023-09-23T15:25:25.233021+0000","last_fullsized":"2023-09-23T15:25:25.233021+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:49.834389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":214,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227909+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:25:25.227909+0000","last_peered":"2023-09-23T15:25:25.227909+0000","last_clean":"2023-09-23T15:25:25.227909+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:25:25.227909+0000","last_undegraded":"2023-09-23T15:25:25.227909+0000","last_fullsized":"2023-09-23T15:25:25.227909+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":301,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T15:25:26.374716+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:25:26.374716+0000","last_peered":"2023-09-23T15:25:26.374716+0000","last_clean":"2023-09-23T15:25:26.374716+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:25:26.374716+0000","last_undegraded":"2023-09-23T15:25:26.374716+0000","last_fullsized":"2023-09-23T15:25:26.374716+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":645,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227838+0000","last_change":"2023-09-23T15:22:27.318923+0000","last_active":"2023-09-23T15:25:25.227838+0000","last_peered":"2023-09-23T15:25:25.227838+0000","last_clean":"2023-09-23T15:25:25.227838+0000","last_became_active":"2023-09-23T15:22:27.318031+0000","last_became_peered":"2023-09-23T15:22:27.318031+0000","last_unstale":"2023-09-23T15:25:25.227838+0000","last_undegraded":"2023-09-23T15:25:25.227838+0000","last_fullsized":"2023-09-23T15:25:25.227838+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:54:01.190995+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":93,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.242258+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:25:25.242258+0000","last_peered":"2023-09-23T15:25:25.242258+0000","last_clean":"2023-09-23T15:25:25.242258+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:25:25.242258+0000","last_undegraded":"2023-09-23T15:25:25.242258+0000","last_fullsized":"2023-09-23T15:25:25.242258+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":890,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228343+0000","last_change":"2023-09-23T15:24:25.979031+0000","last_active":"2023-09-23T15:25:25.228343+0000","last_peered":"2023-09-23T15:25:25.228343+0000","last_clean":"2023-09-23T15:25:25.228343+0000","last_became_active":"2023-09-23T15:22:27.316424+0000","last_became_peered":"2023-09-23T15:22:27.316424+0000","last_unstale":"2023-09-23T15:25:25.228343+0000","last_undegraded":"2023-09-23T15:25:25.228343+0000","last_fullsized":"2023-09-23T15:25:25.228343+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":326,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.323211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":331,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229943+0000","last_change":"2023-09-23T15:21:19.251343+0000","last_active":"2023-09-23T15:25:25.229943+0000","last_peered":"2023-09-23T15:25:25.229943+0000","last_clean":"2023-09-23T15:25:25.229943+0000","last_became_active":"2023-09-23T15:21:19.250911+0000","last_became_peered":"2023-09-23T15:21:19.250911+0000","last_unstale":"2023-09-23T15:25:25.229943+0000","last_undegraded":"2023-09-23T15:25:25.229943+0000","last_fullsized":"2023-09-23T15:25:25.229943+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:40:40.512696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":662,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.231635+0000","last_change":"2023-09-23T15:21:22.264729+0000","last_active":"2023-09-23T15:25:25.231635+0000","last_peered":"2023-09-23T15:25:25.231635+0000","last_clean":"2023-09-23T15:25:25.231635+0000","last_became_active":"2023-09-23T15:21:22.264334+0000","last_became_peered":"2023-09-23T15:21:22.264334+0000","last_unstale":"2023-09-23T15:25:25.231635+0000","last_undegraded":"2023-09-23T15:25:25.231635+0000","last_fullsized":"2023-09-23T15:25:25.231635+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:24.540582+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":773,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227360+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:25:25.227360+0000","last_peered":"2023-09-23T15:25:25.227360+0000","last_clean":"2023-09-23T15:25:25.227360+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:25:25.227360+0000","last_undegraded":"2023-09-23T15:25:25.227360+0000","last_fullsized":"2023-09-23T15:25:25.227360+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":343,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244780+0000","last_change":"2023-09-23T15:21:57.149076+0000","last_active":"2023-09-23T15:25:25.244780+0000","last_peered":"2023-09-23T15:25:25.244780+0000","last_clean":"2023-09-23T15:25:25.244780+0000","last_became_active":"2023-09-23T15:21:57.141723+0000","last_became_peered":"2023-09-23T15:21:57.141723+0000","last_unstale":"2023-09-23T15:25:25.244780+0000","last_undegraded":"2023-09-23T15:25:25.244780+0000","last_fullsized":"2023-09-23T15:25:25.244780+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:13.645010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227171+0000","last_change":"2023-09-23T15:22:27.318646+0000","last_active":"2023-09-23T15:25:25.227171+0000","last_peered":"2023-09-23T15:25:25.227171+0000","last_clean":"2023-09-23T15:25:25.227171+0000","last_became_active":"2023-09-23T15:22:27.318285+0000","last_became_peered":"2023-09-23T15:22:27.318285+0000","last_unstale":"2023-09-23T15:25:25.227171+0000","last_undegraded":"2023-09-23T15:25:25.227171+0000","last_fullsized":"2023-09-23T15:25:25.227171+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:29.772687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":320,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244751+0000","last_change":"2023-09-23T15:21:57.143138+0000","last_active":"2023-09-23T15:25:25.244751+0000","last_peered":"2023-09-23T15:25:25.244751+0000","last_clean":"2023-09-23T15:25:25.244751+0000","last_became_active":"2023-09-23T15:21:57.142785+0000","last_became_peered":"2023-09-23T15:21:57.142785+0000","last_unstale":"2023-09-23T15:25:25.244751+0000","last_undegraded":"2023-09-23T15:25:25.244751+0000","last_fullsized":"2023-09-23T15:25:25.244751+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:49.284935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":340,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.244140+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:25:25.244140+0000","last_peered":"2023-09-23T15:25:25.244140+0000","last_clean":"2023-09-23T15:25:25.244140+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:25:25.244140+0000","last_undegraded":"2023-09-23T15:25:25.244140+0000","last_fullsized":"2023-09-23T15:25:25.244140+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":350,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229274+0000","last_change":"2023-09-23T15:21:57.154787+0000","last_active":"2023-09-23T15:25:25.229274+0000","last_peered":"2023-09-23T15:25:25.229274+0000","last_clean":"2023-09-23T15:25:25.229274+0000","last_became_active":"2023-09-23T15:21:57.142636+0000","last_became_peered":"2023-09-23T15:21:57.142636+0000","last_unstale":"2023-09-23T15:25:25.229274+0000","last_undegraded":"2023-09-23T15:25:25.229274+0000","last_fullsized":"2023-09-23T15:25:25.229274+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:04.909727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.227555+0000","last_change":"2023-09-23T15:21:57.141016+0000","last_active":"2023-09-23T15:25:25.227555+0000","last_peered":"2023-09-23T15:25:25.227555+0000","last_clean":"2023-09-23T15:25:25.227555+0000","last_became_active":"2023-09-23T15:21:57.136037+0000","last_became_peered":"2023-09-23T15:21:57.136037+0000","last_unstale":"2023-09-23T15:25:25.227555+0000","last_undegraded":"2023-09-23T15:25:25.227555+0000","last_fullsized":"2023-09-23T15:25:25.227555+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:38:44.513721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":330,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.229006+0000","last_change":"2023-09-23T15:22:27.307425+0000","last_active":"2023-09-23T15:25:25.229006+0000","last_peered":"2023-09-23T15:25:25.229006+0000","last_clean":"2023-09-23T15:25:25.229006+0000","last_became_active":"2023-09-23T15:22:27.307037+0000","last_became_peered":"2023-09-23T15:22:27.307037+0000","last_unstale":"2023-09-23T15:25:25.229006+0000","last_undegraded":"2023-09-23T15:25:25.229006+0000","last_fullsized":"2023-09-23T15:25:25.229006+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:28.326981+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":50,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.228002+0000","last_change":"2023-09-23T15:21:57.138874+0000","last_active":"2023-09-23T15:25:25.228002+0000","last_peered":"2023-09-23T15:25:25.228002+0000","last_clean":"2023-09-23T15:25:25.228002+0000","last_became_active":"2023-09-23T15:21:57.136459+0000","last_became_peered":"2023-09-23T15:21:57.136459+0000","last_unstale":"2023-09-23T15:25:25.228002+0000","last_undegraded":"2023-09-23T15:25:25.228002+0000","last_fullsized":"2023-09-23T15:25:25.228002+0000","mapping_epoch":296,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":297,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:03:56.125941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":328,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.230478+0000","last_change":"2023-09-23T15:22:26.296926+0000","last_active":"2023-09-23T15:25:25.230478+0000","last_peered":"2023-09-23T15:25:25.230478+0000","last_clean":"2023-09-23T15:25:25.230478+0000","last_became_active":"2023-09-23T15:22:26.296102+0000","last_became_peered":"2023-09-23T15:22:26.296102+0000","last_unstale":"2023-09-23T15:25:25.230478+0000","last_undegraded":"2023-09-23T15:25:25.230478+0000","last_fullsized":"2023-09-23T15:25:25.230478+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:46.958319+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":326,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.230710+0000","last_change":"2023-09-23T15:06:16.133197+0000","last_active":"2023-09-23T15:25:25.230710+0000","last_peered":"2023-09-23T15:25:25.230710+0000","last_clean":"2023-09-23T15:25:25.230710+0000","last_became_active":"2023-09-23T15:06:16.132891+0000","last_became_peered":"2023-09-23T15:06:16.132891+0000","last_unstale":"2023-09-23T15:25:25.230710+0000","last_undegraded":"2023-09-23T15:25:25.230710+0000","last_fullsized":"2023-09-23T15:25:25.230710+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:32.990729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":290,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T15:25:26.423797+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:25:26.423797+0000","last_peered":"2023-09-23T15:25:26.423797+0000","last_clean":"2023-09-23T15:25:26.423797+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:25:26.423797+0000","last_undegraded":"2023-09-23T15:25:26.423797+0000","last_fullsized":"2023-09-23T15:25:26.423797+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T15:25:25.226941+0000","last_change":"2023-09-23T15:22:27.310628+0000","last_active":"2023-09-23T15:25:25.226941+0000","last_peered":"2023-09-23T15:25:25.226941+0000","last_clean":"2023-09-23T15:25:25.226941+0000","last_became_active":"2023-09-23T15:22:27.310332+0000","last_became_peered":"2023-09-23T15:22:27.310332+0000","last_unstale":"2023-09-23T15:25:25.226941+0000","last_undegraded":"2023-09-23T15:25:25.226941+0000","last_fullsized":"2023-09-23T15:25:25.226941+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:18.017657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":5188,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":686385,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":112,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201649,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":348560,"kb_used_data":199816,"kb_used_omap":0,"kb_used_meta":148736,"kb_avail":93396592,"statfs":{"total":95995035648,"available":95638110208,"internally_reserved":0,"allocated":204611584,"data_stored":203574694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":152305664},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:25 2023","interfaces":[{"interface":"back","average":{"1min":3.96,"5min":7.3369999999999997,"15min":3.9489999999999998},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":35.908000000000001,"5min":226.226,"15min":226.226},"last":0.48699999999999999},{"interface":"front","average":{"1min":5.8099999999999996,"5min":8.8290000000000006,"15min":4.6070000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":35.939999999999998,"5min":226.619,"15min":226.619},"last":0.374}]},{"osd":1,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":18.012,"5min":4.9059999999999997,"15min":3.0590000000000002},"min":{"1min":0.39900000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":311.16699999999997,"5min":311.16699999999997,"15min":311.16699999999997},"last":0.51400000000000001},{"interface":"front","average":{"1min":18.634,"5min":5.3079999999999998,"15min":3.5590000000000002},"min":{"1min":0.30399999999999999,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":311.279,"5min":311.279,"15min":311.279},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4580000000000002,"5min":1.9930000000000001,"15min":2.7770000000000001},"min":{"1min":0.313,"5min":0.28799999999999998,"15min":0.24099999999999999},"max":{"1min":8.8219999999999992,"5min":8.8219999999999992,"15min":101.807},"last":0.437},{"interface":"front","average":{"1min":1.994,"5min":2.2210000000000001,"15min":3.331},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.308999999999999,"5min":10.308999999999999,"15min":101.84099999999999},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:25:04 2023","interfaces":[{"interface":"back","average":{"1min":2.3799999999999999,"5min":4.0880000000000001,"15min":3.9089999999999998},"min":{"1min":0.39500000000000002,"5min":0.308,"15min":0.28899999999999998},"max":{"1min":9.8000000000000007,"5min":146.166,"15min":146.166},"last":0.42499999999999999},{"interface":"front","average":{"1min":3.141,"5min":4.5049999999999999,"15min":4.0499999999999998},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":9.8179999999999996,"5min":146.18299999999999,"15min":146.18299999999999},"last":0.57099999999999995}]},{"osd":4,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":3.0219999999999998,"5min":1.696,"15min":2.7069999999999999},"min":{"1min":0.316,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.3550000000000004,"5min":16.771999999999998,"15min":150.119},"last":0.55200000000000005},{"interface":"front","average":{"1min":3.456,"5min":1.5269999999999999,"15min":2.8620000000000001},"min":{"1min":0.315,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":18.023,"5min":18.023,"15min":150.596},"last":0.54100000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:24:48 2023","interfaces":[{"interface":"back","average":{"1min":12.602,"5min":5.2270000000000003,"15min":3.5640000000000001},"min":{"1min":0.35599999999999998,"5min":0.29099999999999998,"15min":0.27800000000000002},"max":{"1min":216.85900000000001,"5min":224.441,"15min":224.441},"last":0.45700000000000002},{"interface":"front","average":{"1min":13.93,"5min":5.742,"15min":3.6669999999999998},"min":{"1min":0.39200000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":216.43799999999999,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:24:30 2023","interfaces":[{"interface":"back","average":{"1min":3.2799999999999998,"5min":2.4409999999999998,"15min":2.3010000000000002},"min":{"1min":0.48999999999999999,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":8.9659999999999993,"5min":9.2629999999999999,"15min":9.2629999999999999},"last":0.502},{"interface":"front","average":{"1min":3.0550000000000002,"5min":2.6230000000000002,"15min":2.5510000000000002},"min":{"1min":0.35699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":10.135,"5min":10.336,"15min":10.336},"last":0.59099999999999997}]}]},{"osd":6,"up_from":304,"seq":1305670058022,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":96584,"kb_used_data":3776,"kb_used_omap":13,"kb_used_meta":92786,"kb_avail":93648568,"statfs":{"total":95995035648,"available":95896133632,"internally_reserved":0,"allocated":3866624,"data_stored":2837799,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":95013219},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":16.183,"5min":4.4199999999999999,"15min":2.46},"min":{"1min":0.26100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":184.53200000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":8.5370000000000008,"5min":4.0090000000000003,"15min":3.2549999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":86.227999999999994,"5min":86.227999999999994,"15min":86.227999999999994},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":9.2279999999999998,"5min":4.0389999999999997,"15min":3.1739999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":86.534000000000006,"5min":86.534000000000006,"15min":86.534000000000006},"last":0.51100000000000001},{"interface":"front","average":{"1min":8.5199999999999996,"5min":3.9929999999999999,"15min":3.2389999999999999},"min":{"1min":0.32100000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":86.144000000000005,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.442}]},{"osd":2,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.706,"5min":1.762,"15min":1.6040000000000001},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":9.8469999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.0430000000000001,"5min":1.99,"15min":1.8140000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":9.1910000000000007,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":0.623}]},{"osd":3,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.4359999999999999,"5min":2.7109999999999999,"15min":2.423},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":41.261000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":0.495},{"interface":"front","average":{"1min":5.1559999999999997,"5min":2.8479999999999999,"15min":2.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":41.408000000000001,"5min":41.408000000000001,"15min":41.408000000000001},"last":0.64200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":2.165,"5min":1.1839999999999999,"15min":1.02},"min":{"1min":0.34300000000000003,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":8.1219999999999999,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":2.754,"5min":2.0070000000000001,"15min":1.883},"min":{"1min":0.39800000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":13.627000000000001,"5min":13.627000000000001,"15min":13.627000000000001},"last":0.55300000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":4.508,"5min":2.2749999999999999,"15min":1.903},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":13.74,"5min":13.74,"15min":13.74},"last":0.57299999999999995},{"interface":"front","average":{"1min":3.194,"5min":1.8260000000000001,"15min":1.5980000000000001},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":15.868,"5min":15.868,"15min":15.868},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:24:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1640000000000001,"5min":1.752,"15min":1.5169999999999999},"min":{"1min":0.46600000000000003,"5min":0.373,"15min":0.373},"max":{"1min":12.023999999999999,"5min":12.023999999999999,"15min":12.023999999999999},"last":0.67800000000000005},{"interface":"front","average":{"1min":3.5259999999999998,"5min":2.1629999999999998,"15min":1.9359999999999999},"min":{"1min":0.42199999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":12.641999999999999,"5min":12.641999999999999,"15min":12.641999999999999},"last":0.66200000000000003}]}]},{"osd":1,"up_from":183,"seq":785979015322,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":668924,"kb_used_data":204212,"kb_used_omap":6,"kb_used_meta":464697,"kb_avail":93076228,"statfs":{"total":95995035648,"available":95310057472,"internally_reserved":0,"allocated":209113088,"data_stored":207519352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":475849803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":2.7599999999999998,"5min":3.1619999999999999,"15min":3.3130000000000002},"min":{"1min":0.223,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":7.6379999999999999,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.26000000000000001},{"interface":"front","average":{"1min":3.6360000000000001,"5min":4.2119999999999997,"15min":4.0999999999999996},"min":{"1min":0.23799999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":17.128,"5min":63.594999999999999,"15min":63.594999999999999},"last":0.27900000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":4.1619999999999999,"5min":2.4750000000000001,"15min":4.0869999999999997},"min":{"1min":0.26500000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":12.429,"5min":12.429,"15min":109.375},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.597,"5min":2.1059999999999999,"15min":3.456},"min":{"1min":0.28100000000000003,"5min":0.219,"15min":0.219},"max":{"1min":7.758,"5min":12.259,"15min":105.16},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":3.8130000000000002,"15min":7.1580000000000004},"min":{"1min":0.32100000000000001,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":7.6820000000000004,"5min":64.540999999999997,"15min":211.405},"last":0.33300000000000002},{"interface":"front","average":{"1min":2.6549999999999998,"5min":4.1879999999999997,"15min":7.6929999999999996},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":15.637,"5min":67.465000000000003,"15min":210.988},"last":1.397}]},{"osd":4,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.7589999999999999,"5min":2.2999999999999998,"15min":3.3879999999999999},"min":{"1min":0.50800000000000001,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":13.112,"5min":13.379,"15min":80.570999999999998},"last":0.41199999999999998},{"interface":"front","average":{"1min":3.343,"5min":2.1640000000000001,"15min":3.407},"min":{"1min":0.313,"5min":0.313,"15min":0.308},"max":{"1min":8.7639999999999993,"5min":12.279999999999999,"15min":80.75},"last":0.47199999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.3719999999999999,"5min":3.9409999999999998,"15min":3.4380000000000002},"min":{"1min":0.374,"5min":0.33900000000000002,"15min":0.27400000000000002},"max":{"1min":15.561,"5min":115.401,"15min":115.401},"last":1.4970000000000001},{"interface":"front","average":{"1min":4.3680000000000003,"5min":3.9159999999999999,"15min":3.4340000000000002},"min":{"1min":0.54500000000000004,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":13.068,"5min":115.169,"15min":115.169},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:24:26 2023","interfaces":[{"interface":"back","average":{"1min":4.3940000000000001,"5min":3.335,"15min":3.1589999999999998},"min":{"1min":0.47599999999999998,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":19.553999999999998,"5min":19.553999999999998,"15min":19.553999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":2.0059999999999998,"5min":2.2869999999999999,"15min":2.3340000000000001},"min":{"1min":0.378,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":4.9770000000000003,"5min":9.3230000000000004,"15min":9.3230000000000004},"last":1.464}]},{"osd":7,"last update":"Sat Sep 23 15:25:10 2023","interfaces":[{"interface":"back","average":{"1min":3.9159999999999999,"5min":2.964,"15min":2.9980000000000002},"min":{"1min":0.71799999999999997,"5min":0.371,"15min":0.371},"max":{"1min":9.3469999999999995,"5min":19.702999999999999,"15min":19.702999999999999},"last":0.57899999999999996},{"interface":"front","average":{"1min":2.9740000000000002,"5min":2.4329999999999998,"15min":2.6549999999999998},"min":{"1min":0.38800000000000001,"5min":0.377,"15min":0.377},"max":{"1min":7.7859999999999996,"5min":20.722000000000001,"15min":20.722000000000001},"last":0.53000000000000003}]}]},{"osd":0,"up_from":95,"seq":408021893372,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898356,"kb_used_data":293676,"kb_used_omap":8,"kb_used_meta":604663,"kb_avail":92846796,"statfs":{"total":95995035648,"available":95075119104,"internally_reserved":0,"allocated":300724224,"data_stored":299260133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":619175508},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:25:02 2023","interfaces":[{"interface":"back","average":{"1min":10.018000000000001,"5min":3.879,"15min":3.8500000000000001},"min":{"1min":0.218,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":97.343000000000004,"5min":97.343000000000004,"15min":97.343000000000004},"last":0.73299999999999998},{"interface":"front","average":{"1min":9.1280000000000001,"5min":3.6589999999999998,"15min":3.681},"min":{"1min":0.23400000000000001,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":96.158000000000001,"5min":96.158000000000001,"15min":96.158000000000001},"last":0.93200000000000005}]},{"osd":2,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.4830000000000001,"5min":2.4020000000000001,"15min":4.3609999999999998},"min":{"1min":0.23300000000000001,"5min":0.223,"15min":0.17000000000000001},"max":{"1min":13.022,"5min":30.510999999999999,"15min":122.20099999999999},"last":0.755},{"interface":"front","average":{"1min":2.9700000000000002,"5min":2.7170000000000001,"15min":4.2359999999999998},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":9.2050000000000001,"5min":30.396999999999998,"15min":122.071},"last":0.65000000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":4.3140000000000001,"5min":4.8300000000000001,"15min":5.8819999999999997},"min":{"1min":0.29099999999999998,"5min":0.27400000000000002,"15min":0.20000000000000001},"max":{"1min":41.380000000000003,"5min":197.78899999999999,"15min":197.78899999999999},"last":0.60099999999999998},{"interface":"front","average":{"1min":4.8079999999999998,"5min":5.2050000000000001,"15min":6.3630000000000004},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.252},"max":{"1min":41.465000000000003,"5min":200.58799999999999,"15min":200.58799999999999},"last":0.68799999999999994}]},{"osd":4,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.2330000000000001,"5min":2.177,"15min":2.278},"min":{"1min":0.35399999999999998,"5min":0.33900000000000002,"15min":0.28000000000000003},"max":{"1min":15.038,"5min":15.358000000000001,"15min":31.341999999999999},"last":0.995},{"interface":"front","average":{"1min":2.327,"5min":1.873,"15min":2.2429999999999999},"min":{"1min":0.35099999999999998,"5min":0.315,"15min":0.27000000000000002},"max":{"1min":8.2629999999999999,"5min":9.8249999999999993,"15min":99.168999999999997},"last":0.56000000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3799999999999999,"5min":2.1880000000000002,"15min":2.456},"min":{"1min":0.39400000000000002,"5min":0.34799999999999998,"15min":0.28599999999999998},"max":{"1min":10.618,"5min":10.768000000000001,"15min":109.369},"last":0.61499999999999999},{"interface":"front","average":{"1min":3.7959999999999998,"5min":2.3879999999999999,"15min":2.528},"min":{"1min":0.39100000000000001,"5min":0.35299999999999998,"15min":0.28599999999999998},"max":{"1min":15.363,"5min":15.436999999999999,"15min":109.387},"last":0.96199999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":3.1970000000000001,"5min":2.887,"15min":2.835},"min":{"1min":0.45900000000000002,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":11.989000000000001,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.90300000000000002},{"interface":"front","average":{"1min":3.4630000000000001,"5min":3.3809999999999998,"15min":3.367},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":15.829000000000001,"5min":17.385000000000002,"15min":17.385000000000002},"last":0.51900000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":4.2779999999999996,"5min":2.4159999999999999,"15min":2.6219999999999999},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.36799999999999999},"max":{"1min":17.712,"5min":17.712,"15min":99.549999999999997},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.2629999999999999,"5min":2.556,"15min":2.2610000000000001},"min":{"1min":0.39300000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":13.706,"5min":15.407999999999999,"15min":31.225999999999999},"last":0.86699999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566319,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909616,"kb_used_data":294760,"kb_used_omap":11,"kb_used_meta":614836,"kb_avail":92835536,"statfs":{"total":95995035648,"available":95063588864,"internally_reserved":0,"allocated":301834240,"data_stored":300372640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":629592848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3209999999999997,"5min":7.2210000000000001,"15min":5.1600000000000001},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":59.308999999999997,"5min":88.622,"15min":88.622},"last":0.377},{"interface":"front","average":{"1min":4.6639999999999997,"5min":7.4119999999999999,"15min":4.8620000000000001},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":56.017000000000003,"5min":88.649000000000001,"15min":88.649000000000001},"last":0.29799999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":2.8860000000000001,"5min":3.9209999999999998,"15min":4.577},"min":{"1min":0.21099999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":9.0129999999999999,"5min":71.819000000000003,"15min":84.641000000000005},"last":0.32400000000000001},{"interface":"front","average":{"1min":3.7749999999999999,"5min":4.0910000000000002,"15min":3.6930000000000001},"min":{"1min":0.26000000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":12.036,"5min":71.805000000000007,"15min":84.751999999999995},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:24:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5409999999999999,"5min":5.5350000000000001,"15min":5.2850000000000001},"min":{"1min":0.24399999999999999,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":8.2970000000000006,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.314},{"interface":"front","average":{"1min":2.5190000000000001,"5min":5.9119999999999999,"15min":5.3650000000000002},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.20300000000000001},"max":{"1min":10.935,"5min":76.742000000000004,"15min":86.158000000000001},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.6379999999999999,"15min":2.4220000000000002},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":10.003,"5min":16.712,"15min":84.608000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.357,"5min":1.5800000000000001,"15min":2.9049999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":8.0519999999999996,"5min":8.0519999999999996,"15min":87.019999999999996},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.8280000000000001,"15min":2.6200000000000001},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.0140000000000002,"5min":7.6980000000000004,"15min":84.498000000000005},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.1560000000000001,"5min":1.845,"15min":2.5259999999999998},"min":{"1min":0.374,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":10.052,"5min":10.808999999999999,"15min":84.664000000000001},"last":0.44}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5099999999999998,"5min":2.3290000000000002,"15min":2.2989999999999999},"min":{"1min":0.36699999999999999,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.4600000000000009,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.96899999999999997,"5min":1.8340000000000001,"15min":1.978},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":3.6909999999999998,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.0659999999999998,"15min":2.5579999999999998},"min":{"1min":0.42599999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":11.199999999999999,"5min":11.199999999999999,"15min":84.596000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":2.6059999999999999,"5min":2.0960000000000001,"15min":2.5489999999999999},"min":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":10.028,"5min":16.074999999999999,"15min":84.783000000000001},"last":0.48399999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402793,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":614776,"kb_used_data":4528,"kb_used_omap":3,"kb_used_meta":610236,"kb_avail":93130376,"statfs":{"total":95995035648,"available":95365505024,"internally_reserved":0,"allocated":4636672,"data_stored":3167720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":624881833},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":3.234,"5min":3.613,"15min":3.4329999999999998},"min":{"1min":0.23499999999999999,"5min":0.183,"15min":0.17899999999999999},"max":{"1min":21.949000000000002,"5min":108.318,"15min":108.318},"last":0.33600000000000002},{"interface":"front","average":{"1min":3.3029999999999999,"5min":4.101,"15min":3.726},"min":{"1min":0.224,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":15.81,"5min":108.889,"15min":108.889},"last":0.73399999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:25:09 2023","interfaces":[{"interface":"back","average":{"1min":4.1760000000000002,"5min":4.7469999999999999,"15min":4.8070000000000004},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":13.603,"5min":126.875,"15min":126.875},"last":0.48299999999999998},{"interface":"front","average":{"1min":4.8650000000000002,"5min":5.532,"15min":5.0529999999999999},"min":{"1min":0.25900000000000001,"5min":0.23999999999999999,"15min":0.22900000000000001},"max":{"1min":22.006,"5min":128.249,"15min":128.249},"last":0.94299999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":2.665,"5min":2.2909999999999999,"15min":3.911},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.22},"max":{"1min":14.853,"5min":15.048999999999999,"15min":109.077},"last":0.60099999999999998},{"interface":"front","average":{"1min":2.1989999999999998,"5min":2.9609999999999999,"15min":4.8559999999999999},"min":{"1min":0.29699999999999999,"5min":0.27500000000000002,"15min":0.25700000000000001},"max":{"1min":9.6489999999999991,"5min":36.582000000000001,"15min":119.72799999999999},"last":0.35899999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.925,"15min":2.2709999999999999},"min":{"1min":0.33000000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":4.9219999999999997,"5min":36.551000000000002,"15min":54.051000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":3.3090000000000002,"5min":1.8660000000000001,"15min":2.2160000000000002},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":13.115,"5min":13.115,"15min":52.228999999999999},"last":0.873}]},{"osd":5,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":3.6669999999999998,"15min":2.9550000000000001},"min":{"1min":0.39900000000000002,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":7.9260000000000002,"5min":148.619,"15min":148.619},"last":0.83199999999999996},{"interface":"front","average":{"1min":3.972,"5min":3.383,"15min":2.9020000000000001},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":13.026999999999999,"5min":148.56999999999999,"15min":148.56999999999999},"last":0.92900000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:25:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7690000000000001,"5min":2.335,"15min":2.2610000000000001},"min":{"1min":0.42799999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":13.079000000000001,"5min":13.079000000000001,"15min":13.079000000000001},"last":0.79200000000000004},{"interface":"front","average":{"1min":2.0179999999999998,"5min":3.0030000000000001,"15min":3.4350000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":8.4269999999999996,"5min":36.517000000000003,"15min":36.517000000000003},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 15:24:52 2023","interfaces":[{"interface":"back","average":{"1min":3.7480000000000002,"5min":3.25,"15min":2.7749999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":13.153,"5min":145.25,"15min":145.25},"last":0.66600000000000004},{"interface":"front","average":{"1min":2.9990000000000001,"5min":3.2879999999999998,"15min":2.613},"min":{"1min":0.39800000000000002,"5min":0.38500000000000001,"15min":0.34799999999999998},"max":{"1min":13.058999999999999,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.85099999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986558,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":359372,"kb_used_data":3460,"kb_used_omap":0,"kb_used_meta":355904,"kb_avail":93385780,"statfs":{"total":95995035648,"available":95627038720,"internally_reserved":0,"allocated":3543040,"data_stored":2510176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":364445696},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.2930000000000001,"5min":4.2069999999999999,"15min":3.887},"min":{"1min":0.26900000000000002,"5min":0.22700000000000001,"15min":0.219},"max":{"1min":7.407,"5min":162.75999999999999,"15min":162.75999999999999},"last":0.35799999999999998},{"interface":"front","average":{"1min":3.3069999999999999,"5min":4.093,"15min":3.8769999999999998},"min":{"1min":0.34599999999999997,"5min":0.22600000000000001,"15min":0.221},"max":{"1min":10.923999999999999,"5min":162.34800000000001,"15min":162.34800000000001},"last":0.70399999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:25:05 2023","interfaces":[{"interface":"back","average":{"1min":2.7090000000000001,"5min":4.2359999999999998,"15min":3.3370000000000002},"min":{"1min":0.314,"5min":0.249,"15min":0.245},"max":{"1min":8.7249999999999996,"5min":113.11199999999999,"15min":113.11199999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.4329999999999998,"5min":4.1600000000000001,"15min":3.7069999999999999},"min":{"1min":0.38900000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.0129999999999999,"5min":113.02,"15min":113.02},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:25:17 2023","interfaces":[{"interface":"back","average":{"1min":2.5880000000000001,"5min":2.3660000000000001,"15min":3.7490000000000001},"min":{"1min":0.30399999999999999,"5min":0.28199999999999997,"15min":0.255},"max":{"1min":10.933,"5min":43.969000000000001,"15min":148.08000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":2.7519999999999998,"5min":2.7719999999999998,"15min":3.968},"min":{"1min":0.34100000000000003,"5min":0.30599999999999999,"15min":0.27800000000000002},"max":{"1min":6.694,"5min":44.006999999999998,"15min":147.88900000000001},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:24:50 2023","interfaces":[{"interface":"back","average":{"1min":3.282,"5min":2.9990000000000001,"15min":6.0659999999999998},"min":{"1min":0.34300000000000003,"5min":0.30399999999999999,"15min":0.252},"max":{"1min":8.1859999999999999,"5min":112.681,"15min":163.97499999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":2.6579999999999999,"5min":3.0289999999999999,"15min":6.0439999999999996},"min":{"1min":0.35999999999999999,"5min":0.32700000000000001,"15min":0.29499999999999998},"max":{"1min":7.1719999999999997,"5min":112.621,"15min":164.447},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:25:05 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.2190000000000001,"15min":1.3879999999999999},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":5.5510000000000002,"5min":6.4409999999999998,"15min":87.912999999999997},"last":0.76600000000000001},{"interface":"front","average":{"1min":1.22,"5min":1.403,"15min":1.3979999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":6.8250000000000002,"5min":14.449999999999999,"15min":88.602999999999994},"last":0.51700000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":1.448,"5min":1.446,"15min":1.446},"min":{"1min":0.33200000000000002,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":6.8010000000000002,"5min":7.6470000000000002,"15min":7.6470000000000002},"last":0.621},{"interface":"front","average":{"1min":1.7090000000000001,"5min":2.121,"15min":2.1899999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":7.5380000000000003,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":0.59299999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7889999999999999,"5min":1.177,"15min":1.0740000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.2999999999999998,"5min":6.0730000000000004,"15min":18.23},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.895,"5min":1.2709999999999999,"15min":1.1890000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":6.3239999999999998,"5min":6.3239999999999998,"15min":14.044},"last":0.73299999999999998}]}]},{"osd":5,"up_from":39,"seq":167503724921,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":394688,"kb_used_data":3704,"kb_used_omap":0,"kb_used_meta":390976,"kb_avail":93350464,"statfs":{"total":95995035648,"available":95590875136,"internally_reserved":0,"allocated":3792896,"data_stored":2768557,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":400359424},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":15,"apply_latency_ms":15,"commit_latency_ns":15000000,"apply_latency_ns":15000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:21 2023","interfaces":[{"interface":"back","average":{"1min":3.1110000000000002,"5min":3.2570000000000001,"15min":3.4380000000000002},"min":{"1min":0.23799999999999999,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.411999999999999,"5min":54.939999999999998,"15min":124.31},"last":0.51000000000000001},{"interface":"front","average":{"1min":2.6440000000000001,"5min":3.2629999999999999,"15min":3.4279999999999999},"min":{"1min":0.24299999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":16.312999999999999,"5min":55.043999999999997,"15min":124.374},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":3.319,"5min":4.875,"15min":3.2799999999999998},"min":{"1min":0.27800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":8.9320000000000004,"5min":86.900999999999996,"15min":124.38800000000001},"last":0.438},{"interface":"front","average":{"1min":2.6429999999999998,"5min":3.867,"15min":3.1160000000000001},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":7.0650000000000004,"5min":66.825000000000003,"15min":124.29000000000001},"last":0.68799999999999994}]},{"osd":2,"last update":"Sat Sep 23 15:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":2.6789999999999998,"15min":3.6890000000000001},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":5.0380000000000003,"5min":74.019000000000005,"15min":193.00999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.9209999999999998,"15min":3.9009999999999998},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":9.125,"5min":73.712000000000003,"15min":193.15700000000001},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":5.6630000000000003,"5min":5.2350000000000003,"15min":6.2359999999999998},"min":{"1min":0.39800000000000002,"5min":0.33400000000000002,"15min":0.27500000000000002},"max":{"1min":55.354999999999997,"5min":266.70299999999997,"15min":266.70299999999997},"last":1.6200000000000001},{"interface":"front","average":{"1min":5.9699999999999998,"5min":6.1470000000000002,"15min":6.3369999999999997},"min":{"1min":0.40400000000000003,"5min":0.30599999999999999,"15min":0.30199999999999999},"max":{"1min":55.417000000000002,"5min":266.80399999999997,"15min":266.80399999999997},"last":0.61199999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:25:11 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.7010000000000001,"15min":2.0390000000000001},"min":{"1min":0.33500000000000002,"5min":0.27800000000000002,"15min":0.26200000000000001},"max":{"1min":8.8689999999999998,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.72099999999999997},{"interface":"front","average":{"1min":2.621,"5min":1.714,"15min":2.089},"min":{"1min":0.35199999999999998,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":8.6349999999999998,"5min":10.516,"15min":138.49600000000001},"last":0.79700000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:24:29 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.4390000000000001,"15min":2.411},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":10.577999999999999,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":3.5249999999999999,"5min":2.633,"15min":2.484},"min":{"1min":0.36599999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":13.257999999999999,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.77500000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:24:49 2023","interfaces":[{"interface":"back","average":{"1min":4.0419999999999998,"5min":1.8260000000000001,"15min":1.478},"min":{"1min":0.40500000000000003,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":15.849,"5min":15.849,"15min":15.849},"last":0.82999999999999996},{"interface":"front","average":{"1min":3.7360000000000002,"5min":1.7090000000000001,"15min":1.45},"min":{"1min":0.435,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":16.134,"5min":16.134,"15min":16.134},"last":0.76200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:25:33.551 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:25:33.551 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:25:33.551 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T15:25:33.551 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-09-23T15:25:33.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-09-23T15:25:33.614 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:33.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:33.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:33.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:33 smithi017 ceph-mon[222878]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:33.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:33 smithi017 ceph-mon[215646]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:33.700 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:25:33.700 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:25:33.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:33.748 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:33 smithi027 ceph-mon[195233]: from='client.36183 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:25:34.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:25:34.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:25:34.702 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' created 2023-09-23T15:25:34.727 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:34.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:34.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:34.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:34 smithi027 ceph-mon[195233]: pgmap v1002: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:34.866 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:34.911 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:34 smithi017 ceph-mon[215646]: pgmap v1002: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:34.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:34.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:34 smithi017 ceph-mon[222878]: pgmap v1002: 93 pgs: 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:34.912 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:34.917 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:34.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:34.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:35.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:35.053 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:35.352 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:35.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:35.455 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:35.488 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:35.541 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:35.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:35.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:35.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T15:25:35.662 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[222878]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.663 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T15:25:35.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:35 smithi017 ceph-mon[215646]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.682 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:35.746 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:35.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:35.761 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:35.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:35.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2959699875' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T15:25:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:35 smithi027 ceph-mon[195233]: failed to encode map e339 with expected crc 2023-09-23T15:25:35.870 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:35.900 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:35.925 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:36.018 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:25:36.032 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:36.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:36.048 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:25:36.072 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:25:36.101 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:25:36.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:25:36.137 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:36.160 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:25:36.170 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:36.179 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:25:36.201 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.203 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:25:36.221 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:36.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:36.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:36.328 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:36.357 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:36.405 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:36.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:36.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:36.541 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: pgmap v1004: 109 pgs: 16 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/502506914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: pgmap v1004: 109 pgs: 16 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/502506914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:25:36.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: pgmap v1004: 109 pgs: 16 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:36.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/502506914' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:36.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:25:36.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:25:36.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:37.091 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:37.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:37.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:37.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:37.229 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:37.281 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:37.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:37.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:37.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:37.415 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:37.467 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:37.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:37.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:37.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:37.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[215646]: failed to encode map e340 with expected crc 2023-09-23T15:25:37.593 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[215646]: failed to encode map e341 with expected crc 2023-09-23T15:25:37.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[215646]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T15:25:37.594 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[222878]: failed to encode map e340 with expected crc 2023-09-23T15:25:37.594 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[222878]: failed to encode map e341 with expected crc 2023-09-23T15:25:37.594 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:37 smithi017 ceph-mon[222878]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T15:25:37.601 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:37.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:37 smithi027 ceph-mon[195233]: failed to encode map e340 with expected crc 2023-09-23T15:25:37.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:37 smithi027 ceph-mon[195233]: failed to encode map e341 with expected crc 2023-09-23T15:25:37.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:37 smithi027 ceph-mon[195233]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T15:25:38.336 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:38] "GET /metrics HTTP/1.1" 200 34018 "" "Prometheus/2.43.0" 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[215646]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[215646]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[215646]: pgmap v1007: 109 pgs: 8 creating+peering, 8 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[222878]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[222878]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:38 smithi017 ceph-mon[222878]: pgmap v1007: 109 pgs: 8 creating+peering, 8 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:38.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:38 smithi027 ceph-mon[195233]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:38 smithi027 ceph-mon[195233]: failed to encode map e341 with expected crc 2023-09-23T15:25:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:38 smithi027 ceph-mon[195233]: pgmap v1007: 109 pgs: 8 creating+peering, 8 unknown, 93 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:38.898 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:38.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:38.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:38.918 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:39.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:39.036 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:39.509 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:25:39.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/947418956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:39.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/947418956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/947418956' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:25:40.553 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:40.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:40.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:40.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: pgmap v1008: 109 pgs: 8 creating+peering, 6 unknown, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[222878]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: pgmap v1008: 109 pgs: 8 creating+peering, 6 unknown, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:40.647 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:25:40.648 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:40 smithi017 ceph-mon[215646]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T15:25:40.658 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:40.690 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64529 7.e 7:7d16ebd8:::benchmark_data_smithi017_422824_object64528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 475.93438800899997, 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.273188133, 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.785 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64529 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.786 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847696+0000", 2023-09-23T15:25:40.787 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0910000000000001e-05 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847703+0000", 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847709+0000", 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4909999999999996e-06 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.788 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847787+0000", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8174000000000001e-05 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847961+0000", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017452599999999999 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.848087+0000", 2023-09-23T15:25:40.789 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012575800000000001 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.850201+0000", 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021134719999999999 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120707+0000", 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.270506674 2023-09-23T15:25:40.790 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120747+0000", 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9394999999999998e-05 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120796+0000", 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9567000000000002e-05 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.791 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120863+0000", 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7117000000000001e-05 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64532 7.e 7:731a4c04:::benchmark_data_smithi017_422824_object64531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:25:40.792 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 475.93098895700001, 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28321638999999998, 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64532 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.793 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851073+0000", 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.794 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851107+0000", 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4767000000000001e-05 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851120+0000", 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2638e-05 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.795 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851128+0000", 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1519999999999992e-06 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851174+0000", 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6133e-05 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118070+0000", 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26689595700000002 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.796 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118233+0000", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016251300000000001 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.125753+0000", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075200090000000002 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.128398+0000", 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026455459999999999 2023-09-23T15:25:40.797 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134191+0000", 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057925729999999996 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134247+0000", 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6369999999999997e-05 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.798 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134291+0000", 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3470999999999998e-05 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64534 7.5 7:a537e92b:::benchmark_data_smithi017_422824_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 475.93092140599998, 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28027905600000003, 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.799 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64534 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.800 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851141+0000", 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.801 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851169+0000", 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7999000000000001e-05 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851177+0000", 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5970000000000003e-06 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851182+0000", 2023-09-23T15:25:40.802 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4040000000000003e-06 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117946+0000", 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266763213 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118119+0000", 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017326 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.803 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118260+0000", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140767 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129302+0000", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01104228 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129468+0000", 2023-09-23T15:25:40.804 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166195 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131318+0000", 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018502740000000001 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131369+0000", 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0327000000000001e-05 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.805 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131421+0000", 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2238999999999999e-05 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64539 7.5 7:aff3ef34:::benchmark_data_smithi017_422824_object64538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 475.930866068, 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28619797000000002, 2023-09-23T15:25:40.806 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64539 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.807 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.808 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851217+0000", 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0457000000000001e-05 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117948+0000", 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26673075899999998 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117957+0000", 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9560000000000003e-06 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.809 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120513+0000", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002556034 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120654+0000", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014076799999999999 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120798+0000", 2023-09-23T15:25:40.810 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144394 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129322+0000", 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085240860000000002 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129497+0000", 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000174737 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.811 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137299+0000", 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078019229999999997 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137345+0000", 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6551000000000002e-05 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137395+0000", 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9852999999999999e-05 2023-09-23T15:25:40.812 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126249 7.8 7:1dd9f547:::benchmark_data_smithi017_422824_object32379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 389.01057265200001, 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30324335800000002, 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.813 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126249 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.814 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771489+0000", 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:25:40.815 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771499+0000", 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0219e-05 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.816 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771514+0000", 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4487999999999999e-05 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771523+0000", 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.817 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771607+0000", 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3652999999999993e-05 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771729+0000", 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012218299999999999 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771846+0000", 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011697900000000001 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.818 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.779901+0000", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080551970000000001 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.782538+0000", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002637393 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074627+0000", 2023-09-23T15:25:40.819 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29208838199999998 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074687+0000", 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0074000000000001e-05 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074734+0000", 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7129000000000002e-05 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.820 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126250 7.8 7:12de4e94:::benchmark_data_smithi017_422824_object32380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 389.01051566299998, 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30340214700000001, 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.821 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126250 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.822 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771547+0000", 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771551+0000", 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.873e-06 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.823 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771562+0000", 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1371e-05 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771569+0000", 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775770+0000", 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042005970000000004 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.824 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775901+0000", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131066 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.776027+0000", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012558299999999999 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.781218+0000", 2023-09-23T15:25:40.825 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051909429999999999 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.788400+0000", 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007182127 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074879+0000", 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28647872800000002 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.826 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074918+0000", 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9526000000000001e-05 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074950+0000", 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1764999999999998e-05 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147540 7.5 7:a064fe8e:::benchmark_data_smithi017_422824_object53670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 352.65335942399997, 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35421415899999997, 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147540 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128703+0000", 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128709+0000", 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2330000000000003e-06 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128731+0000", 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2753000000000001e-05 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128739+0000", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128795+0000", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5846e-05 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128906+0000", 2023-09-23T15:25:40.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111094 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.129011+0000", 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010522 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.132677+0000", 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036654330000000001 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.134043+0000", 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013660079999999999 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482827+0000", 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.348784551 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482880+0000", 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2193e-05 2023-09-23T15:25:40.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482918+0000", 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8566999999999999e-05 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147555 7.e 7:7b3c5bac:::benchmark_data_smithi017_422824_object53685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:25:40.834 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 352.64357089499998, 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35786704400000002, 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147555 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:25:40.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138491+0000", 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138497+0000", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7359999999999998e-06 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138506+0000", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4890000000000001e-06 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138512+0000", 2023-09-23T15:25:40.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.344e-06 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138557+0000", 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5674999999999999e-05 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428127+0000", 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28956913299999998 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130603 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432003+0000", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037461870000000002 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432016+0000", 2023-09-23T15:25:40.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2327000000000001e-05 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496277+0000", 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.064261756000000003 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496320+0000", 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2994999999999997e-05 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496359+0000", 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9004000000000001e-05 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147557 7.5 7:a3342b39:::benchmark_data_smithi017_422824_object53687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 352.64113756099999, 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35537888099999998, 2023-09-23T15:25:40.841 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147557 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140925+0000", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140930+0000", 2023-09-23T15:25:40.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1170000000000003e-06 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428068+0000", 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28713744600000002 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428083+0000", 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5126000000000001e-05 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428149+0000", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010804899999999999 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428331+0000", 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4103000000000005e-05 2023-09-23T15:25:40.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432563+0000", 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042319059999999997 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432991+0000", 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042866099999999999 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496219+0000", 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063227189000000003 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496265+0000", 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6489000000000002e-05 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496305+0000", 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9665000000000001e-05 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.847 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:148579 7.8 7:14985ae7:::benchmark_data_smithi017_422824_object54709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 350.63928335100002, 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27762552400000001, 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.848 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 148579 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142779+0000", 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142785+0000", 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7640000000000002e-06 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142795+0000", 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.042e-05 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142816+0000", 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0897e-05 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143029+0000", 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021310799999999999 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143147+0000", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117836 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143269+0000", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001225 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.151514+0000", 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082451160000000003 2023-09-23T15:25:40.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156566+0000", 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005051303 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156883+0000", 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000317378 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420224+0000", 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.263340776 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420275+0000", 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.058e-05 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420406+0000", 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013101000000000001 2023-09-23T15:25:40.854 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165850 7.8 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 319.32654419900001, 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.360395309, 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.855 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165850 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:25:40.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455522+0000", 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.613e-06 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455530+0000", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2649999999999996e-06 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455537+0000", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1770000000000003e-06 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455589+0000", 2023-09-23T15:25:40.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2590999999999998e-05 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455703+0000", 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011421000000000001 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455815+0000", 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112071 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460827+0000", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050111419999999997 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468359+0000", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075321540000000001 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815826+0000", 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34746724800000001 2023-09-23T15:25:40.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815875+0000", 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8959999999999999e-05 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815914+0000", 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9490999999999998e-05 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165867 7.8 7:1ac8c744:::benchmark_data_smithi017_422824_object71997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 319.30852393999999, 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35163571799999999, 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.862 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165867 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473542+0000", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1200000000000002e-06 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473549+0000", 2023-09-23T15:25:40.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2429999999999999e-06 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473556+0000", 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1999999999999997e-06 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473610+0000", 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3355e-05 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473737+0000", 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126804 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.474340+0000", 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060306200000000002 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482320+0000", 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079798620000000008 2023-09-23T15:25:40.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482361+0000", 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1823000000000003e-05 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825087+0000", 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34272576999999999 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825135+0000", 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7395000000000003e-05 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825175+0000", 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0583000000000001e-05 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.868 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165870 7.8 7:198dc808:::benchmark_data_smithi017_422824_object72000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 319.30564103900002, 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35318146500000003, 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165870 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476421+0000", 2023-09-23T15:25:40.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476428+0000", 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3429999999999998e-06 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476446+0000", 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7997000000000001e-05 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476455+0000", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478546+0000", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002091192 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478670+0000", 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012399199999999999 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478787+0000", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116907 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482437+0000", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036504319999999999 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.483321+0000", 2023-09-23T15:25:40.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088376899999999996 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829506+0000", 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34618508100000001 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829557+0000", 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0427999999999999e-05 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829604+0000", 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7024000000000002e-05 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165871 7.5 7:a7f75625:::benchmark_data_smithi017_422824_object72001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 319.30553426799997, 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27785160399999997, 2023-09-23T15:25:40.875 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165871 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476528+0000", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476533+0000", 2023-09-23T15:25:40.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.318e-06 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476542+0000", 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7059999999999999e-06 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476547+0000", 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6260000000000003e-06 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477473+0000", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00092625100000000001 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477603+0000", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129806 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477726+0000", 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012271 2023-09-23T15:25:40.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.481919+0000", 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041935080000000003 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482075+0000", 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015598599999999999 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754297+0000", 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27222164999999998 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754340+0000", 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2945000000000003e-05 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754381+0000", 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0768e-05 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.881 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179356 7.e 7:7f6e6587:::benchmark_data_smithi017_422824_object85486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.09331183900002, 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33625833300000002, 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.882 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179356 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688750+0000", 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688756+0000", 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2060000000000004e-06 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2061e-05 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688776+0000", 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0299999999999994e-06 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688855+0000", 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8975999999999997e-05 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688972+0000", 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116957 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689084+0000", 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111329 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692868+0000", 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003784379 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695532+0000", 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026644920000000001 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024919+0000", 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32938610699999998 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024969+0000", 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0522000000000002e-05 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025010+0000", 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0778000000000002e-05 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.888 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179365 7.e 7:78918128:::benchmark_data_smithi017_422824_object85495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 299.08109805499998, 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32416663400000001, 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179365 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700964+0000", 2023-09-23T15:25:40.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700970+0000", 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6659999999999996e-06 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700981+0000", 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1184e-05 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700987+0000", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1789999999999996e-06 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701026+0000", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8707000000000001e-05 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701105+0000", 2023-09-23T15:25:40.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9220000000000004e-05 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701175+0000", 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9696000000000005e-05 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704027+0000", 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028522439999999999 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704091+0000", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3796999999999999e-05 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025048+0000", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32095755500000001 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025094+0000", 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5488999999999998e-05 2023-09-23T15:25:40.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025132+0000", 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8133999999999998e-05 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:115285 8.a e318/284 8:5a50f317:::benchmark_data_smithi017_485823_object25148:head v 318'7127, mlcod=318'7124)", 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:32.226513+0000", 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 128.55555048400001, 2023-09-23T15:25:40.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.052668952999999998, 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226513+0000", 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226513+0000", 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226513+0000", 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226522+0000", 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2410000000000001e-06 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226529+0000", 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6680000000000001e-06 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.226534+0000", 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9130000000000002e-06 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.230310+0000", 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037751690000000001 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.230334+0000", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4338e-05 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.279133+0000", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.048799212000000002 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:32.279182+0000", 2023-09-23T15:25:40.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8761999999999998e-05 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:168781 8.f e330/284 8:f49849bd:::benchmark_data_smithi017_485823_object78644:head v 330'10806, mlcod=330'10804)", 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:58.132538+0000", 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.649525807000003, 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063126702000000007, 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132538+0000", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132538+0000", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132537+0000", 2023-09-23T15:25:40.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132555+0000", 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8017e-05 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132569+0000", 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4715000000000001e-05 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132578+0000", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7050000000000005e-06 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132617+0000", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9124e-05 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.132637+0000", 2023-09-23T15:25:40.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0106999999999999e-05 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.136232+0000", 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035948180000000001 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.195664+0000", 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.059432171999999998 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.904 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:168782 8.d e330/287 8:b92c966b:::benchmark_data_smithi017_485823_object78645:head v 330'10397, mlcod=330'10395)", 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.648956937999998, 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078966947999999995, 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:25:40.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133112+0000", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7420000000000003e-06 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133119+0000", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7970000000000001e-06 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133124+0000", 2023-09-23T15:25:40.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6e-06 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133166+0000", 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1900000000000002e-05 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133188+0000", 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2391999999999998e-05 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212024+0000", 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078835815000000004 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212073+0000", 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9292000000000002e-05 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.909 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:168785 8.d e330/287 8:b36bb946:::benchmark_data_smithi017_485823_object78648:head v 330'10398, mlcod=330'10395)", 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:58.152801+0000", 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 42.629262621000002, 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.059301294999999997, 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.152801+0000", 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.152801+0000", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.152800+0000", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.152809+0000", 2023-09-23T15:25:40.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1889999999999996e-06 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.195691+0000", 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.042882358000000002 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.195703+0000", 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.132e-05 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.196030+0000", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000327607 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.196075+0000", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4860000000000001e-05 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212080+0000", 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016004457999999999 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:25:40.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212102+0000", 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2459999999999998e-05 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:25:40.914 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: pgmap v1008: 109 pgs: 8 creating+peering, 6 unknown, 95 active+clean; 4.6 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: failed to encode map e342 with expected crc 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:25:40.915 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:40 smithi027 ceph-mon[195233]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T15:25:41.107 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:41.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:41.211 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:41.240 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:41.523 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 0 (802) 2023-09-23T15:25:41.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:41 smithi017 ceph-mon[215646]: failed to encode map e342 with expected crc 2023-09-23T15:25:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:25:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:41 smithi017 ceph-mon[222878]: failed to encode map e342 with expected crc 2023-09-23T15:25:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:25:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:41 smithi027 ceph-mon[195233]: failed to encode map e342 with expected crc 2023-09-23T15:25:41.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:25:41.907 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:41.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:41.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:25:42.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:42.039 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:42.186 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:42.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:42.190 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:42.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:42.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:42.324 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:42.376 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:42.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:42.381 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:42.397 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:25:42.430 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:42.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:42.530 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:42.534 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T15:25:42.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: pgmap v1010: 109 pgs: 109 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: 1.6 scrub ok 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/257823996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: pgmap v1010: 109 pgs: 109 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: 1.6 scrub ok 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.729 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/257823996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: pgmap v1010: 109 pgs: 109 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: 1.6 scrub ok 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.730 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1660784756' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: failed to encode map e343 with expected crc 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/257823996' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:25:42.862 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:42.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:42.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:42.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:42.994 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:43.692 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:43.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:43.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:43.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:43.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:43.824 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:43.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: failed to encode map e344 with expected crc 2023-09-23T15:25:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:44.161 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:44.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:44.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:44.302 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:44.354 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:44.361 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:44.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:44.492 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:44 smithi027 ceph-mon[195233]: pgmap v1013: 109 pgs: 4 peering, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:44 smithi027 ceph-mon[195233]: 1.e scrub starts 2023-09-23T15:25:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:44 smithi027 ceph-mon[195233]: 1.e scrub ok 2023-09-23T15:25:44.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:44 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:25:44.753 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[215646]: pgmap v1013: 109 pgs: 4 peering, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[215646]: 1.e scrub starts 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[215646]: 1.e scrub ok 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[222878]: pgmap v1013: 109 pgs: 4 peering, 105 active+clean; 4.6 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:25:44.755 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[222878]: 1.e scrub starts 2023-09-23T15:25:44.756 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[222878]: 1.e scrub ok 2023-09-23T15:25:44.756 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:44 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:25:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:44.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:44.787 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T15:25:44.799 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:25:44.799 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:25:44.806 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:44.857 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:44.888 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:44.972 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:25:45.040 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:25:45.040 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_543597 2023-09-23T15:25:45.041 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:25:45.041 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2023-09-23T15:25:45.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:45.042 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:45.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:45.143 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:45.212 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:45.292 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:25:45.323 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:25:45.345 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:25:45.380 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:25:45.415 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:25:45.450 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:25:45.462 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:45.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:45.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:45.491 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:25:45.533 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:25:45.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.537 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.538 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:25:45.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:45.574 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3047805016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:45.575 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3047805016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:45.621 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:45.735 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:25:45.735 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:25:45.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3047805016' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:45.974 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 2806 2790 174.14 174.375 0.00719107 0.00572434 2023-09-23T15:25:46.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:46 smithi027 ceph-mon[195233]: pgmap v1014: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T15:25:46.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:46 smithi017 ceph-mon[222878]: pgmap v1014: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T15:25:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:46 smithi017 ceph-mon[215646]: pgmap v1014: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 102 KiB/s, 0 objects/s recovering 2023-09-23T15:25:46.974 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 5826 5810 181.429 188.75 0.00599171 0.00550616 2023-09-23T15:25:47.072 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:47.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:47.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:47.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:47.230 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:47.245 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:25:47.245 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:25:47.246 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T15:25:47.246 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:25:47.246 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:25:47.246 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T15:25:47.325 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:47.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:47.334 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:47.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:47.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:47.549 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:47.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:47.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:47.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:47.686 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:47.974 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 8144 8128 169.244 144.875 0.00134385 0.0057769 2023-09-23T15:25:48.413 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:48] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T15:25:48.414 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:48 smithi017 ceph-mon[215646]: pgmap v1015: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T15:25:48.414 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:48 smithi017 ceph-mon[222878]: pgmap v1015: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T15:25:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:48 smithi027 ceph-mon[195233]: pgmap v1015: 109 pgs: 4 peering, 105 active+clean; 4.3 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 74 KiB/s, 0 objects/s recovering 2023-09-23T15:25:48.626 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:48.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:48.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:48.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:25:48.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:48.804 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:48.960 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:48.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:48.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:48.974 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 10427 10411 162.603 142.688 0.00795905 0.00613669 2023-09-23T15:25:49.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:49.108 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:49.142 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:49.250 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:49.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:49.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:49.356 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:49.401 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:49.487 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:49.491 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:49 smithi027 ceph-mon[195233]: 1.c scrub starts 2023-09-23T15:25:49.491 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:49 smithi027 ceph-mon[195233]: 1.c scrub ok 2023-09-23T15:25:49.491 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:49.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:49.492 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:49.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[215646]: 1.c scrub starts 2023-09-23T15:25:49.493 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[215646]: 1.c scrub ok 2023-09-23T15:25:49.494 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:49.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[222878]: 1.c scrub starts 2023-09-23T15:25:49.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[222878]: 1.c scrub ok 2023-09-23T15:25:49.494 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:49.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:49.636 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:49.726 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:49.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:49.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:49.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:49.837 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:49.904 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:49.977 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 12832 12816 160.141 150.312 0.00543642 0.00623861 2023-09-23T15:25:50.249 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:25:50.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:50 smithi027 ceph-mon[195233]: pgmap v1016: 109 pgs: 109 active+clean; 78 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s; 67 KiB/s, 0 objects/s recovering 2023-09-23T15:25:50.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:50.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:50 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:25:50.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[215646]: pgmap v1016: 109 pgs: 109 active+clean; 78 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s; 67 KiB/s, 0 objects/s recovering 2023-09-23T15:25:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:50.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:25:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[222878]: pgmap v1016: 109 pgs: 109 active+clean; 78 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 9.2 MiB/s wr, 146 op/s; 67 KiB/s, 0 objects/s recovering 2023-09-23T15:25:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:25:50.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:50 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:25:50.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:25:50.974 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 15643 15627 162.729 175.688 0.00453842 0.00613931 2023-09-23T15:25:51.396 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:51.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:51.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:51.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:51.511 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[215646]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:51.512 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:51 smithi017 ceph-mon[222878]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T15:25:51.561 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:51.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:25:51.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:25:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: failed to encode map e345 with expected crc 2023-09-23T15:25:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1878512971' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:25:51.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:51 smithi027 ceph-mon[195233]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T15:25:51.975 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 17944 17928 160.002 143.812 0.00788237 0.00624485 2023-09-23T15:25:51.978 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:51.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:52.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:52.109 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:52.233 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:52 smithi017 ceph-mon[215646]: pgmap v1018: 109 pgs: 109 active+clean; 678 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:52.534 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:52.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:52.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:52.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:52 smithi017 ceph-mon[222878]: pgmap v1018: 109 pgs: 109 active+clean; 678 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:52.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:52 smithi027 ceph-mon[195233]: pgmap v1018: 109 pgs: 109 active+clean; 678 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:52.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:25:52.680 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:52.749 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:25:52.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:52.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:25:52.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:52.927 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:52.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 19533 19517 152.417 99.3125 0.00511224 0.00655283 2023-09-23T15:25:53.342 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:53.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:53.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:53.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:25:53.474 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:53.518 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:25:53.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:53.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:25:53.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:53.650 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:53.693 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:53.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:53.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:53.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:25:53.862 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:53.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 21299 21283 147.746 110.375 0.00615917 0.00676307 2023-09-23T15:25:54.058 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:54.136 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:25:54.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:54.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:25:54.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:54.303 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:54.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:54 smithi017 ceph-mon[222878]: pgmap v1019: 109 pgs: 109 active+clean; 818 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:54.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:54 smithi017 ceph-mon[215646]: pgmap v1019: 109 pgs: 109 active+clean; 818 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:54.489 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:25:54.490 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2023-09-23T15:25:54.490 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:25:54.491 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:54.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:54.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:54.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:54 smithi027 ceph-mon[195233]: pgmap v1019: 109 pgs: 109 active+clean; 818 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2023-09-23T15:25:54.601 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:25:54.651 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:54.735 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:25:54.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:25:54.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:54.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:25:54.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:25:54.885 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:54.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 23312 23296 145.552 125.812 0.0072669 0.00686422 2023-09-23T15:25:54.980 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:25:54.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:54.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:25:55.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:55.147 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:55.221 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:25:55.222 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:25:55.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 25519 25503 144.853 137.938 0.0103584 0.0068976 2023-09-23T15:25:56.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:56 smithi027 ceph-mon[195233]: pgmap v1020: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-09-23T15:25:56.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:56 smithi017 ceph-mon[215646]: pgmap v1020: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-09-23T15:25:56.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:56 smithi017 ceph-mon[222878]: pgmap v1020: 109 pgs: 109 active+clean; 1.1 GiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s 2023-09-23T15:25:56.763 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:56.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:56.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:56.872 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:56.931 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:56.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 26950 26934 140.235 89.4375 0.00365443 0.00712477 2023-09-23T15:25:57.960 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:57.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:57.976 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 28445 28429 136.636 93.4375 0.0106443 0.00731322 2023-09-23T15:25:58.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:25:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:25:58] "GET /metrics HTTP/1.1" 200 36308 "" "Prometheus/2.43.0" 2023-09-23T15:25:58.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:25:58.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:25:58.148 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:58.280 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:25:58.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:58.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:25:58.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[215646]: pgmap v1021: 109 pgs: 109 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T15:25:58.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:58.290 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:58.291 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[222878]: pgmap v1021: 109 pgs: 109 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T15:25:58.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:58.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:58.336 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:25:58.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:25:58.506 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:58.544 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:25:58.584 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:25:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:58 smithi027 ceph-mon[195233]: pgmap v1021: 109 pgs: 109 active+clean; 1.3 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T15:25:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:58.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:25:58.625 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:25:58.670 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:25:58.705 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:25:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:58.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:25:58.710 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:25:58.747 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:25:58.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:25:58.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:25:58.821 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.825 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:25:58.865 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:25:58.977 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 30030 30014 133.952 99.0625 0.010291 0.00745873 2023-09-23T15:25:59.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4193984909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:59.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:25:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4193984909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:59.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:25:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:59.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4193984909' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:25:59.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:25:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:25:59.874 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:25:59.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:25:59.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:25:59.977 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 31931 31915 132.937 118.812 0.00502087 0.00751698 2023-09-23T15:25:59.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:00.046 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:00 smithi027 ceph-mon[195233]: pgmap v1022: 109 pgs: 109 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:26:00.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:00 smithi017 ceph-mon[215646]: pgmap v1022: 109 pgs: 109 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:26:00.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:00 smithi017 ceph-mon[222878]: pgmap v1022: 109 pgs: 109 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s 2023-09-23T15:26:00.742 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:00.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:00.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:00.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:00.885 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:00.901 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:00.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 33410 33394 130.406 92.4375 0.00608529 0.00766278 2023-09-23T15:26:00.987 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:00.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:00.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:01.094 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:01.139 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:01.227 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:01.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:01.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:01.334 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:01.417 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:01.762 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:01.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:01.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:01.922 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:01.977 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 35134 35118 129.073 107.75 0.0071613 0.00774288 2023-09-23T15:26:02.030 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:02.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:02.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:02.135 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:02.190 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:02.282 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:02.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:02.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:02.310 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:26:02.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[215646]: pgmap v1023: 109 pgs: 109 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T15:26:02.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2573652321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:02.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:02.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[222878]: pgmap v1023: 109 pgs: 109 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T15:26:02.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2573652321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:02.356 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:02 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:02.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:02.443 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:02.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:02 smithi027 ceph-mon[195233]: pgmap v1023: 109 pgs: 109 active+clean; 1.9 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2023-09-23T15:26:03.521 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2573652321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:03.521 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:02 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:03.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:26:03.522 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:26:03.522 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:26:03.522 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2023-09-23T15:26:03.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T15:26:03.523 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 37001 36985 128.385 116.688 0.00754412 0.00778409 2023-09-23T15:26:03.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:03.529 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:03.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:03.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:03.675 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.675 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:03.675 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T15:26:03.675 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.675 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:03 smithi027 ceph-mon[195233]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:03.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:03.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:03.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T15:26:03.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.704 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[222878]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.705 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:03 smithi017 ceph-mon[215646]: failed to encode map e346 with expected crc 2023-09-23T15:26:03.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:03.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 38608 38592 126.913 100.438 0.0288242 0.00786593 2023-09-23T15:26:04.225 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:04.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:04.234 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:04.334 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:04.371 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:04.394 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:04 smithi017 ceph-mon[215646]: pgmap v1025: 109 pgs: 109 active+clean; 2.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T15:26:04.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:04 smithi017 ceph-mon[222878]: pgmap v1025: 109 pgs: 109 active+clean; 2.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T15:26:04.446 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:04.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:04.507 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:04.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:04.563 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:04 smithi027 ceph-mon[195233]: pgmap v1025: 109 pgs: 109 active+clean; 2.0 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 119 MiB/s wr, 1.90k op/s 2023-09-23T15:26:04.649 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:04.786 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:04.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:04.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:04.946 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:04.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:26:04.978564+0000 min lat: 0.00111339 max lat: 0.312352 avg lat: 0.00798236 2023-09-23T15:26:04.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:26:04.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 40086 40070 125.186 92.375 0.00879385 0.00798236 2023-09-23T15:26:05.024 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:05.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:05.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:05.129 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165850 7.8 e271/261 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head v 271'10159, mlcod=271'10156)", 2023-09-23T15:26:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.68497977999999, 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35471572400000001, 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460487+0000", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460486+0000", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460491+0000", 2023-09-23T15:26:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4039999999999997e-06 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460497+0000", 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460502+0000", 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4759999999999997e-06 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460547+0000", 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4233999999999997e-05 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460568+0000", 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1189999999999999e-05 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468865+0000", 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082969110000000006 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815202+0000", 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34633774499999997 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165851 7.1 7:8fdd4296:::benchmark_data_smithi017_422824_object71981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.68413524200002, 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36053184399999999, 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165851 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461331+0000", 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461330+0000", 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461336+0000", 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5899999999999996e-06 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461348+0000", 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2065000000000001e-05 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461357+0000", 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8189999999999998e-06 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9935999999999997e-05 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461554+0000", 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013704900000000001 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121172 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815161+0000", 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35348590600000002 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815321+0000", 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000160006 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821782+0000", 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064606120000000001 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821827+0000", 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5030000000000001e-05 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821863+0000", 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6208999999999998e-05 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165853 7.3 7:c0488a25:::benchmark_data_smithi017_422824_object71983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:26:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.68409433900001, 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35803834699999998, 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165853 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461372+0000", 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461375+0000", 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2899999999999998e-06 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461393+0000", 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8119000000000001e-05 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461404+0000", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0916e-05 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464554+0000", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003149799 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464660+0000", 2023-09-23T15:26:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106097 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464785+0000", 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012551000000000001 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818430+0000", 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35364495699999998 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.818460+0000", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9578999999999999e-05 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819308+0000", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00084795300000000003 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819358+0000", 2023-09-23T15:26:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9982999999999999e-05 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819411+0000", 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2698000000000003e-05 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165854 7.a 7:57b0f785:::benchmark_data_smithi017_422824_object71984:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:26:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.684048448, 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35469288599999999, 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165854 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461418+0000", 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461417+0000", 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461421+0000", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1570000000000002e-06 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461429+0000", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4950000000000002e-06 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461435+0000", 2023-09-23T15:26:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6699999999999999e-06 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461497+0000", 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1896999999999994e-05 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461605+0000", 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000108937 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,5", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461675+0000", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9722999999999996e-05 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815166+0000", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35349039599999998 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:26:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4358999999999996e-05 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816030+0000", 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00077006299999999998 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816073+0000", 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3195999999999998e-05 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816111+0000", 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7719999999999998e-05 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165856 7.2 7:45443312:::benchmark_data_smithi017_422824_object71986:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.68402065700002, 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36007705000000001, 2023-09-23T15:26:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165856 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461446+0000", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461445+0000", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461448+0000", 2023-09-23T15:26:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1530000000000001e-06 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461456+0000", 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1609999999999997e-06 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461461+0000", 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4960000000000004e-06 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464405+0000", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029436470000000002 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464508+0000", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010341899999999999 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.464624+0000", 2023-09-23T15:26:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000115374 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815184+0000", 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35056013400000002 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815339+0000", 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015575800000000001 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821434+0000", 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060949569999999998 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821484+0000", 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9515999999999998e-05 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.821523+0000", 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8986999999999998e-05 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165858 7.3 7:c518666c:::benchmark_data_smithi017_422824_object71988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.68400055799998, 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.36584072299999998, 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165858 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461466+0000", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461468+0000", 2023-09-23T15:26:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4530000000000001e-06 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461474+0000", 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7010000000000001e-06 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.461486+0000", 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.201e-05 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467017+0000", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055312130000000001 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34830823100000002 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815477+0000", 2023-09-23T15:26:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015180900000000001 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.819449+0000", 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039723479999999997 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.823533+0000", 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040831690000000002 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827223+0000", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036904910000000002 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827268+0000", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5204000000000003e-05 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.827307+0000", 2023-09-23T15:26:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8433999999999998e-05 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165860 7.6 e271/223 7:667b3ce6:::benchmark_data_smithi017_422824_object71990:head v 271'10449, mlcod=271'10447)", 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.67897035300001, 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34876393300000003, 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466496+0000", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466495+0000", 2023-09-23T15:26:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466503+0000", 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7440000000000007e-06 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466515+0000", 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2150999999999999e-05 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466523+0000", 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2859999999999999e-06 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466566+0000", 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2422000000000001e-05 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.466587+0000", 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1475e-05 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815220+0000", 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34863227000000002 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815260+0000", 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0509000000000001e-05 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.67809680200003, 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34789368300000001, 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467370+0000", 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467369+0000", 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.467375+0000", 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6760000000000004e-06 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815177+0000", 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.347802218 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2822e-05 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815244+0000", 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4230000000000001e-05 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815253+0000", 2023-09-23T15:26:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1579999999999994e-06 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815263+0000", 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0268999999999999e-05 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165854 7.a e271/223 ondisk, result = 0)", 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:26:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.67639835599999, 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34704774999999999, 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469068+0000", 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469067+0000", 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.469074+0000", 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2230000000000004e-06 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815176+0000", 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.346102403 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815190+0000", 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3959e-05 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815563+0000", 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00037292800000000002 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816024+0000", 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00046142900000000002 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.816116+0000", 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1516999999999998e-05 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.45479.0:165851 7.1 e271/263 ondisk, result = 0)", 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 343.67389271899998, 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34375118199999999, 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471574+0000", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471573+0000", 2023-09-23T15:26:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.471579+0000", 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1940000000000003e-06 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815160+0000", 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.343580518 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815175+0000", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5639000000000001e-05 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815306+0000", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130894 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815315+0000", 2023-09-23T15:26:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5769999999999994e-06 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815325+0000", 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0227999999999999e-05 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179348 7.1 7:83a21fe1:::benchmark_data_smithi017_422824_object85478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:26:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.46364800100002, 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32463872399999999, 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179348 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681819+0000", 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681817+0000", 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681825+0000", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0039999999999999e-06 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681838+0000", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3648e-05 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.681846+0000", 2023-09-23T15:26:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0199999999999994e-06 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685481+0000", 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00363489 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685642+0000", 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016116599999999999 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.685776+0000", 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000133442 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688382+0000", 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026061270000000002 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688535+0000", 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015359299999999999 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006352+0000", 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31781674399999998 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006412+0000", 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0061e-05 2023-09-23T15:26:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006457+0000", 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5016999999999999e-05 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179350 7.1 7:87665378:::benchmark_data_smithi017_422824_object85480:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.456744222, 2023-09-23T15:26:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31832527799999999, 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179350 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688722+0000", 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688721+0000", 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688727+0000", 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.806e-06 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688738+0000", 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1192000000000001e-05 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688745+0000", 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2049999999999996e-06 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688796+0000", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0779000000000003e-05 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688930+0000", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013444299999999999 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689051+0000", 2023-09-23T15:26:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120451 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.691835+0000", 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027841459999999999 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694698+0000", 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028631170000000001 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006962+0000", 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31226363000000001 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007010+0000", 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8396999999999998e-05 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.007048+0000", 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7428999999999997e-05 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179357 7.2 7:4242eeb1:::benchmark_data_smithi017_422824_object85487:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.45669774599997, 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.317811762, 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179357 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688769+0000", 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688772+0000", 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.569e-06 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688782+0000", 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8120000000000002e-06 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688788+0000", 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6980000000000001e-06 2023-09-23T15:26:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688840+0000", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1610000000000002e-05 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688932+0000", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2485999999999995e-05 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,2", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689413+0000", 2023-09-23T15:26:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048049699999999998 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006235+0000", 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31682244500000001 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006406+0000", 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017080200000000001 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006494+0000", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8374000000000005e-05 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006540+0000", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5130000000000003e-05 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006581+0000", 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0924999999999999e-05 2023-09-23T15:26:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179355 7.6 e276/223 7:615b3fe2:::benchmark_data_smithi017_422824_object85485:head v 276'11306, mlcod=276'11301)", 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.45638617200001, 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31722602799999999, 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689080+0000", 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689087+0000", 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5159999999999997e-06 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689096+0000", 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689103+0000", 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6159999999999998e-06 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692603+0000", 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0035004910000000001 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692623+0000", 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0027999999999999e-05 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006273+0000", 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31364994099999999 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006306+0000", 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3408000000000003e-05 2023-09-23T15:26:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:179351 7.c e276/223 7:387cb04d:::benchmark_data_smithi017_422824_object85481:head v 276'11330, mlcod=276'11328)", 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 323.45610641600001, 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31712001899999998, 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689360+0000", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689359+0000", 2023-09-23T15:26:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689369+0000", 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2050000000000008e-06 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689378+0000", 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3030000000000005e-06 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689397+0000", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8868999999999998e-05 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689491+0000", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3949999999999993e-05 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689511+0000", 2023-09-23T15:26:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0625999999999999e-05 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.694710+0000", 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051988340000000003 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.006480+0000", 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31176990500000001 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50576 8.b 8:d9711445:::benchmark_data_smithi017_485823_object50575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.95321076499999, 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1259012579999998, 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50576 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192254+0000", 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192281+0000", 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7236000000000001e-05 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192304+0000", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3314000000000001e-05 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192314+0000", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6269999999999995e-06 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192529+0000", 2023-09-23T15:26:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000214637 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192538+0000", 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119735+0000", 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92719635 2023-09-23T15:26:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119749+0000", 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4219999999999999e-05 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307742+0000", 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.187992758 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307887+0000", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014497199999999999 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.308028+0000", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014144799999999999 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314256+0000", 2023-09-23T15:26:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062275179999999996 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314333+0000", 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6796000000000001e-05 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315870+0000", 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001537524 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315924+0000", 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3733999999999997e-05 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.318157+0000", 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022332469999999998 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50579 8.b 8:dd05f5d9:::benchmark_data_smithi017_485823_object50578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.95309697600001, 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.129065862, 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50579 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192369+0000", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192402+0000", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2693e-05 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192410+0000", 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:26:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192415+0000", 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8710000000000002e-06 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192564+0000", 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014868300000000001 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:26:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192571+0000", 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1160000000000001e-06 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119777+0000", 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92720541499999998 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119787+0000", 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9520000000000006e-06 2023-09-23T15:26:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312151+0000", 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1923646489999999 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312321+0000", 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017003 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312518+0000", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196746 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314638+0000", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021200590000000001 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315028+0000", 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038974000000000002 2023-09-23T15:26:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319363+0000", 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043355499999999996 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319441+0000", 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7822000000000001e-05 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.321435+0000", 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019941450000000001 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:86749 8.b 8:dabf8292:::benchmark_data_smithi017_485823_object86748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 200.01101342600001, 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.252098983, 2023-09-23T15:26:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 86749 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134453+0000", 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134474+0000", 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134481+0000", 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5270000000000001e-06 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134486+0000", 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0710000000000002e-06 2023-09-23T15:26:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137557+0000", 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030706990000000001 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137742+0000", 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018558499999999999 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137914+0000", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017187300000000001 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381133+0000", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.243218563 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.381203+0000", 2023-09-23T15:26:05.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9811e-05 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.384070+0000", 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028675089999999999 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.384134+0000", 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3756000000000004e-05 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.263 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.386552+0000", 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0024180650000000001 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:123160 8.0 8:0e759aed:::benchmark_data_smithi017_485823_object33023:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:45.836772+0000", 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 139.308694632, 2023-09-23T15:26:05.264 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26551778999999998, 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123160 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836772+0000", 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.265 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836772+0000", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836770+0000", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836777+0000", 2023-09-23T15:26:05.266 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4679999999999997e-06 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836788+0000", 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0798000000000001e-05 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836796+0000", 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6899999999999998e-06 2023-09-23T15:26:05.267 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836835+0000", 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8931999999999999e-05 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836921+0000", 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5378999999999997e-05 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:26:05.268 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.836991+0000", 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9970999999999998e-05 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.840281+0000", 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003290304 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.844615+0000", 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043340419999999998 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.269 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102195+0000", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25758028500000002 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102246+0000", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0952999999999998e-05 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102290+0000", 2023-09-23T15:26:05.270 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3438e-05 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:123173 8.0 8:0753281b:::benchmark_data_smithi017_485823_object33036:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:23:45.848841+0000", 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 139.29662544199999, 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25362259700000001, 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:05.271 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 123173 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848841+0000", 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:05.272 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848841+0000", 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848839+0000", 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848848+0000", 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2970000000000004e-06 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848865+0000", 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7393000000000001e-05 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848875+0000", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4220000000000006e-06 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.848928+0000", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3081000000000002e-05 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.849083+0000", 2023-09-23T15:26:05.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000155675 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.849214+0000", 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130612 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.852451+0000", 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032373329999999998 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:45.852657+0000", 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000205329 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102363+0000", 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24970629999999999 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:26:05.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102419+0000", 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5649000000000002e-05 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:23:46.102464+0000", 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5167000000000003e-05 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:05.278 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:26:05.279 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:05.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 41617 41601 123.781 95.6875 0.00739152 0.00807289 2023-09-23T15:26:06.233 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:26:06.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:06 smithi027 ceph-mon[195233]: pgmap v1026: 109 pgs: 109 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T15:26:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:06 smithi017 ceph-mon[215646]: pgmap v1026: 109 pgs: 109 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T15:26:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:06 smithi017 ceph-mon[222878]: pgmap v1026: 109 pgs: 109 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T15:26:06.978 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 42717 42701 121.28 68.75 0.00429083 0.00821319 2023-09-23T15:26:07.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:26:07.857 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:07.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:07.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:07.909 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:07.963 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:07.982 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 43791 43775 118.916 67.125 0.015109 0.0084048 2023-09-23T15:26:08.036 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:08.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:08] "GET /metrics HTTP/1.1" 200 36357 "" "Prometheus/2.43.0" 2023-09-23T15:26:08.314 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:08 smithi027 ceph-mon[195233]: pgmap v1027: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T15:26:08.520 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:08 smithi017 ceph-mon[222878]: pgmap v1027: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T15:26:08.520 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:08 smithi017 ceph-mon[215646]: pgmap v1027: 109 pgs: 109 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2023-09-23T15:26:08.609 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:08.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:08.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:08.714 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:08.795 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:08.807 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:08.941 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:08.980 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 45109 45093 117.391 82.375 0.0149175 0.00851454 2023-09-23T15:26:08.983 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:09.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:09.041 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:09.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:09.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:09.082 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:09.119 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:09.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:09.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:09.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:09.250 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:09.266 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:09.267 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.268 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.269 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:26:09.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4023106927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:09.579 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:09 smithi017 conmon[215622]: 2023-09-23T15:26:09.530+0000 7f618a1a9700 -1 mon.a@0(leader).osd e346 definitely_dead 0 2023-09-23T15:26:09.579 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4023106927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:09.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4023106927' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:09.726 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:09.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:09.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:09.830 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:09.886 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:09.980 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 46146 46130 115.288 64.8125 0.00292947 0.00864247 2023-09-23T15:26:10.333 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:10 smithi027 ceph-mon[195233]: pgmap v1028: 109 pgs: 109 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:26:10.334 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:10 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2030278628' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.334 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.334 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:10 smithi027 ceph-mon[195233]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:26:10.372 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.4. 2023-09-23T15:26:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[215646]: pgmap v1028: 109 pgs: 109 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:26:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2030278628' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[215646]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:26:10.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[222878]: pgmap v1028: 109 pgs: 109 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:26:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2030278628' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T15:26:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:10 smithi017 ceph-mon[222878]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T15:26:10.981 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 46604 46588 111.955 28.625 0.00191834 0.0087237 2023-09-23T15:26:11.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[215646]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.350 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[215646]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:26:11.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[215646]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T15:26:11.351 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[215646]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.558 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:11.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:11.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:11 smithi027 ceph-mon[195233]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:11 smithi027 ceph-mon[195233]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:26:11.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:11 smithi027 ceph-mon[195233]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T15:26:11.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:11 smithi027 ceph-mon[195233]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[222878]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[222878]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T15:26:11.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[222878]: osdmap e347: 8 total, 8 up, 7 in 2023-09-23T15:26:11.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:11 smithi017 ceph-mon[222878]: failed to encode map e347 with expected crc 2023-09-23T15:26:11.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:11.690 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:11.735 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:11.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:11.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:11.868 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:11.967 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:11.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:11.970 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:11.981 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 46604 46588 107.809 0 - 0.0087237 2023-09-23T15:26:12.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:12.105 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:12.157 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:12.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:12.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:12.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:12.292 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:12.342 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:12.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:12.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: pgmap v1030: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:12 smithi027 ceph-mon[195233]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.446 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:12.476 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:12.569 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:12.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:12.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:12.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:12.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: pgmap v1030: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[222878]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: pgmap v1030: 109 pgs: 109 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 97 MiB/s wr, 1.55k op/s 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: osdmap e348: 8 total, 8 up, 7 in 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:12 smithi017 ceph-mon[215646]: failed to encode map e348 with expected crc 2023-09-23T15:26:12.770 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:12.871 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:12.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:12.976 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:12.981 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 47246 47230 105.392 20.0625 0.00900649 0.00948347 2023-09-23T15:26:13.040 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:13.121 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:26:13.164 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:13.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:13.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:13.319 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:13.407 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:26:13.428 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:13.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:13.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:13.534 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2062468882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:13.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2062468882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: osdmap e349: 8 total, 8 up, 7 in 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2062468882' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: failed to encode map e349 with expected crc 2023-09-23T15:26:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:13.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:13.981 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 48448 48432 104.349 75.125 0.0113328 0.00957781 2023-09-23T15:26:14.102 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:14.362 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:14.367 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:14.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:14.468 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:14.471 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: pgmap v1033: 109 pgs: 4 remapped+peering, 105 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.472 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:14 smithi027 ceph-mon[195233]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.544 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:14.633 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:14.680 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:14.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:14.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: pgmap v1033: 109 pgs: 4 remapped+peering, 105 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[222878]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: pgmap v1033: 109 pgs: 4 remapped+peering, 105 active+clean; 2.8 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: osdmap e350: 8 total, 8 up, 7 in 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.700 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:14 smithi017 ceph-mon[215646]: failed to encode map e350 with expected crc 2023-09-23T15:26:14.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:14.984 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 49617 49601 103.302 73.0625 0.0139544 0.00967407 2023-09-23T15:26:15.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T15:26:15.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.740 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[215646]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.741 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:15 smithi017 ceph-mon[222878]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: osdmap e351: 8 total, 8 up, 7 in 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:15 smithi027 ceph-mon[195233]: failed to encode map e351 with expected crc 2023-09-23T15:26:15.903 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:15.985 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 50518 50502 101.777 56.3125 0.0118802 0.00982043 2023-09-23T15:26:16.296 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:16.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:16.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:16.474 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:16.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:26:16.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:16 smithi017 ceph-mon[215646]: pgmap v1036: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 638 op/s; 2927/143454 objects degraded (2.040%); 2717/143454 objects misplaced (1.894%); 1.5 MiB/s, 2 keys/s, 29 objects/s recovering 2023-09-23T15:26:16.551 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:16 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 2927/143454 objects degraded (2.040%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:26:16.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:16 smithi017 ceph-mon[222878]: pgmap v1036: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 638 op/s; 2927/143454 objects degraded (2.040%); 2717/143454 objects misplaced (1.894%); 1.5 MiB/s, 2 keys/s, 29 objects/s recovering 2023-09-23T15:26:16.551 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:16 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 2927/143454 objects degraded (2.040%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:26:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:16 smithi027 ceph-mon[195233]: pgmap v1036: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 40 MiB/s wr, 638 op/s; 2927/143454 objects degraded (2.040%); 2717/143454 objects misplaced (1.894%); 1.5 MiB/s, 2 keys/s, 29 objects/s recovering 2023-09-23T15:26:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:16 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 2927/143454 objects degraded (2.040%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:26:16.628 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:26:16.628 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:26:16.628 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T15:26:16.628 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:26:16.701 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:16.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:16.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:16.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:16.870 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:16.955 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:16.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:16.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 51609 51593 100.706 68.1875 0.0182692 0.00992374 2023-09-23T15:26:17.033 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:26:17.033 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:26:17.064 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:17.087 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:17.131 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:17.154 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:17.176 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:17.228 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:17.265 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:17.269 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:17.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:17.272 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:17.274 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:17.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:17.424 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:17.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3264324201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:17.445 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:17.476 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:17.519 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:17.540 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.541 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:26:17.780 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3264324201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:17.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3264324201' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:17.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 52475 52459 99.2945 54.125 0.00257809 0.0100462 2023-09-23T15:26:18.024 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:18] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T15:26:18.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:18.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:18.446 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:18.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:18.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:26:18.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:18.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:18 smithi027 ceph-mon[195233]: pgmap v1037: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 2927/146061 objects degraded (2.004%); 2717/146061 objects misplaced (1.860%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-23T15:26:18.599 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:18.687 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:18.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:18.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:18.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:18.841 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:18.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:18 smithi017 ceph-mon[215646]: pgmap v1037: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 2927/146061 objects degraded (2.004%); 2717/146061 objects misplaced (1.860%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-23T15:26:18.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:18 smithi017 ceph-mon[222878]: pgmap v1037: 109 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 peering, 4 remapped+peering, 101 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 39 MiB/s wr, 628 op/s; 2927/146061 objects degraded (2.004%); 2717/146061 objects misplaced (1.860%); 1.1 MiB/s, 1 keys/s, 22 objects/s recovering 2023-09-23T15:26:18.954 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:18.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:18.959 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:18.992 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 53612 53596 98.4644 71.0625 0.0148577 0.010151 2023-09-23T15:26:19.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:19.097 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:19.162 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:19.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:19.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:19.217 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:19.272 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:19.323 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:19.405 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:19.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:19.510 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:19.580 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:19.993 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 54434 54418 97.1194 51.375 0.00983137 0.0102585 2023-09-23T15:26:20.515 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:20.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:20.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:20.618 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:20.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:20 smithi017 ceph-mon[215646]: pgmap v1038: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 5849/149031 objects degraded (3.925%); 2717/149031 objects misplaced (1.823%); 985 KiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T15:26:20.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:20 smithi017 ceph-mon[222878]: pgmap v1038: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 5849/149031 objects degraded (3.925%); 2717/149031 objects misplaced (1.823%); 985 KiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T15:26:20.681 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:20 smithi027 ceph-mon[195233]: pgmap v1038: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 peering, 104 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 36 MiB/s wr, 580 op/s; 5849/149031 objects degraded (3.925%); 2717/149031 objects misplaced (1.823%); 985 KiB/s, 1 keys/s, 20 objects/s recovering 2023-09-23T15:26:20.894 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:20.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:20.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:20.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 55278 55262 95.877 52.75 0.0288371 0.0104244 2023-09-23T15:26:21.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:21.084 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:21.224 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:21.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:21.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:21.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:21.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:26:21.386 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:21.460 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:21.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:21.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:21.497 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:21.558 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:26:21.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:21.696 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:26:21.713 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:21.713 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:26:21.750 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:21.851 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:21.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:21.853 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:26:21.853 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:26:21.853 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:26:21.853 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:26:21.853 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:26:21.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:21.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:21.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:21.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 56194 56178 94.8334 57.25 0.0116295 0.0105386 2023-09-23T15:26:22.047 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: pgmap v1039: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s; 5849/160914 objects degraded (3.635%); 2457/160914 objects misplaced (1.527%); 2.8 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/160914 objects degraded (3.635%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:22 smithi027 ceph-mon[195233]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T15:26:22.636 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:22.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:22.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:22.916 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: pgmap v1039: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s; 5849/160914 objects degraded (3.635%); 2457/160914 objects misplaced (1.527%); 2.8 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:26:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/160914 objects degraded (3.635%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[222878]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: pgmap v1039: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 60 MiB/s wr, 962 op/s; 5849/160914 objects degraded (3.635%); 2457/160914 objects misplaced (1.527%); 2.8 MiB/s, 1 keys/s, 48 objects/s recovering 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/160914 objects degraded (3.635%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: failed to encode map e352 with expected crc 2023-09-23T15:26:22.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2924440334' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:22.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:22 smithi017 ceph-mon[215646]: osdmap e352: 8 total, 8 up, 7 in 2023-09-23T15:26:22.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 56946 56930 93.5752 47 0.015246 0.0106821 2023-09-23T15:26:23.028 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:23.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:23.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:23.058 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:26:23.091 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1040,"stamp":"2023-09-23T15:26:21.494291+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3518175155,"num_objects":53638,"num_object_clones":0,"num_object_copies":160914,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5849,"num_objects_misplaced":2457,"num_objects_unfound":0,"num_objects_dirty":53638,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":54517,"num_write_kb":3443765,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":533,"num_bytes_recovered":32451476,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":58023,"ondisk_log_size":58023,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13455628,"kb_used_data":10354964,"kb_used_omap":44,"kb_used_meta":3100563,"kb_avail":642760436,"statfs":{"total":671965249536,"available":658186686464,"internally_reserved":0,"allocated":10603483136,"data_stored":10593905827,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3174977467},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,2,0,0,5,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":504692736,"num_objects":7701,"num_object_clones":0,"num_object_copies":23103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5849,"num_objects_misplaced":2457,"num_objects_unfound":0,"num_objects_dirty":7701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7701,"num_write_kb":492864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":391,"num_bytes_recovered":23532304,"num_keys_recovered":10,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.119664"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":66,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.394191+0000","last_change":"2023-09-23T15:26:11.368279+0000","last_active":"2023-09-23T15:26:13.394191+0000","last_peered":"2023-09-23T15:26:13.394191+0000","last_clean":"2023-09-23T15:26:13.394191+0000","last_became_active":"2023-09-23T15:26:11.367801+0000","last_became_peered":"2023-09-23T15:26:11.367801+0000","last_unstale":"2023-09-23T15:26:13.394191+0000","last_undegraded":"2023-09-23T15:26:13.394191+0000","last_fullsized":"2023-09-23T15:26:13.394191+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:26.099110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":273,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380689+0000","last_change":"2023-09-23T15:26:11.385221+0000","last_active":"2023-09-23T15:26:13.380689+0000","last_peered":"2023-09-23T15:26:13.380689+0000","last_clean":"2023-09-23T15:26:13.380689+0000","last_became_active":"2023-09-23T15:26:11.384389+0000","last_became_peered":"2023-09-23T15:26:11.384389+0000","last_unstale":"2023-09-23T15:26:13.380689+0000","last_undegraded":"2023-09-23T15:26:13.380689+0000","last_fullsized":"2023-09-23T15:26:13.380689+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:30.414963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":265,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.393630+0000","last_change":"2023-09-23T15:26:11.368570+0000","last_active":"2023-09-23T15:26:13.393630+0000","last_peered":"2023-09-23T15:26:13.393630+0000","last_clean":"2023-09-23T15:26:13.393630+0000","last_became_active":"2023-09-23T15:26:11.368201+0000","last_became_peered":"2023-09-23T15:26:11.368201+0000","last_unstale":"2023-09-23T15:26:13.393630+0000","last_undegraded":"2023-09-23T15:26:13.393630+0000","last_fullsized":"2023-09-23T15:26:13.393630+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:38.276306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":330,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.399225+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:26:13.399225+0000","last_peered":"2023-09-23T15:26:13.399225+0000","last_clean":"2023-09-23T15:26:13.399225+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:26:13.399225+0000","last_undegraded":"2023-09-23T15:26:13.399225+0000","last_fullsized":"2023-09-23T15:26:13.399225+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":190,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.391423+0000","last_change":"2023-09-23T15:25:42.528502+0000","last_active":"2023-09-23T15:26:13.391423+0000","last_peered":"2023-09-23T15:26:13.391423+0000","last_clean":"2023-09-23T15:26:13.391423+0000","last_became_active":"2023-09-23T15:25:42.528034+0000","last_became_peered":"2023-09-23T15:25:42.528034+0000","last_unstale":"2023-09-23T15:26:13.391423+0000","last_undegraded":"2023-09-23T15:26:13.391423+0000","last_fullsized":"2023-09-23T15:26:13.391423+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.419787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":301,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.387127+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:26:13.387127+0000","last_peered":"2023-09-23T15:26:13.387127+0000","last_clean":"2023-09-23T15:26:13.387127+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:26:13.387127+0000","last_undegraded":"2023-09-23T15:26:13.387127+0000","last_fullsized":"2023-09-23T15:26:13.387127+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":299,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.398834+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:26:13.398834+0000","last_peered":"2023-09-23T15:26:13.398834+0000","last_clean":"2023-09-23T15:26:13.398834+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:26:13.398834+0000","last_undegraded":"2023-09-23T15:26:13.398834+0000","last_fullsized":"2023-09-23T15:26:13.398834+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":312,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.394838+0000","last_change":"2023-09-23T15:25:42.529084+0000","last_active":"2023-09-23T15:26:13.394838+0000","last_peered":"2023-09-23T15:26:13.394838+0000","last_clean":"2023-09-23T15:26:13.394838+0000","last_became_active":"2023-09-23T15:25:42.528357+0000","last_became_peered":"2023-09-23T15:25:42.528357+0000","last_unstale":"2023-09-23T15:26:13.394838+0000","last_undegraded":"2023-09-23T15:26:13.394838+0000","last_fullsized":"2023-09-23T15:26:13.394838+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:21.317285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":727,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.399298+0000","last_change":"2023-09-23T15:26:14.399298+0000","last_active":"2023-09-23T15:26:14.399298+0000","last_peered":"2023-09-23T15:26:14.399298+0000","last_clean":"2023-09-23T15:26:14.399298+0000","last_became_active":"2023-09-23T15:26:14.398931+0000","last_became_peered":"2023-09-23T15:26:14.398931+0000","last_unstale":"2023-09-23T15:26:14.399298+0000","last_undegraded":"2023-09-23T15:26:14.399298+0000","last_fullsized":"2023-09-23T15:26:14.399298+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:28.275467+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":351,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380455+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:26:13.380455+0000","last_peered":"2023-09-23T15:26:13.380455+0000","last_clean":"2023-09-23T15:26:13.380455+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:26:13.380455+0000","last_undegraded":"2023-09-23T15:26:13.380455+0000","last_fullsized":"2023-09-23T15:26:13.380455+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":268,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388743+0000","last_change":"2023-09-23T15:25:42.526106+0000","last_active":"2023-09-23T15:26:13.388743+0000","last_peered":"2023-09-23T15:26:13.388743+0000","last_clean":"2023-09-23T15:26:13.388743+0000","last_became_active":"2023-09-23T15:25:42.525734+0000","last_became_peered":"2023-09-23T15:25:42.525734+0000","last_unstale":"2023-09-23T15:26:13.388743+0000","last_undegraded":"2023-09-23T15:26:13.388743+0000","last_fullsized":"2023-09-23T15:26:13.388743+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:09.036516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":111,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.390322+0000","last_change":"2023-09-23T15:26:11.403429+0000","last_active":"2023-09-23T15:26:13.390322+0000","last_peered":"2023-09-23T15:26:13.390322+0000","last_clean":"2023-09-23T15:26:13.390322+0000","last_became_active":"2023-09-23T15:26:11.383762+0000","last_became_peered":"2023-09-23T15:26:11.383762+0000","last_unstale":"2023-09-23T15:26:13.390322+0000","last_undegraded":"2023-09-23T15:26:13.390322+0000","last_fullsized":"2023-09-23T15:26:13.390322+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:39.355290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.396179+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:26:13.396179+0000","last_peered":"2023-09-23T15:26:13.396179+0000","last_clean":"2023-09-23T15:26:13.396179+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:26:13.396179+0000","last_undegraded":"2023-09-23T15:26:13.396179+0000","last_fullsized":"2023-09-23T15:26:13.396179+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":57,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.393332+0000","last_change":"2023-09-23T15:26:11.396846+0000","last_active":"2023-09-23T15:26:13.393332+0000","last_peered":"2023-09-23T15:26:13.393332+0000","last_clean":"2023-09-23T15:26:13.393332+0000","last_became_active":"2023-09-23T15:26:11.380259+0000","last_became_peered":"2023-09-23T15:26:11.380259+0000","last_unstale":"2023-09-23T15:26:13.393332+0000","last_undegraded":"2023-09-23T15:26:13.393332+0000","last_fullsized":"2023-09-23T15:26:13.393332+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:32.176239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":238,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.386173+0000","last_change":"2023-09-23T15:26:11.374626+0000","last_active":"2023-09-23T15:26:13.386173+0000","last_peered":"2023-09-23T15:26:13.386173+0000","last_clean":"2023-09-23T15:26:13.386173+0000","last_became_active":"2023-09-23T15:26:11.374231+0000","last_became_peered":"2023-09-23T15:26:11.374231+0000","last_unstale":"2023-09-23T15:26:13.386173+0000","last_undegraded":"2023-09-23T15:26:13.386173+0000","last_fullsized":"2023-09-23T15:26:13.386173+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:17:23.765320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":661,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.392028+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:26:13.392028+0000","last_peered":"2023-09-23T15:26:13.392028+0000","last_clean":"2023-09-23T15:26:13.392028+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:26:13.392028+0000","last_undegraded":"2023-09-23T15:26:13.392028+0000","last_fullsized":"2023-09-23T15:26:13.392028+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":115,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388933+0000","last_change":"2023-09-23T15:26:11.388830+0000","last_active":"2023-09-23T15:26:13.388933+0000","last_peered":"2023-09-23T15:26:13.388933+0000","last_clean":"2023-09-23T15:26:13.388933+0000","last_became_active":"2023-09-23T15:26:11.379056+0000","last_became_peered":"2023-09-23T15:26:11.379056+0000","last_unstale":"2023-09-23T15:26:13.388933+0000","last_undegraded":"2023-09-23T15:26:13.388933+0000","last_fullsized":"2023-09-23T15:26:13.388933+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:26:42.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":300,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.397156+0000","last_change":"2023-09-23T15:25:42.527397+0000","last_active":"2023-09-23T15:26:13.397156+0000","last_peered":"2023-09-23T15:26:13.397156+0000","last_clean":"2023-09-23T15:26:13.397156+0000","last_became_active":"2023-09-23T15:25:42.527091+0000","last_became_peered":"2023-09-23T15:25:42.527091+0000","last_unstale":"2023-09-23T15:26:13.397156+0000","last_undegraded":"2023-09-23T15:26:13.397156+0000","last_fullsized":"2023-09-23T15:26:13.397156+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:58.238273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":709,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380047+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:26:13.380047+0000","last_peered":"2023-09-23T15:26:13.380047+0000","last_clean":"2023-09-23T15:26:13.380047+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:26:13.380047+0000","last_undegraded":"2023-09-23T15:26:13.380047+0000","last_fullsized":"2023-09-23T15:26:13.380047+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.391057+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:26:13.391057+0000","last_peered":"2023-09-23T15:26:13.391057+0000","last_clean":"2023-09-23T15:26:13.391057+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:26:13.391057+0000","last_undegraded":"2023-09-23T15:26:13.391057+0000","last_fullsized":"2023-09-23T15:26:13.391057+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":336,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380602+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:26:13.380602+0000","last_peered":"2023-09-23T15:26:13.380602+0000","last_clean":"2023-09-23T15:26:13.380602+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:26:13.380602+0000","last_undegraded":"2023-09-23T15:26:13.380602+0000","last_fullsized":"2023-09-23T15:26:13.380602+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":659,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.410996+0000","last_change":"2023-09-23T15:26:14.410996+0000","last_active":"2023-09-23T15:26:14.410996+0000","last_peered":"2023-09-23T15:26:14.410996+0000","last_clean":"2023-09-23T15:26:14.410996+0000","last_became_active":"2023-09-23T15:26:14.410628+0000","last_became_peered":"2023-09-23T15:26:14.410628+0000","last_unstale":"2023-09-23T15:26:14.410996+0000","last_undegraded":"2023-09-23T15:26:14.410996+0000","last_fullsized":"2023-09-23T15:26:14.410996+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:54.330177+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":109,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388662+0000","last_change":"2023-09-23T15:26:11.381231+0000","last_active":"2023-09-23T15:26:13.388662+0000","last_peered":"2023-09-23T15:26:13.388662+0000","last_clean":"2023-09-23T15:26:13.388662+0000","last_became_active":"2023-09-23T15:26:11.380888+0000","last_became_peered":"2023-09-23T15:26:11.380888+0000","last_unstale":"2023-09-23T15:26:13.388662+0000","last_undegraded":"2023-09-23T15:26:13.388662+0000","last_fullsized":"2023-09-23T15:26:13.388662+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:12.014493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":317,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.399291+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:26:13.399291+0000","last_peered":"2023-09-23T15:26:13.399291+0000","last_clean":"2023-09-23T15:26:13.399291+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:26:13.399291+0000","last_undegraded":"2023-09-23T15:26:13.399291+0000","last_fullsized":"2023-09-23T15:26:13.399291+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":667,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380269+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:26:13.380269+0000","last_peered":"2023-09-23T15:26:13.380269+0000","last_clean":"2023-09-23T15:26:13.380269+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:26:13.380269+0000","last_undegraded":"2023-09-23T15:26:13.380269+0000","last_fullsized":"2023-09-23T15:26:13.380269+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":287,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385788+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:26:13.385788+0000","last_peered":"2023-09-23T15:26:13.385788+0000","last_clean":"2023-09-23T15:26:13.385788+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:26:13.385788+0000","last_undegraded":"2023-09-23T15:26:13.385788+0000","last_fullsized":"2023-09-23T15:26:13.385788+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":301,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.389072+0000","last_change":"2023-09-23T15:25:42.532898+0000","last_active":"2023-09-23T15:26:13.389072+0000","last_peered":"2023-09-23T15:26:13.389072+0000","last_clean":"2023-09-23T15:26:13.389072+0000","last_became_active":"2023-09-23T15:25:42.532572+0000","last_became_peered":"2023-09-23T15:25:42.532572+0000","last_unstale":"2023-09-23T15:26:13.389072+0000","last_undegraded":"2023-09-23T15:26:13.389072+0000","last_fullsized":"2023-09-23T15:26:13.389072+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:55.632166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"351'3373","reported_seq":3394,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.237707+0000","last_change":"2023-09-23T15:25:35.709660+0000","last_active":"2023-09-23T15:26:19.237707+0000","last_peered":"2023-09-23T15:26:19.237707+0000","last_clean":"2023-09-23T15:26:19.237707+0000","last_became_active":"2023-09-23T15:25:35.709292+0000","last_became_peered":"2023-09-23T15:25:35.709292+0000","last_unstale":"2023-09-23T15:26:19.237707+0000","last_undegraded":"2023-09-23T15:26:19.237707+0000","last_fullsized":"2023-09-23T15:26:19.237707+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3373,"log_dups_size":0,"ondisk_log_size":3373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:09.665343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221052928,"num_objects":3373,"num_object_clones":0,"num_object_copies":10119,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3373,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3373,"num_write_kb":215872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":633,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.396785+0000","last_change":"2023-09-23T15:26:14.396785+0000","last_active":"2023-09-23T15:26:14.396785+0000","last_peered":"2023-09-23T15:26:14.396785+0000","last_clean":"2023-09-23T15:26:14.396785+0000","last_became_active":"2023-09-23T15:26:14.396456+0000","last_became_peered":"2023-09-23T15:26:14.396456+0000","last_unstale":"2023-09-23T15:26:14.396785+0000","last_undegraded":"2023-09-23T15:26:14.396785+0000","last_fullsized":"2023-09-23T15:26:14.396785+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:47.610498+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":105,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388991+0000","last_change":"2023-09-23T15:26:11.400287+0000","last_active":"2023-09-23T15:26:13.388991+0000","last_peered":"2023-09-23T15:26:13.388991+0000","last_clean":"2023-09-23T15:26:13.388991+0000","last_became_active":"2023-09-23T15:26:11.380840+0000","last_became_peered":"2023-09-23T15:26:11.380840+0000","last_unstale":"2023-09-23T15:26:13.388991+0000","last_undegraded":"2023-09-23T15:26:13.388991+0000","last_fullsized":"2023-09-23T15:26:13.388991+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:09.995914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":343,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.379940+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:26:13.379940+0000","last_peered":"2023-09-23T15:26:13.379940+0000","last_clean":"2023-09-23T15:26:13.379940+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:26:13.379940+0000","last_undegraded":"2023-09-23T15:26:13.379940+0000","last_fullsized":"2023-09-23T15:26:13.379940+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"351'3270","reported_seq":3299,"reported_epoch":351,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:26:17.134459+0000","last_change":"2023-09-23T15:26:12.489819+0000","last_active":"2023-09-23T15:26:17.134459+0000","last_peered":"2023-09-23T15:26:17.134459+0000","last_clean":"2023-09-23T15:26:10.351458+0000","last_became_active":"2023-09-23T15:26:12.482684+0000","last_became_peered":"2023-09-23T15:26:12.482684+0000","last_unstale":"2023-09-23T15:26:17.134459+0000","last_undegraded":"2023-09-23T15:26:12.376377+0000","last_fullsized":"2023-09-23T15:26:12.360595+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3270,"log_dups_size":0,"ondisk_log_size":3270,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:02.300238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214302720,"num_objects":3270,"num_object_clones":0,"num_object_copies":9810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3270,"num_write_kb":209280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3","objects":348},{"shards":"1,3,4","objects":2922}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":544,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.401968+0000","last_change":"2023-09-23T15:25:48.080098+0000","last_active":"2023-09-23T15:26:13.401968+0000","last_peered":"2023-09-23T15:26:13.401968+0000","last_clean":"2023-09-23T15:26:13.401968+0000","last_became_active":"2023-09-23T15:25:42.528109+0000","last_became_peered":"2023-09-23T15:25:42.528109+0000","last_unstale":"2023-09-23T15:26:13.401968+0000","last_undegraded":"2023-09-23T15:26:13.401968+0000","last_fullsized":"2023-09-23T15:26:13.401968+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:48.079935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:48.079935+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:30.039456+0000","scrub_duration":0.0050498640000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":105,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.389951+0000","last_change":"2023-09-23T15:26:11.405379+0000","last_active":"2023-09-23T15:26:13.389951+0000","last_peered":"2023-09-23T15:26:13.389951+0000","last_clean":"2023-09-23T15:26:13.389951+0000","last_became_active":"2023-09-23T15:26:11.384740+0000","last_became_peered":"2023-09-23T15:26:11.384740+0000","last_unstale":"2023-09-23T15:26:13.389951+0000","last_undegraded":"2023-09-23T15:26:13.389951+0000","last_fullsized":"2023-09-23T15:26:13.389951+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:07.211920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.393530+0000","last_change":"2023-09-23T15:26:11.367504+0000","last_active":"2023-09-23T15:26:13.393530+0000","last_peered":"2023-09-23T15:26:13.393530+0000","last_clean":"2023-09-23T15:26:13.393530+0000","last_became_active":"2023-09-23T15:26:11.367129+0000","last_became_peered":"2023-09-23T15:26:11.367129+0000","last_unstale":"2023-09-23T15:26:13.393530+0000","last_undegraded":"2023-09-23T15:26:13.393530+0000","last_fullsized":"2023-09-23T15:26:13.393530+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:17.725798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"351'3381","reported_seq":3401,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.869459+0000","last_change":"2023-09-23T15:25:35.702721+0000","last_active":"2023-09-23T15:26:19.869459+0000","last_peered":"2023-09-23T15:26:19.869459+0000","last_clean":"2023-09-23T15:26:19.869459+0000","last_became_active":"2023-09-23T15:25:35.702416+0000","last_became_peered":"2023-09-23T15:25:35.702416+0000","last_unstale":"2023-09-23T15:26:19.869459+0000","last_undegraded":"2023-09-23T15:26:19.869459+0000","last_fullsized":"2023-09-23T15:26:19.869459+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3381,"log_dups_size":0,"ondisk_log_size":3381,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:06.939333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221577216,"num_objects":3381,"num_object_clones":0,"num_object_copies":10143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3381,"num_write_kb":216384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":578,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.412884+0000","last_change":"2023-09-23T15:26:14.412884+0000","last_active":"2023-09-23T15:26:14.412884+0000","last_peered":"2023-09-23T15:26:14.412884+0000","last_clean":"2023-09-23T15:26:14.412884+0000","last_became_active":"2023-09-23T15:26:14.412536+0000","last_became_peered":"2023-09-23T15:26:14.412536+0000","last_unstale":"2023-09-23T15:26:14.412884+0000","last_undegraded":"2023-09-23T15:26:14.412884+0000","last_fullsized":"2023-09-23T15:26:14.412884+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:47.772813+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":91,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.390237+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:26:13.390237+0000","last_peered":"2023-09-23T15:26:13.390237+0000","last_clean":"2023-09-23T15:26:13.390237+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:26:13.390237+0000","last_undegraded":"2023-09-23T15:26:13.390237+0000","last_fullsized":"2023-09-23T15:26:13.390237+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":314,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385161+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:26:13.385161+0000","last_peered":"2023-09-23T15:26:13.385161+0000","last_clean":"2023-09-23T15:26:13.385161+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:26:13.385161+0000","last_undegraded":"2023-09-23T15:26:13.385161+0000","last_fullsized":"2023-09-23T15:26:13.385161+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"351'3380","reported_seq":3400,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.870863+0000","last_change":"2023-09-23T15:25:35.723494+0000","last_active":"2023-09-23T15:26:19.870863+0000","last_peered":"2023-09-23T15:26:19.870863+0000","last_clean":"2023-09-23T15:26:19.870863+0000","last_became_active":"2023-09-23T15:25:35.723097+0000","last_became_peered":"2023-09-23T15:25:35.723097+0000","last_unstale":"2023-09-23T15:26:19.870863+0000","last_undegraded":"2023-09-23T15:26:19.870863+0000","last_fullsized":"2023-09-23T15:26:19.870863+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3380,"log_dups_size":0,"ondisk_log_size":3380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:58.863529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221511680,"num_objects":3380,"num_object_clones":0,"num_object_copies":10140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3380,"num_write_kb":216320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":559,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.408758+0000","last_change":"2023-09-23T15:26:14.408758+0000","last_active":"2023-09-23T15:26:14.408758+0000","last_peered":"2023-09-23T15:26:14.408758+0000","last_clean":"2023-09-23T15:26:14.408758+0000","last_became_active":"2023-09-23T15:26:14.408423+0000","last_became_peered":"2023-09-23T15:26:14.408423+0000","last_unstale":"2023-09-23T15:26:14.408758+0000","last_undegraded":"2023-09-23T15:26:14.408758+0000","last_fullsized":"2023-09-23T15:26:14.408758+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:42.833633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:42.833633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:53.130509+0000","scrub_duration":0.003114112,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":327,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380332+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:26:13.380332+0000","last_peered":"2023-09-23T15:26:13.380332+0000","last_clean":"2023-09-23T15:26:13.380332+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:26:13.380332+0000","last_undegraded":"2023-09-23T15:26:13.380332+0000","last_fullsized":"2023-09-23T15:26:13.380332+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":342,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.392133+0000","last_change":"2023-09-23T15:26:11.360512+0000","last_active":"2023-09-23T15:26:13.392133+0000","last_peered":"2023-09-23T15:26:13.392133+0000","last_clean":"2023-09-23T15:26:13.392133+0000","last_became_active":"2023-09-23T15:26:11.360114+0000","last_became_peered":"2023-09-23T15:26:11.360114+0000","last_unstale":"2023-09-23T15:26:13.392133+0000","last_undegraded":"2023-09-23T15:26:13.392133+0000","last_fullsized":"2023-09-23T15:26:13.392133+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:17:29.368629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"351'3405","reported_seq":3426,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.493517+0000","last_change":"2023-09-23T15:25:42.524482+0000","last_active":"2023-09-23T15:26:19.493517+0000","last_peered":"2023-09-23T15:26:19.493517+0000","last_clean":"2023-09-23T15:26:19.493517+0000","last_became_active":"2023-09-23T15:25:42.524107+0000","last_became_peered":"2023-09-23T15:25:42.524107+0000","last_unstale":"2023-09-23T15:26:19.493517+0000","last_undegraded":"2023-09-23T15:26:19.493517+0000","last_fullsized":"2023-09-23T15:26:19.493517+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3405,"log_dups_size":0,"ondisk_log_size":3405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:18.831761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223150080,"num_objects":3405,"num_object_clones":0,"num_object_copies":10215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3405,"num_write_kb":217920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":587,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388966+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:26:13.388966+0000","last_peered":"2023-09-23T15:26:13.388966+0000","last_clean":"2023-09-23T15:26:13.388966+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:26:13.388966+0000","last_undegraded":"2023-09-23T15:26:13.388966+0000","last_fullsized":"2023-09-23T15:26:13.388966+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":51,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.392229+0000","last_change":"2023-09-23T15:26:11.374595+0000","last_active":"2023-09-23T15:26:13.392229+0000","last_peered":"2023-09-23T15:26:13.392229+0000","last_clean":"2023-09-23T15:26:13.392229+0000","last_became_active":"2023-09-23T15:26:11.360257+0000","last_became_peered":"2023-09-23T15:26:11.360257+0000","last_unstale":"2023-09-23T15:26:13.392229+0000","last_undegraded":"2023-09-23T15:26:13.392229+0000","last_fullsized":"2023-09-23T15:26:13.392229+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:33.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":79,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385634+0000","last_change":"2023-09-23T15:26:11.371620+0000","last_active":"2023-09-23T15:26:13.385634+0000","last_peered":"2023-09-23T15:26:13.385634+0000","last_clean":"2023-09-23T15:26:13.385634+0000","last_became_active":"2023-09-23T15:26:11.371004+0000","last_became_peered":"2023-09-23T15:26:11.371004+0000","last_unstale":"2023-09-23T15:26:13.385634+0000","last_undegraded":"2023-09-23T15:26:13.385634+0000","last_fullsized":"2023-09-23T15:26:13.385634+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:05:41.106442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.0","version":"351'3308","reported_seq":3329,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:17.167161+0000","last_change":"2023-09-23T15:25:35.722919+0000","last_active":"2023-09-23T15:26:17.167161+0000","last_peered":"2023-09-23T15:26:17.167161+0000","last_clean":"2023-09-23T15:26:17.167161+0000","last_became_active":"2023-09-23T15:25:35.722667+0000","last_became_peered":"2023-09-23T15:25:35.722667+0000","last_unstale":"2023-09-23T15:26:17.167161+0000","last_undegraded":"2023-09-23T15:26:17.167161+0000","last_fullsized":"2023-09-23T15:26:17.167161+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3308,"log_dups_size":0,"ondisk_log_size":3308,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:51.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216793088,"num_objects":3308,"num_object_clones":0,"num_object_copies":9924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3308,"num_write_kb":211712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":655,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380174+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:26:13.380174+0000","last_peered":"2023-09-23T15:26:13.380174+0000","last_clean":"2023-09-23T15:26:13.380174+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:26:13.380174+0000","last_undegraded":"2023-09-23T15:26:13.380174+0000","last_fullsized":"2023-09-23T15:26:13.380174+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":308,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385255+0000","last_change":"2023-09-23T15:26:11.369439+0000","last_active":"2023-09-23T15:26:13.385255+0000","last_peered":"2023-09-23T15:26:13.385255+0000","last_clean":"2023-09-23T15:26:13.385255+0000","last_became_active":"2023-09-23T15:26:11.369036+0000","last_became_peered":"2023-09-23T15:26:11.369036+0000","last_unstale":"2023-09-23T15:26:13.385255+0000","last_undegraded":"2023-09-23T15:26:13.385255+0000","last_fullsized":"2023-09-23T15:26:13.385255+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:54.756776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":239,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.278323+0000","last_change":"2023-09-23T15:26:11.370435+0000","last_active":"2023-09-23T15:26:14.278323+0000","last_peered":"2023-09-23T15:26:14.278323+0000","last_clean":"2023-09-23T15:26:14.278323+0000","last_became_active":"2023-09-23T15:26:11.370002+0000","last_became_peered":"2023-09-23T15:26:11.370002+0000","last_unstale":"2023-09-23T15:26:14.278323+0000","last_undegraded":"2023-09-23T15:26:14.278323+0000","last_fullsized":"2023-09-23T15:26:14.278323+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:28.143906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"351'3331","reported_seq":3351,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.449721+0000","last_change":"2023-09-23T15:25:35.712664+0000","last_active":"2023-09-23T15:26:19.449721+0000","last_peered":"2023-09-23T15:26:19.449721+0000","last_clean":"2023-09-23T15:26:19.449721+0000","last_became_active":"2023-09-23T15:25:35.712363+0000","last_became_peered":"2023-09-23T15:25:35.712363+0000","last_unstale":"2023-09-23T15:26:19.449721+0000","last_undegraded":"2023-09-23T15:26:19.449721+0000","last_fullsized":"2023-09-23T15:26:19.449721+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3331,"log_dups_size":0,"ondisk_log_size":3331,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:33:38.768683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218300416,"num_objects":3331,"num_object_clones":0,"num_object_copies":9993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3331,"num_write_kb":213184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":608,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.400781+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:26:13.400781+0000","last_peered":"2023-09-23T15:26:13.400781+0000","last_clean":"2023-09-23T15:26:13.400781+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:26:13.400781+0000","last_undegraded":"2023-09-23T15:26:13.400781+0000","last_fullsized":"2023-09-23T15:26:13.400781+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":235,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.278246+0000","last_change":"2023-09-23T15:26:11.371002+0000","last_active":"2023-09-23T15:26:14.278246+0000","last_peered":"2023-09-23T15:26:14.278246+0000","last_clean":"2023-09-23T15:26:14.278246+0000","last_became_active":"2023-09-23T15:26:11.370662+0000","last_became_peered":"2023-09-23T15:26:11.370662+0000","last_unstale":"2023-09-23T15:26:14.278246+0000","last_undegraded":"2023-09-23T15:26:14.278246+0000","last_fullsized":"2023-09-23T15:26:14.278246+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:02.511171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":238,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385959+0000","last_change":"2023-09-23T15:26:11.375798+0000","last_active":"2023-09-23T15:26:13.385959+0000","last_peered":"2023-09-23T15:26:13.385959+0000","last_clean":"2023-09-23T15:26:13.385959+0000","last_became_active":"2023-09-23T15:26:11.375443+0000","last_became_peered":"2023-09-23T15:26:11.375443+0000","last_unstale":"2023-09-23T15:26:13.385959+0000","last_undegraded":"2023-09-23T15:26:13.385959+0000","last_fullsized":"2023-09-23T15:26:13.385959+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:07.695507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"351'3169","reported_seq":3189,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:17.120958+0000","last_change":"2023-09-23T15:25:42.527999+0000","last_active":"2023-09-23T15:26:17.120958+0000","last_peered":"2023-09-23T15:26:17.120958+0000","last_clean":"2023-09-23T15:26:17.120958+0000","last_became_active":"2023-09-23T15:25:42.527725+0000","last_became_peered":"2023-09-23T15:25:42.527725+0000","last_unstale":"2023-09-23T15:26:17.120958+0000","last_undegraded":"2023-09-23T15:26:17.120958+0000","last_fullsized":"2023-09-23T15:26:17.120958+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3169,"log_dups_size":0,"ondisk_log_size":3169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:59.569058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207683584,"num_objects":3169,"num_object_clones":0,"num_object_copies":9507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3169,"num_write_kb":202816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":667,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380168+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:26:13.380168+0000","last_peered":"2023-09-23T15:26:13.380168+0000","last_clean":"2023-09-23T15:26:13.380168+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:26:13.380168+0000","last_undegraded":"2023-09-23T15:26:13.380168+0000","last_fullsized":"2023-09-23T15:26:13.380168+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":310,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.394775+0000","last_change":"2023-09-23T15:25:42.529276+0000","last_active":"2023-09-23T15:26:13.394775+0000","last_peered":"2023-09-23T15:26:13.394775+0000","last_clean":"2023-09-23T15:26:13.394775+0000","last_became_active":"2023-09-23T15:25:42.528783+0000","last_became_peered":"2023-09-23T15:25:42.528783+0000","last_unstale":"2023-09-23T15:26:13.394775+0000","last_undegraded":"2023-09-23T15:26:13.394775+0000","last_fullsized":"2023-09-23T15:26:13.394775+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:22.381897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":269,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.394767+0000","last_change":"2023-09-23T15:26:11.380590+0000","last_active":"2023-09-23T15:26:14.394767+0000","last_peered":"2023-09-23T15:26:14.394767+0000","last_clean":"2023-09-23T15:26:14.394767+0000","last_became_active":"2023-09-23T15:26:11.380152+0000","last_became_peered":"2023-09-23T15:26:11.380152+0000","last_unstale":"2023-09-23T15:26:14.394767+0000","last_undegraded":"2023-09-23T15:26:14.394767+0000","last_fullsized":"2023-09-23T15:26:14.394767+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:27:44.803182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"351'3452","reported_seq":3472,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.870175+0000","last_change":"2023-09-23T15:25:35.716554+0000","last_active":"2023-09-23T15:26:19.870175+0000","last_peered":"2023-09-23T15:26:19.870175+0000","last_clean":"2023-09-23T15:26:19.870175+0000","last_became_active":"2023-09-23T15:25:35.716257+0000","last_became_peered":"2023-09-23T15:25:35.716257+0000","last_unstale":"2023-09-23T15:26:19.870175+0000","last_undegraded":"2023-09-23T15:26:19.870175+0000","last_fullsized":"2023-09-23T15:26:19.870175+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3452,"log_dups_size":0,"ondisk_log_size":3452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:37.096030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226230272,"num_objects":3452,"num_object_clones":0,"num_object_copies":10356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3452,"num_write_kb":220928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":548,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.387183+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:26:13.387183+0000","last_peered":"2023-09-23T15:26:13.387183+0000","last_clean":"2023-09-23T15:26:13.387183+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:26:13.387183+0000","last_undegraded":"2023-09-23T15:26:13.387183+0000","last_fullsized":"2023-09-23T15:26:13.387183+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":551,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.397420+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:26:13.397420+0000","last_peered":"2023-09-23T15:26:13.397420+0000","last_clean":"2023-09-23T15:26:13.397420+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:26:13.397420+0000","last_undegraded":"2023-09-23T15:26:13.397420+0000","last_fullsized":"2023-09-23T15:26:13.397420+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":327,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380527+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:26:13.380527+0000","last_peered":"2023-09-23T15:26:13.380527+0000","last_clean":"2023-09-23T15:26:13.380527+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:26:13.380527+0000","last_undegraded":"2023-09-23T15:26:13.380527+0000","last_fullsized":"2023-09-23T15:26:13.380527+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":304,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.383653+0000","last_change":"2023-09-23T15:26:11.374600+0000","last_active":"2023-09-23T15:26:13.383653+0000","last_peered":"2023-09-23T15:26:13.383653+0000","last_clean":"2023-09-23T15:26:13.383653+0000","last_became_active":"2023-09-23T15:26:11.374199+0000","last_became_peered":"2023-09-23T15:26:11.374199+0000","last_unstale":"2023-09-23T15:26:13.383653+0000","last_undegraded":"2023-09-23T15:26:13.383653+0000","last_fullsized":"2023-09-23T15:26:13.383653+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:52.247541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"351'3366","reported_seq":3387,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.278556+0000","last_change":"2023-09-23T15:25:35.708965+0000","last_active":"2023-09-23T15:26:19.278556+0000","last_peered":"2023-09-23T15:26:19.278556+0000","last_clean":"2023-09-23T15:26:19.278556+0000","last_became_active":"2023-09-23T15:25:35.708632+0000","last_became_peered":"2023-09-23T15:25:35.708632+0000","last_unstale":"2023-09-23T15:26:19.278556+0000","last_undegraded":"2023-09-23T15:26:19.278556+0000","last_fullsized":"2023-09-23T15:26:19.278556+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3366,"log_dups_size":0,"ondisk_log_size":3366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:15.667562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220594176,"num_objects":3366,"num_object_clones":0,"num_object_copies":10098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3366,"num_write_kb":215424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":648,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380059+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:26:13.380059+0000","last_peered":"2023-09-23T15:26:13.380059+0000","last_clean":"2023-09-23T15:26:13.380059+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:26:13.380059+0000","last_undegraded":"2023-09-23T15:26:13.380059+0000","last_fullsized":"2023-09-23T15:26:13.380059+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":886,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380451+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:26:13.380451+0000","last_peered":"2023-09-23T15:26:13.380451+0000","last_clean":"2023-09-23T15:26:13.380451+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:26:13.380451+0000","last_undegraded":"2023-09-23T15:26:13.380451+0000","last_fullsized":"2023-09-23T15:26:13.380451+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":119,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.293116+0000","last_change":"2023-09-23T15:26:11.365938+0000","last_active":"2023-09-23T15:26:14.293116+0000","last_peered":"2023-09-23T15:26:14.293116+0000","last_clean":"2023-09-23T15:26:14.293116+0000","last_became_active":"2023-09-23T15:26:11.365355+0000","last_became_peered":"2023-09-23T15:26:11.365355+0000","last_unstale":"2023-09-23T15:26:14.293116+0000","last_undegraded":"2023-09-23T15:26:14.293116+0000","last_fullsized":"2023-09-23T15:26:14.293116+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:40.583184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":248,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.397084+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:26:13.397084+0000","last_peered":"2023-09-23T15:26:13.397084+0000","last_clean":"2023-09-23T15:26:13.397084+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:26:13.397084+0000","last_undegraded":"2023-09-23T15:26:13.397084+0000","last_fullsized":"2023-09-23T15:26:13.397084+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"351'3391","reported_seq":3411,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.867105+0000","last_change":"2023-09-23T15:25:35.715315+0000","last_active":"2023-09-23T15:26:19.867105+0000","last_peered":"2023-09-23T15:26:19.867105+0000","last_clean":"2023-09-23T15:26:19.867105+0000","last_became_active":"2023-09-23T15:25:35.715056+0000","last_became_peered":"2023-09-23T15:25:35.715056+0000","last_unstale":"2023-09-23T15:26:19.867105+0000","last_undegraded":"2023-09-23T15:26:19.867105+0000","last_fullsized":"2023-09-23T15:26:19.867105+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3391,"log_dups_size":0,"ondisk_log_size":3391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:18.608733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":222232576,"num_objects":3391,"num_object_clones":0,"num_object_copies":10173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3391,"num_write_kb":217024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":726,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.401428+0000","last_change":"2023-09-23T15:26:14.401428+0000","last_active":"2023-09-23T15:26:14.401428+0000","last_peered":"2023-09-23T15:26:14.401428+0000","last_clean":"2023-09-23T15:26:14.401428+0000","last_became_active":"2023-09-23T15:26:14.401034+0000","last_became_peered":"2023-09-23T15:26:14.401034+0000","last_unstale":"2023-09-23T15:26:14.401428+0000","last_undegraded":"2023-09-23T15:26:14.401428+0000","last_fullsized":"2023-09-23T15:26:14.401428+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:11.721227+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":968,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.382671+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:26:13.382671+0000","last_peered":"2023-09-23T15:26:13.382671+0000","last_clean":"2023-09-23T15:26:13.382671+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:26:13.382671+0000","last_undegraded":"2023-09-23T15:26:13.382671+0000","last_fullsized":"2023-09-23T15:26:13.382671+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":243,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.394907+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:26:13.394907+0000","last_peered":"2023-09-23T15:26:13.394907+0000","last_clean":"2023-09-23T15:26:13.394907+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:26:13.394907+0000","last_undegraded":"2023-09-23T15:26:13.394907+0000","last_fullsized":"2023-09-23T15:26:13.394907+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":252,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.395900+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:26:13.395900+0000","last_peered":"2023-09-23T15:26:13.395900+0000","last_clean":"2023-09-23T15:26:13.395900+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:26:13.395900+0000","last_undegraded":"2023-09-23T15:26:13.395900+0000","last_fullsized":"2023-09-23T15:26:13.395900+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"351'3317","reported_seq":3337,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.275762+0000","last_change":"2023-09-23T15:25:42.528955+0000","last_active":"2023-09-23T15:26:19.275762+0000","last_peered":"2023-09-23T15:26:19.275762+0000","last_clean":"2023-09-23T15:26:19.275762+0000","last_became_active":"2023-09-23T15:25:42.528662+0000","last_became_peered":"2023-09-23T15:25:42.528662+0000","last_unstale":"2023-09-23T15:26:19.275762+0000","last_undegraded":"2023-09-23T15:26:19.275762+0000","last_fullsized":"2023-09-23T15:26:19.275762+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3317,"log_dups_size":0,"ondisk_log_size":3317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:56.711464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217382912,"num_objects":3317,"num_object_clones":0,"num_object_copies":9951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3317,"num_write_kb":212288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":680,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.382679+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:26:13.382679+0000","last_peered":"2023-09-23T15:26:13.382679+0000","last_clean":"2023-09-23T15:26:13.382679+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:26:13.382679+0000","last_undegraded":"2023-09-23T15:26:13.382679+0000","last_fullsized":"2023-09-23T15:26:13.382679+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":670,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388824+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:26:13.388824+0000","last_peered":"2023-09-23T15:26:13.388824+0000","last_clean":"2023-09-23T15:26:13.388824+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:26:13.388824+0000","last_undegraded":"2023-09-23T15:26:13.388824+0000","last_fullsized":"2023-09-23T15:26:13.388824+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":248,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380362+0000","last_change":"2023-09-23T15:25:42.527275+0000","last_active":"2023-09-23T15:26:13.380362+0000","last_peered":"2023-09-23T15:26:13.380362+0000","last_clean":"2023-09-23T15:26:13.380362+0000","last_became_active":"2023-09-23T15:25:42.526943+0000","last_became_peered":"2023-09-23T15:25:42.526943+0000","last_unstale":"2023-09-23T15:26:13.380362+0000","last_undegraded":"2023-09-23T15:26:13.380362+0000","last_fullsized":"2023-09-23T15:26:13.380362+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:40.809855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":312,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385775+0000","last_change":"2023-09-23T15:26:11.382742+0000","last_active":"2023-09-23T15:26:13.385775+0000","last_peered":"2023-09-23T15:26:13.385775+0000","last_clean":"2023-09-23T15:26:13.385775+0000","last_became_active":"2023-09-23T15:26:11.382454+0000","last_became_peered":"2023-09-23T15:26:11.382454+0000","last_unstale":"2023-09-23T15:26:13.385775+0000","last_undegraded":"2023-09-23T15:26:13.385775+0000","last_fullsized":"2023-09-23T15:26:13.385775+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:58.937098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"351'3406","reported_seq":3427,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.280528+0000","last_change":"2023-09-23T15:25:35.708972+0000","last_active":"2023-09-23T15:26:19.280528+0000","last_peered":"2023-09-23T15:26:19.280528+0000","last_clean":"2023-09-23T15:26:19.280528+0000","last_became_active":"2023-09-23T15:25:35.708745+0000","last_became_peered":"2023-09-23T15:25:35.708745+0000","last_unstale":"2023-09-23T15:26:19.280528+0000","last_undegraded":"2023-09-23T15:26:19.280528+0000","last_fullsized":"2023-09-23T15:26:19.280528+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3406,"log_dups_size":0,"ondisk_log_size":3406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:56.572163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223215616,"num_objects":3406,"num_object_clones":0,"num_object_copies":10218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3406,"num_write_kb":217984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":675,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.401665+0000","last_change":"2023-09-23T15:26:14.401665+0000","last_active":"2023-09-23T15:26:14.401665+0000","last_peered":"2023-09-23T15:26:14.401665+0000","last_clean":"2023-09-23T15:26:14.401665+0000","last_became_active":"2023-09-23T15:26:14.401311+0000","last_became_peered":"2023-09-23T15:26:14.401311+0000","last_unstale":"2023-09-23T15:26:14.401665+0000","last_undegraded":"2023-09-23T15:26:14.401665+0000","last_fullsized":"2023-09-23T15:26:14.401665+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:04.328053+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":95,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388726+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:26:13.388726+0000","last_peered":"2023-09-23T15:26:13.388726+0000","last_clean":"2023-09-23T15:26:13.388726+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:26:13.388726+0000","last_undegraded":"2023-09-23T15:26:13.388726+0000","last_fullsized":"2023-09-23T15:26:13.388726+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":328,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.391938+0000","last_change":"2023-09-23T15:26:11.373443+0000","last_active":"2023-09-23T15:26:13.391938+0000","last_peered":"2023-09-23T15:26:13.391938+0000","last_clean":"2023-09-23T15:26:13.391938+0000","last_became_active":"2023-09-23T15:26:11.373168+0000","last_became_peered":"2023-09-23T15:26:11.373168+0000","last_unstale":"2023-09-23T15:26:13.391938+0000","last_undegraded":"2023-09-23T15:26:13.391938+0000","last_fullsized":"2023-09-23T15:26:13.391938+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:43.871002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":331,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.395482+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:26:14.395482+0000","last_peered":"2023-09-23T15:26:14.395482+0000","last_clean":"2023-09-23T15:26:14.395482+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:26:14.395482+0000","last_undegraded":"2023-09-23T15:26:14.395482+0000","last_fullsized":"2023-09-23T15:26:14.395482+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"351'3293","reported_seq":4041,"reported_epoch":351,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T15:26:19.477235+0000","last_change":"2023-09-23T15:26:12.521535+0000","last_active":"2023-09-23T15:26:19.477235+0000","last_peered":"2023-09-23T15:26:19.477235+0000","last_clean":"2023-09-23T15:26:10.351231+0000","last_became_active":"2023-09-23T15:26:12.469771+0000","last_became_peered":"2023-09-23T15:26:12.469771+0000","last_unstale":"2023-09-23T15:26:19.477235+0000","last_undegraded":"2023-09-23T15:26:19.477235+0000","last_fullsized":"2023-09-23T15:26:12.363885+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3293,"log_dups_size":0,"ondisk_log_size":3293,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:26.913358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810048,"num_objects":3293,"num_object_clones":0,"num_object_copies":9879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2457,"num_objects_unfound":0,"num_objects_dirty":3293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3293,"num_write_kb":210752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":359,"num_bytes_recovered":23527424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":2457},{"shards":"1,7","objects":836}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":639,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385877+0000","last_change":"2023-09-23T15:25:41.310902+0000","last_active":"2023-09-23T15:26:13.385877+0000","last_peered":"2023-09-23T15:26:13.385877+0000","last_clean":"2023-09-23T15:26:13.385877+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:26:13.385877+0000","last_undegraded":"2023-09-23T15:26:13.385877+0000","last_fullsized":"2023-09-23T15:26:13.385877+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:41.310801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:41.310801+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:40.274800+0000","scrub_duration":0.0019631739999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1102,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.405982+0000","last_change":"2023-09-23T15:26:14.405982+0000","last_active":"2023-09-23T15:26:14.405982+0000","last_peered":"2023-09-23T15:26:14.405982+0000","last_clean":"2023-09-23T15:26:14.405982+0000","last_became_active":"2023-09-23T15:26:14.405360+0000","last_became_peered":"2023-09-23T15:26:14.405360+0000","last_unstale":"2023-09-23T15:26:14.405982+0000","last_undegraded":"2023-09-23T15:26:14.405982+0000","last_fullsized":"2023-09-23T15:26:14.405982+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:09.602850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":228,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.397296+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:26:13.397296+0000","last_peered":"2023-09-23T15:26:13.397296+0000","last_clean":"2023-09-23T15:26:13.397296+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:26:13.397296+0000","last_undegraded":"2023-09-23T15:26:13.397296+0000","last_fullsized":"2023-09-23T15:26:13.397296+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":315,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.396438+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:26:13.396438+0000","last_peered":"2023-09-23T15:26:13.396438+0000","last_clean":"2023-09-23T15:26:13.396438+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:26:13.396438+0000","last_undegraded":"2023-09-23T15:26:13.396438+0000","last_fullsized":"2023-09-23T15:26:13.396438+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"351'3403","reported_seq":3423,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:19.460897+0000","last_change":"2023-09-23T15:25:35.714862+0000","last_active":"2023-09-23T15:26:19.460897+0000","last_peered":"2023-09-23T15:26:19.460897+0000","last_clean":"2023-09-23T15:26:19.460897+0000","last_became_active":"2023-09-23T15:25:35.714552+0000","last_became_peered":"2023-09-23T15:25:35.714552+0000","last_unstale":"2023-09-23T15:26:19.460897+0000","last_undegraded":"2023-09-23T15:26:19.460897+0000","last_fullsized":"2023-09-23T15:26:19.460897+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3403,"log_dups_size":0,"ondisk_log_size":3403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:35:31.672142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":223019008,"num_objects":3403,"num_object_clones":0,"num_object_copies":10209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3403,"num_write_kb":217792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":611,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.400502+0000","last_change":"2023-09-23T15:26:14.400502+0000","last_active":"2023-09-23T15:26:14.400502+0000","last_peered":"2023-09-23T15:26:14.400502+0000","last_clean":"2023-09-23T15:26:14.400502+0000","last_became_active":"2023-09-23T15:26:14.400112+0000","last_became_peered":"2023-09-23T15:26:14.400112+0000","last_unstale":"2023-09-23T15:26:14.400502+0000","last_undegraded":"2023-09-23T15:26:14.400502+0000","last_fullsized":"2023-09-23T15:26:14.400502+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:28.174971+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":107,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.383580+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:26:13.383580+0000","last_peered":"2023-09-23T15:26:13.383580+0000","last_clean":"2023-09-23T15:26:13.383580+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:26:13.383580+0000","last_undegraded":"2023-09-23T15:26:13.383580+0000","last_fullsized":"2023-09-23T15:26:13.383580+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":944,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.411224+0000","last_change":"2023-09-23T15:26:14.411224+0000","last_active":"2023-09-23T15:26:14.411224+0000","last_peered":"2023-09-23T15:26:14.411224+0000","last_clean":"2023-09-23T15:26:14.411224+0000","last_became_active":"2023-09-23T15:26:14.410592+0000","last_became_peered":"2023-09-23T15:26:14.410592+0000","last_unstale":"2023-09-23T15:26:14.411224+0000","last_undegraded":"2023-09-23T15:26:14.411224+0000","last_fullsized":"2023-09-23T15:26:14.411224+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:13.276904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":291,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.388519+0000","last_change":"2023-09-23T15:25:42.526102+0000","last_active":"2023-09-23T15:26:13.388519+0000","last_peered":"2023-09-23T15:26:13.388519+0000","last_clean":"2023-09-23T15:26:13.388519+0000","last_became_active":"2023-09-23T15:25:42.525794+0000","last_became_peered":"2023-09-23T15:25:42.525794+0000","last_unstale":"2023-09-23T15:26:13.388519+0000","last_undegraded":"2023-09-23T15:26:13.388519+0000","last_fullsized":"2023-09-23T15:26:13.388519+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:10.472481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"351'3365","reported_seq":3394,"reported_epoch":351,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:26:19.467631+0000","last_change":"2023-09-23T15:26:12.496201+0000","last_active":"2023-09-23T15:26:19.467631+0000","last_peered":"2023-09-23T15:26:19.467631+0000","last_clean":"2023-09-23T15:26:10.342805+0000","last_became_active":"2023-09-23T15:26:12.482799+0000","last_became_peered":"2023-09-23T15:26:12.482799+0000","last_unstale":"2023-09-23T15:26:19.467631+0000","last_undegraded":"2023-09-23T15:26:12.388779+0000","last_fullsized":"2023-09-23T15:26:12.364534+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":3365,"log_dups_size":0,"ondisk_log_size":3365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:24.405535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220528640,"num_objects":3365,"num_object_clones":0,"num_object_copies":10095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2927,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3365,"num_write_kb":215360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":2927},{"shards":"3,7","objects":438}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":531,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380186+0000","last_change":"2023-09-23T15:25:42.524359+0000","last_active":"2023-09-23T15:26:13.380186+0000","last_peered":"2023-09-23T15:26:13.380186+0000","last_clean":"2023-09-23T15:26:13.380186+0000","last_became_active":"2023-09-23T15:25:42.523974+0000","last_became_peered":"2023-09-23T15:25:42.523974+0000","last_unstale":"2023-09-23T15:26:13.380186+0000","last_undegraded":"2023-09-23T15:26:13.380186+0000","last_fullsized":"2023-09-23T15:26:13.380186+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:29.868140+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":787,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.387045+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:26:13.387045+0000","last_peered":"2023-09-23T15:26:13.387045+0000","last_clean":"2023-09-23T15:26:13.387045+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:26:13.387045+0000","last_undegraded":"2023-09-23T15:26:13.387045+0000","last_fullsized":"2023-09-23T15:26:13.387045+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":371,"reported_epoch":351,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.397913+0000","last_change":"2023-09-23T15:26:11.392818+0000","last_active":"2023-09-23T15:26:14.397913+0000","last_peered":"2023-09-23T15:26:14.397913+0000","last_clean":"2023-09-23T15:26:14.397913+0000","last_became_active":"2023-09-23T15:26:11.381709+0000","last_became_peered":"2023-09-23T15:26:11.381709+0000","last_unstale":"2023-09-23T15:26:14.397913+0000","last_undegraded":"2023-09-23T15:26:14.397913+0000","last_fullsized":"2023-09-23T15:26:14.397913+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:25:17.533999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":75,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380084+0000","last_change":"2023-09-23T15:26:11.381262+0000","last_active":"2023-09-23T15:26:13.380084+0000","last_peered":"2023-09-23T15:26:13.380084+0000","last_clean":"2023-09-23T15:26:13.380084+0000","last_became_active":"2023-09-23T15:26:11.381000+0000","last_became_peered":"2023-09-23T15:26:11.381000+0000","last_unstale":"2023-09-23T15:26:13.380084+0000","last_undegraded":"2023-09-23T15:26:13.380084+0000","last_fullsized":"2023-09-23T15:26:13.380084+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:44.385695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":343,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.396348+0000","last_change":"2023-09-23T15:26:11.382012+0000","last_active":"2023-09-23T15:26:13.396348+0000","last_peered":"2023-09-23T15:26:13.396348+0000","last_clean":"2023-09-23T15:26:13.396348+0000","last_became_active":"2023-09-23T15:26:11.381015+0000","last_became_peered":"2023-09-23T15:26:11.381015+0000","last_unstale":"2023-09-23T15:26:13.396348+0000","last_undegraded":"2023-09-23T15:26:13.396348+0000","last_fullsized":"2023-09-23T15:26:13.396348+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:23.252585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":355,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.384868+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:26:13.384868+0000","last_peered":"2023-09-23T15:26:13.384868+0000","last_clean":"2023-09-23T15:26:13.384868+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:26:13.384868+0000","last_undegraded":"2023-09-23T15:26:13.384868+0000","last_fullsized":"2023-09-23T15:26:13.384868+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":378,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380426+0000","last_change":"2023-09-23T15:26:11.390603+0000","last_active":"2023-09-23T15:26:13.380426+0000","last_peered":"2023-09-23T15:26:13.380426+0000","last_clean":"2023-09-23T15:26:13.380426+0000","last_became_active":"2023-09-23T15:26:11.380210+0000","last_became_peered":"2023-09-23T15:26:11.380210+0000","last_unstale":"2023-09-23T15:26:13.380426+0000","last_undegraded":"2023-09-23T15:26:13.380426+0000","last_fullsized":"2023-09-23T15:26:13.380426+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:11.388278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.400925+0000","last_change":"2023-09-23T15:26:11.374560+0000","last_active":"2023-09-23T15:26:13.400925+0000","last_peered":"2023-09-23T15:26:13.400925+0000","last_clean":"2023-09-23T15:26:13.400925+0000","last_became_active":"2023-09-23T15:26:11.374162+0000","last_became_peered":"2023-09-23T15:26:11.374162+0000","last_unstale":"2023-09-23T15:26:13.400925+0000","last_undegraded":"2023-09-23T15:26:13.400925+0000","last_fullsized":"2023-09-23T15:26:13.400925+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:14.653081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":358,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.380435+0000","last_change":"2023-09-23T15:26:11.398686+0000","last_active":"2023-09-23T15:26:13.380435+0000","last_peered":"2023-09-23T15:26:13.380435+0000","last_clean":"2023-09-23T15:26:13.380435+0000","last_became_active":"2023-09-23T15:26:11.384966+0000","last_became_peered":"2023-09-23T15:26:11.384966+0000","last_unstale":"2023-09-23T15:26:13.380435+0000","last_undegraded":"2023-09-23T15:26:13.380435+0000","last_fullsized":"2023-09-23T15:26:13.380435+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:18.728025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:14.291307+0000","last_change":"2023-09-23T15:26:11.364508+0000","last_active":"2023-09-23T15:26:14.291307+0000","last_peered":"2023-09-23T15:26:14.291307+0000","last_clean":"2023-09-23T15:26:14.291307+0000","last_became_active":"2023-09-23T15:26:11.364120+0000","last_became_peered":"2023-09-23T15:26:11.364120+0000","last_unstale":"2023-09-23T15:26:14.291307+0000","last_undegraded":"2023-09-23T15:26:14.291307+0000","last_fullsized":"2023-09-23T15:26:14.291307+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:11.688978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":30,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.386062+0000","last_change":"2023-09-23T15:25:42.524583+0000","last_active":"2023-09-23T15:26:13.386062+0000","last_peered":"2023-09-23T15:26:13.386062+0000","last_clean":"2023-09-23T15:26:13.386062+0000","last_became_active":"2023-09-23T15:25:42.524186+0000","last_became_peered":"2023-09-23T15:25:42.524186+0000","last_unstale":"2023-09-23T15:26:13.386062+0000","last_undegraded":"2023-09-23T15:26:13.386062+0000","last_fullsized":"2023-09-23T15:26:13.386062+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:16.880947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.382596+0000","last_change":"2023-09-23T15:25:42.530670+0000","last_active":"2023-09-23T15:26:13.382596+0000","last_peered":"2023-09-23T15:26:13.382596+0000","last_clean":"2023-09-23T15:26:13.382596+0000","last_became_active":"2023-09-23T15:25:42.529030+0000","last_became_peered":"2023-09-23T15:25:42.529030+0000","last_unstale":"2023-09-23T15:26:13.382596+0000","last_undegraded":"2023-09-23T15:26:13.382596+0000","last_fullsized":"2023-09-23T15:26:13.382596+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:17.984611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":304,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.398935+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:26:13.398935+0000","last_peered":"2023-09-23T15:26:13.398935+0000","last_clean":"2023-09-23T15:26:13.398935+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:26:13.398935+0000","last_undegraded":"2023-09-23T15:26:13.398935+0000","last_fullsized":"2023-09-23T15:26:13.398935+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":93,"reported_epoch":350,"state":"active+clean","last_fresh":"2023-09-23T15:26:13.385940+0000","last_change":"2023-09-23T15:26:11.368721+0000","last_active":"2023-09-23T15:26:13.385940+0000","last_peered":"2023-09-23T15:26:13.385940+0000","last_clean":"2023-09-23T15:26:13.385940+0000","last_became_active":"2023-09-23T15:26:11.368413+0000","last_became_peered":"2023-09-23T15:26:11.368413+0000","last_unstale":"2023-09-23T15:26:13.385940+0000","last_undegraded":"2023-09-23T15:26:13.385940+0000","last_fullsized":"2023-09-23T15:26:13.385940+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:00.765945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":10068,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":3513384960,"num_objects":53610,"num_object_clones":0,"num_object_copies":160830,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5849,"num_objects_misplaced":2457,"num_objects_unfound":0,"num_objects_dirty":53610,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":53610,"num_write_kb":3431040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":359,"num_bytes_recovered":23527424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10594549760,"data_stored":10594549760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":53610,"ondisk_log_size":53610,"up":48,"acting":45,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":130,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201659,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2208244,"kb_used_data":2029036,"kb_used_omap":0,"kb_used_meta":179200,"kb_avail":91536908,"statfs":{"total":95995035648,"available":93733793792,"internally_reserved":0,"allocated":2077732864,"data_stored":2076643608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":183500800},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:25 2023","interfaces":[{"interface":"back","average":{"1min":3.96,"5min":7.3369999999999997,"15min":3.9489999999999998},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":35.908000000000001,"5min":226.226,"15min":226.226},"last":0.33400000000000002},{"interface":"front","average":{"1min":5.8099999999999996,"5min":8.8290000000000006,"15min":4.6070000000000002},"min":{"1min":0.254,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":35.939999999999998,"5min":226.619,"15min":226.619},"last":1.51}]},{"osd":1,"last update":"Sat Sep 23 15:26:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":5.319,"15min":3.085},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":56.746000000000002,"5min":311.16699999999997,"15min":311.16699999999997},"last":2.391},{"interface":"front","average":{"1min":3.3029999999999999,"5min":5.5250000000000004,"15min":3.5470000000000002},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":56.899999999999999,"5min":311.279,"15min":311.279},"last":0.57599999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:25:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4580000000000002,"5min":1.9930000000000001,"15min":2.7770000000000001},"min":{"1min":0.313,"5min":0.28799999999999998,"15min":0.24099999999999999},"max":{"1min":8.8219999999999992,"5min":8.8219999999999992,"15min":101.807},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.994,"5min":2.2210000000000001,"15min":3.331},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.308999999999999,"5min":10.308999999999999,"15min":101.84099999999999},"last":0.63900000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:26:04 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.1949999999999998,"15min":3.8220000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28899999999999998},"max":{"1min":5.4710000000000001,"5min":41.588999999999999,"15min":146.166},"last":2.915},{"interface":"front","average":{"1min":1.097,"5min":2.6509999999999998,"15min":3.9820000000000002},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":5.7400000000000002,"5min":41.542000000000002,"15min":146.18299999999999},"last":2.9009999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:25:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.855,"15min":1.95},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.867,"5min":16.771999999999998,"15min":120.587},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.413,"5min":1.7270000000000001,"15min":2.0299999999999998},"min":{"1min":0.311,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.0649999999999995,"5min":18.023,"15min":121.056},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:25:48 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":5.2850000000000001,"15min":3.2469999999999999},"min":{"1min":0.36599999999999999,"5min":0.29099999999999998,"15min":0.28999999999999998},"max":{"1min":4.2809999999999997,"5min":224.441,"15min":224.441},"last":0.747},{"interface":"front","average":{"1min":1.3959999999999999,"5min":5.9160000000000004,"15min":3.4790000000000001},"min":{"1min":0.317,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.6539999999999999,"5min":224.28700000000001,"15min":224.28700000000001},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:25:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":2.3290000000000002,"15min":2.2639999999999998},"min":{"1min":0.34799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.358000000000001,"5min":10.358000000000001,"15min":10.358000000000001},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.7609999999999999,"5min":2.472,"15min":2.5},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.7050000000000001,"5min":10.336,"15min":10.336},"last":0.58799999999999997}]}]},{"osd":6,"up_from":304,"seq":1305670058032,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":767472,"kb_used_data":656104,"kb_used_omap":13,"kb_used_meta":111346,"kb_avail":92977680,"statfs":{"total":95995035648,"available":95209144320,"internally_reserved":0,"allocated":671850496,"data_stored":670765243,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":114018659},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":4.7229999999999999,"15min":2.5609999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":46.740000000000002,"5min":184.53200000000001,"15min":184.53200000000001},"last":0.28499999999999998},{"interface":"front","average":{"1min":3.5760000000000001,"5min":4.149,"15min":3.3010000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":46.549999999999997,"5min":86.227999999999994,"15min":86.227999999999994},"last":0.98399999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":3.7650000000000001,"15min":3.0830000000000002},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4470000000000001,"5min":86.534000000000006,"15min":86.534000000000006},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.625,"15min":3.1160000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.4770000000000003,"5min":86.144000000000005,"15min":86.144000000000005},"last":1.391}]},{"osd":2,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.6599999999999999,"15min":1.5700000000000001},"min":{"1min":0.38400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7999999999999998,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":1.6200000000000001},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.8120000000000001,"15min":1.7549999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":2.2330000000000001,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":4.7539999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":2.468,"15min":2.3420000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.6340000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.9690000000000001,"5min":2.7879999999999998,"15min":2.444},"min":{"1min":0.41299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.266,"5min":41.408000000000001,"15min":41.408000000000001},"last":1.0229999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.137,"15min":1.0049999999999999},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.5880000000000001,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.008,"5min":1.845,"15min":1.829},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.8289999999999997,"5min":13.627000000000001,"15min":13.627000000000001},"last":1.0620000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.1030000000000002,"15min":1.845},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.1970000000000001,"5min":13.74,"15min":13.74},"last":4.8940000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.663,"15min":1.5429999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.8069999999999999,"5min":15.868,"15min":15.868},"last":0.90900000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.766,"15min":1.522},"min":{"1min":0.47099999999999997,"5min":0.373,"15min":0.373},"max":{"1min":6.7210000000000001,"5min":12.023999999999999,"15min":12.023999999999999},"last":5.0039999999999996},{"interface":"front","average":{"1min":1.224,"5min":2.0430000000000001,"15min":1.8959999999999999},"min":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.8040000000000003,"5min":12.641999999999999,"15min":12.641999999999999},"last":4.8639999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015332,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2454552,"kb_used_data":1957840,"kb_used_omap":6,"kb_used_meta":496697,"kb_avail":91290600,"statfs":{"total":95995035648,"available":93481574400,"internally_reserved":0,"allocated":2004828160,"data_stored":2003138691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":508617803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.6389999999999998,"15min":3.105},"min":{"1min":0.20100000000000001,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":2.3159999999999998,"5min":60.817999999999998,"15min":60.817999999999998},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.82899999999999996,"5min":3.5600000000000001,"15min":3.8479999999999999},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.21099999999999999},"max":{"1min":3.3519999999999999,"5min":63.594999999999999,"15min":63.594999999999999},"last":0.85899999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.3109999999999999,"15min":3.8500000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.5209999999999999,"5min":12.429,"15min":109.375},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.913,"15min":3.2189999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":2.2839999999999998,"5min":12.259,"15min":105.16},"last":0.42199999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.6619999999999999,"15min":6.4939999999999998},"min":{"1min":0.33300000000000002,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":3.0619999999999998,"5min":64.540999999999997,"15min":211.405},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.155,"5min":2.9569999999999999,"15min":6.9320000000000004},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.246},"max":{"1min":3.3980000000000001,"5min":67.465000000000003,"15min":210.988},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.1139999999999999,"15min":3.2639999999999998},"min":{"1min":0.34399999999999997,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":4.0519999999999996,"5min":13.379,"15min":80.570999999999998},"last":4.0519999999999996},{"interface":"front","average":{"1min":0.75800000000000001,"5min":2.0880000000000001,"15min":3.2210000000000001},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.308},"max":{"1min":2.1190000000000002,"5min":12.279999999999999,"15min":80.75},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":3.8500000000000001,"15min":3.2730000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.27400000000000002},"max":{"1min":5.29,"5min":115.401,"15min":115.401},"last":5.29},{"interface":"front","average":{"1min":1.173,"5min":3.7599999999999998,"15min":3.319},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.4269999999999996,"5min":115.169,"15min":115.169},"last":3.2930000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":3.149,"5min":3.351,"15min":3.1640000000000001},"min":{"1min":0.46300000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":9.3729999999999993,"5min":19.553999999999998,"15min":19.553999999999998},"last":3.3180000000000001},{"interface":"front","average":{"1min":2.5369999999999999,"5min":2.323,"15min":2.3460000000000001},"min":{"1min":0.42099999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":15.605,"5min":15.605,"15min":15.605},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":2.7240000000000002,"15min":2.8410000000000002},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.73,"5min":19.702999999999999,"15min":19.702999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.3719999999999999,"15min":2.4889999999999999},"min":{"1min":0.47799999999999998,"5min":0.377,"15min":0.377},"max":{"1min":2.157,"5min":20.722000000000001,"15min":20.722000000000001},"last":1.9910000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893382,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1789040,"kb_used_data":1166312,"kb_used_omap":8,"kb_used_meta":622711,"kb_avail":91956112,"statfs":{"total":95995035648,"available":94163058688,"internally_reserved":0,"allocated":1194303488,"data_stored":1192765681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":637656660},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,5,6],"upper_bound":256},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:26:05 2023","interfaces":[{"interface":"back","average":{"1min":8.2409999999999997,"5min":4.9989999999999997,"15min":4.1130000000000004},"min":{"1min":0.28100000000000003,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":110.3,"5min":110.3,"15min":110.3},"last":4.8449999999999998},{"interface":"front","average":{"1min":8.2119999999999997,"5min":4.806,"15min":3.915},"min":{"1min":0.28699999999999998,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":110.35599999999999,"5min":110.35599999999999,"15min":110.35599999999999},"last":3.3210000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.96,"5min":2.2999999999999998,"15min":4.3609999999999998},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.17000000000000001},"max":{"1min":7.6840000000000002,"5min":30.510999999999999,"15min":122.20099999999999},"last":3.6539999999999999},{"interface":"front","average":{"1min":2.1030000000000002,"5min":2.5070000000000001,"15min":4.29},"min":{"1min":0.22600000000000001,"5min":0.20499999999999999,"15min":0.18099999999999999},"max":{"1min":10.874000000000001,"5min":30.396999999999998,"15min":122.071},"last":4.2359999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:25:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":4.4379999999999997,"15min":5.8899999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.20499999999999999},"max":{"1min":8.5399999999999991,"5min":197.78899999999999,"15min":197.78899999999999},"last":4.5469999999999997},{"interface":"front","average":{"1min":1.597,"5min":4.4870000000000001,"15min":6.3410000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":10.923,"5min":200.58799999999999,"15min":200.58799999999999},"last":4.7850000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":2.1579999999999999,"15min":2.2240000000000002},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.28000000000000003},"max":{"1min":6.9409999999999998,"5min":15.358000000000001,"15min":31.341999999999999},"last":1.5600000000000001},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.97,"15min":2.331},"min":{"1min":0.33100000000000002,"5min":0.32900000000000001,"15min":0.27000000000000002},"max":{"1min":8.5830000000000002,"5min":9.8249999999999993,"15min":99.168999999999997},"last":3.5339999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.0550000000000002,"5min":2.2170000000000001,"15min":2.4790000000000001},"min":{"1min":0.313,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":8.0850000000000009,"5min":10.768000000000001,"15min":109.369},"last":4.7519999999999998},{"interface":"front","average":{"1min":1.7490000000000001,"5min":2.4199999999999999,"15min":2.5259999999999998},"min":{"1min":0.433,"5min":0.35299999999999998,"15min":0.312},"max":{"1min":7.4260000000000002,"5min":15.436999999999999,"15min":109.387},"last":1.512}]},{"osd":6,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.125,"5min":2.75,"15min":2.79},"min":{"1min":0.45600000000000002,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":7.0229999999999997,"5min":11.989000000000001,"15min":11.989000000000001},"last":1.6160000000000001},{"interface":"front","average":{"1min":2.0939999999999999,"5min":3.1280000000000001,"15min":3.2829999999999999},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.5869999999999997,"5min":17.385000000000002,"15min":17.385000000000002},"last":3.581}]},{"osd":7,"last update":"Sat Sep 23 15:25:30 2023","interfaces":[{"interface":"back","average":{"1min":2.5310000000000001,"5min":2.5299999999999998,"15min":2.6539999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.36799999999999999},"max":{"1min":8.1310000000000002,"5min":17.712,"15min":99.549999999999997},"last":9.2550000000000008},{"interface":"front","average":{"1min":1.752,"5min":2.5449999999999999,"15min":2.2440000000000002},"min":{"1min":0.46000000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":6.9880000000000004,"5min":15.407999999999999,"15min":31.225999999999999},"last":1.591}]}]},{"osd":2,"up_from":105,"seq":450971566329,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1455104,"kb_used_data":836152,"kb_used_omap":11,"kb_used_meta":618932,"kb_avail":92290048,"statfs":{"total":95995035648,"available":94505009152,"internally_reserved":0,"allocated":856219648,"data_stored":854670111,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":633787152},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":4.3209999999999997,"5min":7.2210000000000001,"15min":5.1600000000000001},"min":{"1min":0.21199999999999999,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":59.308999999999997,"5min":88.622,"15min":88.622},"last":0.28499999999999998},{"interface":"front","average":{"1min":4.6639999999999997,"5min":7.4119999999999999,"15min":4.8620000000000001},"min":{"1min":0.26900000000000002,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":56.017000000000003,"5min":88.649000000000001,"15min":88.649000000000001},"last":0.26500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":3.6059999999999999,"15min":4.3109999999999999},"min":{"1min":0.32200000000000001,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":5.5300000000000002,"5min":71.819000000000003,"15min":84.641000000000005},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.8149999999999999,"15min":3.5750000000000002},"min":{"1min":0.24399999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.96,"5min":71.805000000000007,"15min":84.751999999999995},"last":5.1280000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6559999999999997,"15min":5.4500000000000002},"min":{"1min":0.314,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":48.527000000000001,"5min":77.031999999999996,"15min":87.108000000000004},"last":0.34899999999999998},{"interface":"front","average":{"1min":3.7650000000000001,"5min":4.9699999999999998,"15min":5.532},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.20300000000000001},"max":{"1min":48.371000000000002,"5min":76.742000000000004,"15min":86.158000000000001},"last":2.7040000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.6799999999999999,"5min":1.6379999999999999,"15min":2.4220000000000002},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":10.003,"5min":16.712,"15min":84.608000000000004},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.357,"5min":1.5800000000000001,"15min":2.9049999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":8.0519999999999996,"5min":8.0519999999999996,"15min":87.019999999999996},"last":0.76000000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":1.8340000000000001,"5min":1.8280000000000001,"15min":2.6200000000000001},"min":{"1min":0.33500000000000002,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":6.0140000000000002,"5min":7.6980000000000004,"15min":84.498000000000005},"last":2.6600000000000001},{"interface":"front","average":{"1min":2.1560000000000001,"5min":1.845,"15min":2.5259999999999998},"min":{"1min":0.374,"5min":0.317,"15min":0.30199999999999999},"max":{"1min":10.052,"5min":10.808999999999999,"15min":84.664000000000001},"last":0.318}]},{"osd":6,"last update":"Sat Sep 23 15:25:29 2023","interfaces":[{"interface":"back","average":{"1min":1.855,"5min":2.2429999999999999,"15min":2.27},"min":{"1min":0.39600000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.3109999999999999,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.1339999999999999,"5min":1.651,"15min":1.917},"min":{"1min":0.42199999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":6.4950000000000001,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.81100000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:25:22 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":2.0659999999999998,"15min":2.5579999999999998},"min":{"1min":0.42599999999999999,"5min":0.34999999999999998,"15min":0.31900000000000001},"max":{"1min":11.199999999999999,"5min":11.199999999999999,"15min":84.596000000000004},"last":0.61299999999999999},{"interface":"front","average":{"1min":2.6059999999999999,"5min":2.0960000000000001,"15min":2.5489999999999999},"min":{"1min":0.40699999999999997,"5min":0.38200000000000001,"15min":0.34200000000000003},"max":{"1min":10.028,"5min":16.074999999999999,"15min":84.783000000000001},"last":0.48699999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402803,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2404156,"kb_used_data":1753716,"kb_used_omap":3,"kb_used_meta":650428,"kb_avail":91340996,"statfs":{"total":95995035648,"available":93533179904,"internally_reserved":0,"allocated":1795805184,"data_stored":1794255902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":666038441},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":3.375,"15min":3.359},"min":{"1min":0.23200000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.557,"5min":108.318,"15min":108.318},"last":7.5670000000000002},{"interface":"front","average":{"1min":1.9350000000000001,"5min":3.798,"15min":3.6349999999999998},"min":{"1min":0.26900000000000002,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":13.445,"5min":108.889,"15min":108.889},"last":8.2669999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:26:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":3.6640000000000001,"15min":4.5750000000000002},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.952,"5min":126.875,"15min":126.875},"last":4.2830000000000004},{"interface":"front","average":{"1min":1.0169999999999999,"5min":4.2939999999999996,"15min":4.798},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.22900000000000001},"max":{"1min":4.0990000000000002,"5min":128.249,"15min":128.249},"last":7.6820000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":2.1499999999999999,"15min":3.2509999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.22},"max":{"1min":8.8460000000000001,"5min":15.048999999999999,"15min":63.009999999999998},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.5979999999999999,"15min":4.1230000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":13.047000000000001,"5min":36.582000000000001,"15min":119.449},"last":8.2439999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.7190000000000001,"15min":2.2250000000000001},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":4.899,"5min":36.551000000000002,"15min":54.051000000000002},"last":8.2170000000000005},{"interface":"front","average":{"1min":1.403,"5min":1.7749999999999999,"15min":2.141},"min":{"1min":0.36699999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0659999999999998,"5min":13.115,"15min":52.228999999999999},"last":0.40200000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":3.4670000000000001,"15min":2.7519999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":4.9450000000000003,"5min":148.619,"15min":148.619},"last":8.1530000000000005},{"interface":"front","average":{"1min":1.3009999999999999,"5min":3.2749999999999999,"15min":2.6339999999999999},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.1829999999999998,"5min":148.56999999999999,"15min":148.56999999999999},"last":0.748}]},{"osd":6,"last update":"Sat Sep 23 15:25:29 2023","interfaces":[{"interface":"back","average":{"1min":2.7690000000000001,"5min":2.335,"15min":2.2610000000000001},"min":{"1min":0.42799999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":13.079000000000001,"5min":13.079000000000001,"15min":13.079000000000001},"last":0.432},{"interface":"front","average":{"1min":2.0179999999999998,"5min":3.0030000000000001,"15min":3.4350000000000001},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":8.4269999999999996,"5min":36.517000000000003,"15min":36.517000000000003},"last":8.1899999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":3.1520000000000001,"15min":2.6179999999999999},"min":{"1min":0.38500000000000001,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":5.7329999999999997,"5min":145.25,"15min":145.25},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.89500000000000002,"5min":3.1150000000000002,"15min":2.5070000000000001},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34799999999999998},"max":{"1min":3.9769999999999999,"5min":145.00899999999999,"15min":145.00899999999999},"last":0.47499999999999998}]}]},{"osd":4,"up_from":0,"seq":133143986568,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724931,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377060,"kb_used_data":1955804,"kb_used_omap":0,"kb_used_meta":421248,"kb_avail":91368092,"statfs":{"total":95995035648,"available":93560926208,"internally_reserved":0,"allocated":2002743296,"data_stored":2001666591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":431357952},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:21 2023","interfaces":[{"interface":"back","average":{"1min":3.1110000000000002,"5min":3.2570000000000001,"15min":3.4380000000000002},"min":{"1min":0.23799999999999999,"5min":0.217,"15min":0.20599999999999999},"max":{"1min":16.411999999999999,"5min":54.939999999999998,"15min":124.31},"last":50.460000000000001},{"interface":"front","average":{"1min":2.6440000000000001,"5min":3.2629999999999999,"15min":3.4279999999999999},"min":{"1min":0.24299999999999999,"5min":0.24099999999999999,"15min":0.23300000000000001},"max":{"1min":16.312999999999999,"5min":55.043999999999997,"15min":124.374},"last":50.357999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":3.5960000000000001,"5min":4.165,"15min":3.4089999999999998},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":50.444000000000003,"5min":86.900999999999996,"15min":124.38800000000001},"last":0.32500000000000001},{"interface":"front","average":{"1min":3.5470000000000002,"5min":2.9060000000000001,"15min":3.2010000000000001},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":49.253,"5min":49.253,"15min":124.29000000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:25:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0419999999999998,"5min":2.6789999999999998,"15min":3.6890000000000001},"min":{"1min":0.29699999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":5.0380000000000003,"5min":74.019000000000005,"15min":193.00999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":3.3450000000000002,"5min":2.9209999999999998,"15min":3.9009999999999998},"min":{"1min":0.316,"5min":0.29299999999999998,"15min":0.26800000000000002},"max":{"1min":9.125,"5min":73.712000000000003,"15min":193.15700000000001},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 15:25:51 2023","interfaces":[{"interface":"back","average":{"1min":6.3680000000000003,"5min":6.2699999999999996,"15min":6.3899999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":86.206999999999994,"5min":266.70299999999997,"15min":266.70299999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":6.0259999999999998,"5min":7.0590000000000002,"15min":6.6239999999999997},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":86.266999999999996,"5min":266.80399999999997,"15min":266.80399999999997},"last":0.57699999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.732,"15min":1.869},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":1.5089999999999999,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.7490000000000001,"15min":1.944},"min":{"1min":0.38100000000000001,"5min":0.29599999999999999,"15min":0.26000000000000001},"max":{"1min":3.202,"5min":10.516,"15min":138.49600000000001},"last":0.52900000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:25:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":2.3220000000000001,"15min":2.3719999999999999},"min":{"1min":0.35399999999999998,"5min":0.308,"15min":0.308},"max":{"1min":8.6989999999999998,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.56200000000000006},{"interface":"front","average":{"1min":1.278,"5min":2.4060000000000001,"15min":2.4079999999999999},"min":{"1min":0.36099999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":5.0069999999999997,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.48599999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:25:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.996,"15min":1.3009999999999999},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":8.3040000000000003,"5min":15.849,"15min":15.849},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.7130000000000001,"15min":1.2410000000000001},"min":{"1min":0.34999999999999998,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":2.2989999999999999,"5min":16.134,"15min":16.134},"last":0.61699999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":995164160,"data_stored":995164160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1904803840,"data_stored":1904803840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":661520384,"data_stored":661520384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1800732672,"data_stored":1800732672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":567869440,"data_stored":567869440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2008285184,"data_stored":2008285184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":672661504,"data_stored":672661504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1983512576,"data_stored":1983512576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:26:23.097 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T15:26:23.134 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:23.223 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:23.386 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:23.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:23.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:23.449 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:23.550 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:23.629 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:23.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:23 smithi017 ceph-mon[222878]: from='client.46193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:23.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:23 smithi017 ceph-mon[215646]: from='client.46193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:23.707 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:23.709 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:23.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:23.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:23.813 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:23.847 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:23 smithi027 ceph-mon[195233]: from='client.46193 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:23.852 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:23.976 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:23.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:23.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:23.997 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 57931 57915 92.7547 61.5625 0.0121118 0.0107749 2023-09-23T15:26:24.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:24.129 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:24.523 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:24 smithi027 ceph-mon[195233]: pgmap v1041: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/163644 objects degraded (3.574%); 2457/163644 objects misplaced (1.501%); 2.7 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T15:26:24.523 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:24.691 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:24.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:24.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:24.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:24 smithi017 ceph-mon[215646]: pgmap v1041: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/163644 objects degraded (3.574%); 2457/163644 objects misplaced (1.501%); 2.7 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T15:26:24.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:24.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:24 smithi017 ceph-mon[222878]: pgmap v1041: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.3 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/163644 objects degraded (3.574%); 2457/163644 objects misplaced (1.501%); 2.7 MiB/s, 1 keys/s, 46 objects/s recovering 2023-09-23T15:26:24.877 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:24.887 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:24.887 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:24.992 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:24.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:24.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:24.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:26:24.998982+0000 min lat: 0.00111339 max lat: 2.1508 avg lat: 0.0108498 2023-09-23T15:26:24.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:26:24.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 58994 58978 92.0942 66.4375 0.0241078 0.0108498 2023-09-23T15:26:25.100 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:25.143 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:25.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:26:25.223 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:25.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:25.331 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:25.461 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:25.632 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:25.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:25.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:25.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:25.804 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:25.905 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:25.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:25.998 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 59705 59689 90.9323 44.4375 0.00290418 0.0109669 2023-09-23T15:26:26.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:26.053 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:26.156 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:26.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:26.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:26.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:26.265 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:26.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:26.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:26.347 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:26.383 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:26.415 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:26:26.415 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:26:26.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.45479.0:126248 7.9 e250/223 7:91f1ba0e:::benchmark_data_smithi017_422824_object32378:head v 250'7794, mlcod=250'7792)", 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.771734+0000", 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 434.63632032999999, 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30521936799999999, 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.771734+0000", 2023-09-23T15:26:26.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.771734+0000", 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.771734+0000", 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.771739+0000", 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7610000000000004e-06 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067341+0000", 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.295601744 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067356+0000", 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4596000000000001e-05 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067476+0000", 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012045299999999999 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067500+0000", 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3887e-05 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.076917+0000", 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094173009999999995 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.076954+0000", 2023-09-23T15:26:26.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6245999999999998e-05 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45479.0:126259 7.1 7:80e682e3:::benchmark_data_smithi017_422824_object32389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:19:11.780713+0000", 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 434.62734148099997, 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.304055205, 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.422 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45479", 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 126259 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780713+0000", 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780713+0000", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780712+0000", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:11.780717+0000", 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.305e-06 2023-09-23T15:26:26.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067607+0000", 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28688945399999999 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.067616+0000", 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9779999999999994e-06 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.071049+0000", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034332109999999998 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.071215+0000", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016636900000000001 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.071343+0000", 2023-09-23T15:26:26.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127842 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.076914+0000", 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055707229999999996 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.076966+0000", 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1928000000000002e-05 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.084634+0000", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007667831 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.084674+0000", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0528999999999998e-05 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:19:12.084768+0000", 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4215000000000005e-05 2023-09-23T15:26:26.428 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50596 8.2 8.a11b4e02 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 270.19654544600002, 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91699152100000003, 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.429 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50596 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211508+0000", 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211609+0000", 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101167 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211623+0000", 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3767e-05 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211630+0000", 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3939999999999998e-06 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211691+0000", 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1008000000000003e-05 2023-09-23T15:26:26.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211700+0000", 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3070000000000008e-06 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128474+0000", 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91677326699999995 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.433 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128501+0000", 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7019999999999999e-05 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50597 8.2 8.538bc622 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 270.19641245899999, 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91703620299999999, 2023-09-23T15:26:26.434 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50597 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211687+0000", 2023-09-23T15:26:26.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5200000000000001e-05 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211692+0000", 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.541e-06 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211696+0000", 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.489e-06 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211726+0000", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9879999999999999e-05 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211731+0000", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9139999999999999e-06 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128668+0000", 2023-09-23T15:26:26.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91693728100000005 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128678+0000", 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0438999999999999e-05 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87479 8.2 8:4a52ba5c:::benchmark_data_smithi017_485823_object87478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.439 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.872844601, 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99550063799999999, 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87479 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535209+0000", 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:26.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535289+0000", 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0311999999999999e-05 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535304+0000", 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4995e-05 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535312+0000", 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3559999999999993e-06 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535462+0000", 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015008400000000001 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535476+0000", 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3451e-05 2023-09-23T15:26:26.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.512985+0000", 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97750865600000003 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.513085+0000", 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010079800000000001 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.513208+0000", 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012306200000000001 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520104+0000", 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068957640000000004 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520176+0000", 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1971999999999995e-05 2023-09-23T15:26:26.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530643+0000", 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010466824 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530681+0000", 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8059000000000003e-05 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530711+0000", 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9646999999999999e-05 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87489 8.2 8:48f7c14f:::benchmark_data_smithi017_485823_object87488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.849520439, 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97232053399999996, 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.447 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87489 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558532+0000", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558643+0000", 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011113200000000001 2023-09-23T15:26:26.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558657+0000", 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3875e-05 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558665+0000", 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4529999999999996e-06 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558745+0000", 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9914999999999996e-05 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558758+0000", 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3682e-05 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515668+0000", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.956909644 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515798+0000", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130142 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515906+0000", 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108136 2023-09-23T15:26:26.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520204+0000", 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042975110000000004 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520266+0000", 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2539000000000006e-05 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530807+0000", 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010540546 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530834+0000", 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7095000000000001e-05 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530855+0000", 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0899000000000001e-05 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87525 8.2 8:406bbda9:::benchmark_data_smithi017_485823_object87524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:26:26.455 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.77683945300001, 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89979054000000003, 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87525 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:26:26.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631213+0000", 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631296+0000", 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3195999999999994e-05 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631314+0000", 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7529000000000001e-05 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631322+0000", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.011e-06 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631390+0000", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7599000000000001e-05 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631404+0000", 2023-09-23T15:26:26.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4299e-05 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518240+0000", 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88683647899999996 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518345+0000", 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010480399999999999 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518447+0000", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101828 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525282+0000", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068350889999999999 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530609+0000", 2023-09-23T15:26:26.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053267449999999999 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530947+0000", 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033794000000000001 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530978+0000", 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1445999999999997e-05 2023-09-23T15:26:26.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531006+0000", 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7543999999999999e-05 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87536 8.2 8:4ea85f1f:::benchmark_data_smithi017_485823_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:26:26.463 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.76656188600001, 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.889614613, 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87536 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641491+0000", 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641551+0000", 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0201000000000002e-05 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641570+0000", 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8638e-05 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641577+0000", 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.114e-06 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641632+0000", 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4963999999999998e-05 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641642+0000", 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0412e-05 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520296+0000", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87865352100000005 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520391+0000", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4883999999999995e-05 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520486+0000", 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5741000000000004e-05 2023-09-23T15:26:26.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.527976+0000", 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074899900000000002 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530771+0000", 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002794487 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531039+0000", 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000268165 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531071+0000", 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1652999999999997e-05 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531107+0000", 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6788000000000001e-05 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87554 8.2 8:45ba82f7:::benchmark_data_smithi017_485823_object87553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.72411097899999, 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.84732607400000004, 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.471 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87554 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683930+0000", 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999866 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683994+0000", 2023-09-23T15:26:26.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3552000000000004e-05 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684021+0000", 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6920999999999999e-05 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684030+0000", 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.431e-06 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684096+0000", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5705999999999997e-05 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684105+0000", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2650000000000002e-06 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522506+0000", 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83840049699999997 2023-09-23T15:26:26.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522650+0000", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014449800000000001 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522832+0000", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018217 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530617+0000", 2023-09-23T15:26:26.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077846809999999999 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530890+0000", 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027289300000000002 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531180+0000", 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028948599999999999 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531231+0000", 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1178999999999999e-05 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531270+0000", 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9011000000000003e-05 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87579 8.2 8:4fb1f62c:::benchmark_data_smithi017_485823_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.68564144600001, 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81508181099999999, 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87579 2023-09-23T15:26:26.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722411+0000", 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722470+0000", 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8721999999999998e-05 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722488+0000", 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8365000000000001e-05 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722497+0000", 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2730000000000002e-06 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722573+0000", 2023-09-23T15:26:26.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6279000000000006e-05 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722581+0000", 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8970000000000008e-06 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525313+0000", 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80273269199999997 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525443+0000", 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129498 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525551+0000", 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108167 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530778+0000", 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052264729999999997 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531338+0000", 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00056077399999999999 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537387+0000", 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006049034 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537448+0000", 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0513000000000002e-05 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537495+0000", 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7086000000000003e-05 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87580 8.2 8:43e296c2:::benchmark_data_smithi017_485823_object87579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.68143912100001, 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81100082500000004, 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.487 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87580 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726692+0000", 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7746999999999999e-05 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726706+0000", 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3783e-05 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726711+0000", 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8690000000000003e-06 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726755+0000", 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4307000000000001e-05 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726765+0000", 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4509999999999998e-06 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528023+0000", 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80125852099999995 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528145+0000", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121804 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528294+0000", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149209 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530913+0000", 2023-09-23T15:26:26.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026183740000000001 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531392+0000", 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000479421 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537542+0000", 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061496830000000004 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537587+0000", 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5498999999999999e-05 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537616+0000", 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9082e-05 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87589 8.2 8:42faca38:::benchmark_data_smithi017_485823_object87588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.668425609, 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80387859699999997, 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.495 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87589 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739627+0000", 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739727+0000", 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9987000000000003e-05 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739743+0000", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6314999999999999e-05 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739750+0000", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4060000000000002e-06 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739825+0000", 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4869000000000004e-05 2023-09-23T15:26:26.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739839+0000", 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4426999999999999e-05 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531435+0000", 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79159639699999995 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531561+0000", 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012580999999999999 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531709+0000", 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014819400000000001 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537509+0000", 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005799762 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537642+0000", 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013332700000000001 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543446+0000", 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058038229999999996 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543479+0000", 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2761e-05 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543508+0000", 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8581999999999998e-05 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87592 8.2 8:47cf9f2f:::benchmark_data_smithi017_485823_object87591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.664421422, 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.82188067499999995, 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87592 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743627+0000", 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999933 2023-09-23T15:26:26.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743699+0000", 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2266000000000002e-05 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743718+0000", 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9366000000000001e-05 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743725+0000", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.17e-06 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743784+0000", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8798000000000002e-05 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743796+0000", 2023-09-23T15:26:26.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.131e-05 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.534749+0000", 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79095367599999999 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.534881+0000", 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131784 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.535016+0000", 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134943 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537647+0000", 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026313199999999999 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543417+0000", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057694920000000002 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565443+0000", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022026104000000001 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565484+0000", 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0667999999999998e-05 2023-09-23T15:26:26.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565514+0000", 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0358e-05 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87613 8.2 8:43730902:::benchmark_data_smithi017_485823_object87612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:26:26.510 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.618461938, 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77606009899999995, 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87613 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789591+0000", 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789673+0000", 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2378000000000003e-05 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789693+0000", 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9440999999999999e-05 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789701+0000", 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8909999999999995e-06 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789760+0000", 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9577999999999998e-05 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789775+0000", 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4209999999999999e-05 2023-09-23T15:26:26.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537670+0000", 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74789507200000005 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537762+0000", 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2693000000000002e-05 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537896+0000", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013394599999999999 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543526+0000", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056293139999999998 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552772+0000", 2023-09-23T15:26:26.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092463460000000008 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565600+0000", 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012828482 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565631+0000", 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0514999999999998e-05 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565653+0000", 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1815e-05 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87614 8.2 8:4007368f:::benchmark_data_smithi017_485823_object87613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.618339193, 2023-09-23T15:26:26.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77602658099999999, 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87614 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:26.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789774+0000", 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9268999999999998e-05 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789783+0000", 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6319999999999997e-06 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789787+0000", 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2769999999999999e-06 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789842+0000", 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5387999999999999e-05 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789856+0000", 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3838e-05 2023-09-23T15:26:26.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540554+0000", 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75069737299999995 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540682+0000", 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128467 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540827+0000", 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014544800000000001 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552719+0000", 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011891626000000001 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552805+0000", 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5820000000000002e-05 2023-09-23T15:26:26.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565690+0000", 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012885122000000001 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565719+0000", 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9071e-05 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565742+0000", 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2929e-05 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87629 8.2 8:4962ff39:::benchmark_data_smithi017_485823_object87628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.583359237, 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74115534500000002, 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87629 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824694+0000", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824745+0000", 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1140999999999997e-05 2023-09-23T15:26:26.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824760+0000", 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5500999999999999e-05 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824767+0000", 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1519999999999997e-06 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824832+0000", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5376999999999994e-05 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824845+0000", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2704999999999999e-05 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543560+0000", 2023-09-23T15:26:26.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.71871572399999994 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543668+0000", 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107358 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543760+0000", 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1947999999999994e-05 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552725+0000", 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089653790000000008 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565394+0000", 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012669249 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565785+0000", 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000390541 2023-09-23T15:26:26.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565813+0000", 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7770999999999999e-05 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565851+0000", 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8061e-05 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87637 8.2 8:403d4373:::benchmark_data_smithi017_485823_object87636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.56678420099999, 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.724673546, 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87637 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841268+0000", 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841309+0000", 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0831000000000003e-05 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841327+0000", 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7875000000000001e-05 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841334+0000", 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4340000000000003e-06 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841387+0000", 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2632e-05 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841397+0000", 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8209999999999997e-06 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546335+0000", 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70493799099999999 2023-09-23T15:26:26.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546500+0000", 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000165198 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546680+0000", 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017972499999999999 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565404+0000", 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018724034000000001 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565569+0000", 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016548299999999999 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565892+0000", 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032258200000000002 2023-09-23T15:26:26.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565923+0000", 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0905999999999999e-05 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565944+0000", 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1389000000000001e-05 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87654 8.2 8:4971efb7:::benchmark_data_smithi017_485823_object87653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.514868638, 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.68569409699999995, 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.542 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87654 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:26:26.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893184+0000", 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893221+0000", 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7132000000000003e-05 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893243+0000", 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1987000000000001e-05 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893250+0000", 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5619999999999998e-06 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893307+0000", 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7126999999999997e-05 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893318+0000", 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0847e-05 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549540+0000", 2023-09-23T15:26:26.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.65622212700000004 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549659+0000", 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011890199999999999 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549804+0000", 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014447 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565410+0000", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015606594 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.569942+0000", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045315420000000004 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578781+0000", 2023-09-23T15:26:26.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088392560000000002 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578836+0000", 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4809000000000003e-05 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578880+0000", 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4035000000000001e-05 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.549 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87661 8.2 8:4515bd6f:::benchmark_data_smithi017_485823_object87660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.49055345799999, 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66148114999999996, 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87661 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917500+0000", 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917552+0000", 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1702000000000001e-05 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917568+0000", 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6622000000000001e-05 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917574+0000", 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9739999999999999e-06 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917641+0000", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6390000000000006e-05 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917652+0000", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1897e-05 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552825+0000", 2023-09-23T15:26:26.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63517245499999997 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552984+0000", 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015915600000000001 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.553150+0000", 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000166009 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565417+0000", 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012266716 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.569974+0000", 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045568479999999996 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578915+0000", 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089418580000000004 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578952+0000", 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6470000000000001e-05 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578982+0000", 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0361999999999998e-05 2023-09-23T15:26:26.557 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87704 8.2 8:40e179a2:::benchmark_data_smithi017_485823_object87703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 219.24776275299999, 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.41878299099999999, 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:26.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87704 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:26.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160289+0000", 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160399+0000", 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011008100000000001 2023-09-23T15:26:26.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160438+0000", 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160452+0000", 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3516000000000001e-05 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160572+0000", 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012081700000000001 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160590+0000", 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8016000000000002e-05 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556242+0000", 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39565125400000001 2023-09-23T15:26:26.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556355+0000", 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011383399999999999 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556489+0000", 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013384700000000001 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565754+0000", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092646399999999993 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.570003+0000", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042494630000000002 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579011+0000", 2023-09-23T15:26:26.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090077760000000003 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579045+0000", 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4317999999999999e-05 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579075+0000", 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9383e-05 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.565 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.566 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:26.566 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:26.566 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:26:26.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:26.567 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:26.567 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:26.567 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:26.568 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:26.568 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:26.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.574 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:26:26.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:26.750 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:26 smithi017 ceph-mon[215646]: pgmap v1042: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/171168 objects degraded (3.417%); 2204/171168 objects misplaced (1.288%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:26.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1360149759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:26.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:26 smithi017 ceph-mon[222878]: pgmap v1042: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/171168 objects degraded (3.417%); 2204/171168 objects misplaced (1.288%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:26.753 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1360149759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:26.758 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:26.839 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:26.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:26.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:26 smithi027 ceph-mon[195233]: pgmap v1042: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 923 op/s; 5849/171168 objects degraded (3.417%); 2204/171168 objects misplaced (1.288%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:26.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1360149759' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:26.944 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:26.972 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:26.998 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 60611 60595 90.1157 56.625 0.00583726 0.0110743 2023-09-23T15:26:27.020 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:27.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:27.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:27.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:27.163 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:27.541 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:27.555 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:27.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:27.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:26:27.662 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:27.719 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:27.800 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:27.808 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/171168 objects degraded (3.417%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:27.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:27.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:27.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/171168 objects degraded (3.417%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/171168 objects degraded (3.417%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:27.958 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:27.998 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 61480 61464 89.2834 54.3125 0.00819066 0.0111932 2023-09-23T15:26:28.038 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:28.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:28.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:28.195 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:28.227 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:28] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T15:26:28.229 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:28.300 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:28.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:28.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:28.411 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:28.484 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:28.561 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[215646]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 5849/174537 objects degraded (3.351%); 2204/174537 objects misplaced (1.263%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:28.561 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:28.561 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[222878]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 5849/174537 objects degraded (3.351%); 2204/174537 objects misplaced (1.263%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:28.561 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:28.562 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:28.562 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3711499640' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.562 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.582 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:28.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:28.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:28.778 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:28.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:28 smithi027 ceph-mon[195233]: pgmap v1043: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 59 MiB/s wr, 948 op/s; 5849/174537 objects degraded (3.351%); 2204/174537 objects misplaced (1.263%); 3.2 MiB/s, 52 objects/s recovering 2023-09-23T15:26:28.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:28.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:28.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3711499640' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.894 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:28.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:28.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:28.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:28.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3711499640' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.905 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T15:26:28.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 62558 62542 88.7847 67.375 0.00622125 0.0112583 2023-09-23T15:26:29.002 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:29.083 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:29.183 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:29.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:29.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:29.200 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 9 pgp_num to 16 2023-09-23T15:26:29.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:29.382 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:29.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:26:29.957 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:29.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:29.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:29.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 63289 63273 87.8273 45.6875 0.00331774 0.0113768 2023-09-23T15:26:30.062 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:30.102 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:30.178 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:30.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:30.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:30.223 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s; 5849/177021 objects degraded (3.304%); 2204/177021 objects misplaced (1.245%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[215646]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s; 5849/177021 objects degraded (3.304%); 2204/177021 objects misplaced (1.245%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T15:26:30.224 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.225 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:30 smithi017 ceph-mon[222878]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.284 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: pgmap v1044: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 58 MiB/s wr, 932 op/s; 5849/177021 objects degraded (3.304%); 2204/177021 objects misplaced (1.245%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T15:26:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T15:26:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: osdmap e353: 8 total, 8 up, 7 in 2023-09-23T15:26:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:30 smithi027 ceph-mon[195233]: failed to encode map e353 with expected crc 2023-09-23T15:26:30.356 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:30.498 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:30.501 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:30.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:30.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:30.701 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:30.770 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:26:30.770 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:26:30.860 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:30.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:30.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:30.966 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:30.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 64171 64155 87.1167 55.125 0.0120445 0.0114732 2023-09-23T15:26:31.036 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:31.223 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:26:31.313 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:31.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:31.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:31.423 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:31.469 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:31.477 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:31.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:31.764 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:31.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:31.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:31.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:31.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:26:31.932 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:31.999 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 64989 64973 86.3511 51.125 0.0244832 0.0115747 2023-09-23T15:26:32.401 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:32.588 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: pgmap v1046: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 5849/187764 objects degraded (3.115%); 1925/187764 objects misplaced (1.025%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:32.588 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:26:32.588 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.588 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:32.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[215646]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T15:26:32.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: pgmap v1046: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 5849/187764 objects degraded (3.115%); 1925/187764 objects misplaced (1.025%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:32.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:26:32.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:32.591 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:32 smithi017 ceph-mon[222878]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T15:26:32.594 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: pgmap v1046: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.8 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 961 op/s; 5849/187764 objects degraded (3.115%); 1925/187764 objects misplaced (1.025%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: failed to encode map e354 with expected crc 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1446329785' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:32 smithi027 ceph-mon[195233]: osdmap e354: 8 total, 8 up, 7 in 2023-09-23T15:26:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:32.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:32.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:32.804 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:32.990 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:32.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:32.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:33.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 65781 65765 85.5721 49.5 0.0135088 0.0116809 2023-09-23T15:26:33.098 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:33.140 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:33.196 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:33.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:33.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:33.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:33.343 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:33 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/187764 objects degraded (3.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:33.346 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:33.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:33.397 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:33.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:33.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:33.418 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:33.452 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:33.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:33.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:33.538 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:33.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:33 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/187764 objects degraded (3.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:33.567 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:33 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/187764 objects degraded (3.115%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:33.574 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:33.579 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:33.611 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:33.657 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.683 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:26:33.715 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:33.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:33.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:33.822 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:33.874 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:34.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 66695 66679 84.9918 57.125 0.0125826 0.0117605 2023-09-23T15:26:34.023 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:34.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:34.029 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:34.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:34.157 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:34.415 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:34.419 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:34 smithi027 ceph-mon[195233]: pgmap v1048: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/190329 objects degraded (3.073%); 1925/190329 objects misplaced (1.011%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T15:26:34.419 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/787026385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:34.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:34.421 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:34.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:34.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:34 smithi017 ceph-mon[215646]: pgmap v1048: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/190329 objects degraded (3.073%); 1925/190329 objects misplaced (1.011%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T15:26:34.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/787026385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:34.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:34 smithi017 ceph-mon[222878]: pgmap v1048: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 3.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/190329 objects degraded (3.073%); 1925/190329 objects misplaced (1.011%); 2.2 MiB/s, 34 objects/s recovering 2023-09-23T15:26:34.590 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/787026385' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:34.597 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:34.716 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:34.725 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:26:34.725 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:26:34.725 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:26:34.725 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:26:34.725 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:26:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:34.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:34.827 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:34.908 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:34.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:26:35.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 67509 67493 84.3098 50.875 0.0107776 0.0118532 2023-09-23T15:26:35.027 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:35.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:35.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:35.125 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:26:35.126 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:26:35.126 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T15:26:35.126 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:26:35.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:35.349 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:35.429 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:35.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:35.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:35.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:35.665 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:35.671 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:35.957 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:35.965 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:26:35.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:35.968 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:36.006 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 68364 68348 83.7047 53.4375 0.00700652 0.0119393 2023-09-23T15:26:36.069 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:36.114 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:36.207 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:36.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:36.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:36.223 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:36.258 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:36 smithi017 ceph-mon[215646]: pgmap v1049: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 5849/197181 objects degraded (2.966%); 1650/197181 objects misplaced (0.837%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:36.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:36.398 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:36.529 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:36.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:36.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:36.537 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:36 smithi017 ceph-mon[222878]: pgmap v1049: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 5849/197181 objects degraded (2.966%); 1650/197181 objects misplaced (0.837%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:36.557 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:36.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:36 smithi027 ceph-mon[195233]: pgmap v1049: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 59 MiB/s wr, 942 op/s; 5849/197181 objects degraded (2.966%); 1650/197181 objects misplaced (0.837%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:36.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:26:36.638 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:36.746 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:36.807 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:36.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:36.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:36.911 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:37.008 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 69159 69143 83.0496 49.6875 0.0115865 0.0120332 2023-09-23T15:26:37.011 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:37.410 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:37.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:37.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:37.526 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:37.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:37 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:26:37.552 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/197181 objects degraded (2.966%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:37.555 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:37.602 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:37.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:37.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:37.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:37 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:26:37.606 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/197181 objects degraded (2.966%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:37.606 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:37 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:26:37.606 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/197181 objects degraded (2.966%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:37.705 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:37.731 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:37.788 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:37.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:37.792 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:37.893 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:37.980 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:38.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 69849 69833 82.2967 43.125 0.0082406 0.0121453 2023-09-23T15:26:38.278 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:38] "GET /metrics HTTP/1.1" 200 36382 "" "Prometheus/2.43.0" 2023-09-23T15:26:38.279 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:38 smithi017 ceph-mon[222878]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s; 5849/200256 objects degraded (2.921%); 1650/200256 objects misplaced (0.824%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:38.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:38 smithi027 ceph-mon[195233]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s; 5849/200256 objects degraded (2.921%); 1650/200256 objects misplaced (0.824%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:38.373 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:38.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:38.379 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:38.480 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:38.551 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:38.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:38 smithi017 ceph-mon[215646]: pgmap v1050: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 60 MiB/s wr, 967 op/s; 5849/200256 objects degraded (2.921%); 1650/200256 objects misplaced (0.824%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T15:26:38.678 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:26:38.678 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50575 8.4 8:2aa5975c:::benchmark_data_smithi017_485823_object50574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.48461145599998, 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1322413290000002, 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50575 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:26:38.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189078+0000", 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189126+0000", 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7305000000000002e-05 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189145+0000", 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9389e-05 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189150+0000", 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0950000000000003e-06 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189213+0000", 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2564000000000006e-05 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189224+0000", 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.097e-05 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136177+0000", 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94695325500000005 2023-09-23T15:26:38.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136190+0000", 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3512e-05 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296297+0000", 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.160106919 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296465+0000", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016758800000000001 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296632+0000", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016706199999999999 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305668+0000", 2023-09-23T15:26:38.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090357079999999999 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310724+0000", 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050558169999999998 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319181+0000", 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084576249999999999 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319233+0000", 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2042000000000001e-05 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321321+0000", 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020875210000000002 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50584 8.4 8:2701dc0f:::benchmark_data_smithi017_485823_object50583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.47486243600002, 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1246680649999998, 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50584 2023-09-23T15:26:38.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198827+0000", 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198852+0000", 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4725000000000001e-05 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198864+0000", 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2480000000000001e-05 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198869+0000", 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8330000000000001e-06 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198914+0000", 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5084999999999999e-05 2023-09-23T15:26:38.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198921+0000", 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6320000000000002e-06 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136226+0000", 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93730514799999998 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136234+0000", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0450000000000001e-06 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298417+0000", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.162183065 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298513+0000", 2023-09-23T15:26:38.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5981000000000001e-05 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298613+0000", 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100184 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305680+0000", 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070664450000000002 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310775+0000", 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050952819999999996 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321371+0000", 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010595751 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321413+0000", 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2296999999999998e-05 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.323496+0000", 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020832870000000001 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.696 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50598 8.4 8:23276db8:::benchmark_data_smithi017_485823_object50597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.462836434, 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1164891140000002, 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50598 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.698 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210890+0000", 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6159999999999999e-05 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210905+0000", 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5471000000000001e-05 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210912+0000", 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7299999999999999e-06 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210973+0000", 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1427999999999995e-05 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210982+0000", 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4239999999999993e-06 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136258+0000", 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92527595200000001 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136265+0000", 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.413e-06 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300042+0000", 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.163777327 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300173+0000", 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013054000000000001 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300273+0000", 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9741999999999995e-05 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.308013+0000", 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077400969999999996 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310837+0000", 2023-09-23T15:26:38.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028244099999999999 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.325081+0000", 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014243281 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.325147+0000", 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6896999999999993e-05 2023-09-23T15:26:38.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327344+0000", 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021961089999999999 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50603 8.4 8:2629f6f1:::benchmark_data_smithi017_485823_object50602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:26:38.705 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.45911425399999, 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1148396169999999, 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50603 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214576+0000", 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214595+0000", 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9415e-05 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214607+0000", 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1593000000000001e-05 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214612+0000", 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2560000000000002e-06 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214654+0000", 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2367000000000003e-05 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214660+0000", 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.755e-06 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136288+0000", 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92162821399999995 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136296+0000", 2023-09-23T15:26:38.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9079999999999995e-06 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301680+0000", 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.165384051 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301775+0000", 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4308000000000006e-05 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301887+0000", 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112654 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.308024+0000", 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061371790000000004 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.712 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310884+0000", 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028592689999999998 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327407+0000", 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016523044000000001 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327471+0000", 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4053999999999994e-05 2023-09-23T15:26:38.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.329416+0000", 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001945348 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50611 8.4 8:263c3c34:::benchmark_data_smithi017_485823_object50610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:26:38.714 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.44892197199999, 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1099267500000001, 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50611 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224767+0000", 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224833+0000", 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6395000000000004e-05 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224864+0000", 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1495999999999998e-05 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224888+0000", 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4000999999999999e-05 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224936+0000", 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7963999999999997e-05 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224942+0000", 2023-09-23T15:26:38.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8150000000000002e-06 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136321+0000", 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91137871999999998 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136329+0000", 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6939999999999995e-06 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303349+0000", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.16701991 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303507+0000", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015815600000000001 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303691+0000", 2023-09-23T15:26:38.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018426799999999999 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310737+0000", 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070460219999999999 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.311156+0000", 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041903600000000002 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.332349+0000", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021193132 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.332434+0000", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4668999999999999e-05 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334696+0000", 2023-09-23T15:26:38.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022617779999999999 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50612 8.4 8:2bb28cd9:::benchmark_data_smithi017_485823_object50611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 282.44860118299999, 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1116500949999999, 2023-09-23T15:26:38.723 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50612 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225089+0000", 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225146+0000", 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7031999999999999e-05 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225156+0000", 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0304e-05 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225160+0000", 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0199999999999996e-06 2023-09-23T15:26:38.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225201+0000", 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0935e-05 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225209+0000", 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9500000000000001e-06 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.727 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136352+0000", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91114233099999997 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136359+0000", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5009999999999998e-06 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305733+0000", 2023-09-23T15:26:38.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1693735030000001 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305920+0000", 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018741100000000001 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.306064+0000", 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014344600000000001 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310849+0000", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047855119999999996 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.312187+0000", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001338452 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334753+0000", 2023-09-23T15:26:38.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022565686000000001 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334800+0000", 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.655e-05 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.336740+0000", 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001940048 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.731 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148874 8.4 8:22ed0ae9:::benchmark_data_smithi017_485823_object58737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.84581909799999, 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99896163400000004, 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.732 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148874 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:26:38.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827871+0000", 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827874+0000", 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.311e-06 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827879+0000", 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4600000000000002e-06 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827883+0000", 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3730000000000001e-06 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831017+0000", 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031349569999999998 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831029+0000", 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1875000000000001e-05 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782403+0000", 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95137329500000001 2023-09-23T15:26:38.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782500+0000", 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7763999999999994e-05 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782578+0000", 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7972999999999994e-05 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794469+0000", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011890258000000001 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826626+0000", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032157673999999997 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826744+0000", 2023-09-23T15:26:38.738 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117747 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826793+0000", 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9308000000000003e-05 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826833+0000", 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0000000000000003e-05 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.739 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148906 8.4 8:29f45dea:::benchmark_data_smithi017_485823_object58769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.80513813499999, 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95839901500000002, 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.740 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148906 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.741 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868552+0000", 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868555+0000", 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.742 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868558+0000", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7699999999999999e-06 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868561+0000", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868612+0000", 2023-09-23T15:26:38.743 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0853000000000003e-05 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868618+0000", 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4529999999999999e-06 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785580+0000", 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91696206300000005 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.744 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785720+0000", 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139677 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785862+0000", 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014233400000000001 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.745 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799990+0000", 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014128043999999999 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826695+0000", 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026704894 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826890+0000", 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019431599999999999 2023-09-23T15:26:38.746 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826928+0000", 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8148000000000001e-05 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826952+0000", 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4049e-05 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.747 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148912 8.4 8:2ad9ecb5:::benchmark_data_smithi017_485823_object58775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.80118457099999, 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.954622631, 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.748 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148912 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872505+0000", 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872512+0000", 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8390000000000001e-06 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872524+0000", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2469e-05 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872530+0000", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.139e-06 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872582+0000", 2023-09-23T15:26:38.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1983e-05 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872592+0000", 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3710000000000006e-06 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788471+0000", 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91587959200000002 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788569+0000", 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8047000000000005e-05 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788661+0000", 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1267000000000006e-05 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799997+0000", 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011336841 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827002+0000", 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027004980000000001 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827082+0000", 2023-09-23T15:26:38.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9484e-05 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827107+0000", 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5216999999999999e-05 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827129+0000", 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1801e-05 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.755 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148934 8.4 8:2bdf4584:::benchmark_data_smithi017_485823_object58797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.76947629899999, 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92312028400000001, 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148934 2023-09-23T15:26:38.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904213+0000", 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904224+0000", 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0525e-05 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904236+0000", 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2649999999999999e-05 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904243+0000", 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.241e-06 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904339+0000", 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6711000000000003e-05 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904349+0000", 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7340000000000003e-06 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791162+0000", 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88681287099999995 2023-09-23T15:26:38.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791285+0000", 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012337700000000001 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791419+0000", 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134029 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804693+0000", 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013273785 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827031+0000", 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022337704999999999 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827243+0000", 2023-09-23T15:26:38.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021219399999999999 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827294+0000", 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.113e-05 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827335+0000", 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0677999999999999e-05 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.763 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148942 8.4 8:2f2d7443:::benchmark_data_smithi017_485823_object58805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.763194779, 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91690406000000002, 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.764 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148942 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910495+0000", 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910500+0000", 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.57e-06 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910514+0000", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.365e-05 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910522+0000", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7740000000000004e-06 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910590+0000", 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8472000000000004e-05 2023-09-23T15:26:38.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910602+0000", 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1343e-05 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794512+0000", 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88391032800000002 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794644+0000", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013237700000000001 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794779+0000", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134709 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804705+0000", 2023-09-23T15:26:38.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099256569999999992 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827184+0000", 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022478946999999999 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827355+0000", 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171628 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827381+0000", 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5789999999999999e-05 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827400+0000", 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8995e-05 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148943 8.4 8:26834e88:::benchmark_data_smithi017_485823_object58806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.76315136100001, 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.916991004, 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148943 2023-09-23T15:26:38.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910542+0000", 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7889999999999999e-06 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910554+0000", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2216e-05 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910560+0000", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1990000000000003e-06 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910623+0000", 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3254000000000001e-05 2023-09-23T15:26:38.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910628+0000", 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797471+0000", 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88684291900000001 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797586+0000", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114815 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797686+0000", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100275 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810164+0000", 2023-09-23T15:26:38.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012477492 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827205+0000", 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017040726999999999 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827444+0000", 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023902400000000001 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827492+0000", 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8707e-05 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827530+0000", 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8102000000000002e-05 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148947 8.4 8:25191517:::benchmark_data_smithi017_485823_object58810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.758814718, 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91295790200000004, 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.780 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148947 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:26:38.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914879+0000", 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.683e-06 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914884+0000", 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8189999999999997e-06 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914888+0000", 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.121e-06 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914950+0000", 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1359999999999995e-05 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914961+0000", 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.076e-05 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800046+0000", 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88508585500000003 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800194+0000", 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014714700000000001 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800323+0000", 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129197 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810173+0000", 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098501830000000002 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827554+0000", 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017381285 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827782+0000", 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022772600000000001 2023-09-23T15:26:38.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827812+0000", 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0433000000000002e-05 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827834+0000", 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1628e-05 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148958 8.4 8:2e39e1d6:::benchmark_data_smithi017_485823_object58821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.74520606199999, 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89949832200000002, 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148958 2023-09-23T15:26:38.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928483+0000", 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928493+0000", 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0638999999999999e-05 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928514+0000", 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0859000000000001e-05 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928523+0000", 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5860000000000005e-06 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928590+0000", 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7299999999999996e-05 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928600+0000", 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7189999999999996e-06 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802452+0000", 2023-09-23T15:26:38.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87385171800000006 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802535+0000", 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3803999999999999e-05 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802599+0000", 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3928000000000002e-05 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815177+0000", 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012578119 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827593+0000", 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012415981 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827890+0000", 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029696900000000001 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827945+0000", 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4586999999999998e-05 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827983+0000", 2023-09-23T15:26:38.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8109999999999999e-05 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148960 8.4 8:2ab580ee:::benchmark_data_smithi017_485823_object58823:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.929877+0000", 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.74381370399999, 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89836297700000001, 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148960 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929877+0000", 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929877+0000", 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929877+0000", 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929881+0000", 2023-09-23T15:26:38.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.882e-06 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929894+0000", 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2957e-05 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929899+0000", 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5569999999999997e-06 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929940+0000", 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1298999999999999e-05 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.929946+0000", 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1630000000000001e-06 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804756+0000", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87480927500000005 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804881+0000", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012502300000000001 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804998+0000", 2023-09-23T15:26:38.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117149 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815185+0000", 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010187689 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827613+0000", 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01242736 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828148+0000", 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00053557699999999997 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828201+0000", 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2305000000000002e-05 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828240+0000", 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9369999999999997e-05 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148985 8.4 8:24cf954e:::benchmark_data_smithi017_485823_object58848:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:27.057223+0000", 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.616467696, 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77115606299999995, 2023-09-23T15:26:38.804 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148985 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057223+0000", 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057223+0000", 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057222+0000", 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057229+0000", 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.678e-06 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057246+0000", 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7235e-05 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057253+0000", 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5810000000000002e-06 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057329+0000", 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6482000000000004e-05 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.057338+0000", 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7649999999999999e-06 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.807711+0000", 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75037226499999998 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.807842+0000", 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013127200000000001 2023-09-23T15:26:38.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.807980+0000", 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138166 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.820994+0000", 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013013821 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827650+0000", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006656492 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828292+0000", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064208600000000005 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828338+0000", 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5680000000000003e-05 2023-09-23T15:26:38.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828379+0000", 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1174999999999998e-05 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148994 8.4 8:219ac14f:::benchmark_data_smithi017_485823_object58857:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.812 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:27.070762+0000", 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.60292894099999, 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75784714600000003, 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148994 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070762+0000", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070762+0000", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070760+0000", 2023-09-23T15:26:38.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070770+0000", 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.183e-06 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070790+0000", 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0548999999999999e-05 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070797+0000", 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3780000000000003e-06 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070874+0000", 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6154000000000003e-05 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.070889+0000", 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5739e-05 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810222+0000", 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.73933238700000004 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810344+0000", 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012204100000000001 2023-09-23T15:26:38.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810465+0000", 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121614 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.821001+0000", 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010535539 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827669+0000", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066679010000000004 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828521+0000", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00085247199999999995 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828570+0000", 2023-09-23T15:26:38.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8717999999999997e-05 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828609+0000", 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9048999999999998e-05 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:149006 8.4 8:29213eea:::benchmark_data_smithi017_485823_object58869:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:27.090061+0000", 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.583629984, 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.73869153700000001, 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.821 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 149006 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090061+0000", 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090061+0000", 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090059+0000", 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090067+0000", 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0600000000000008e-06 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090086+0000", 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8436000000000001e-05 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090093+0000", 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5209999999999997e-06 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090181+0000", 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8005999999999997e-05 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.090194+0000", 2023-09-23T15:26:38.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.307e-05 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.813147+0000", 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.722952751 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.813242+0000", 2023-09-23T15:26:38.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5203000000000003e-05 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.813313+0000", 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1051999999999997e-05 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826259+0000", 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012946025999999999 2023-09-23T15:26:38.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827724+0000", 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001464485 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828665+0000", 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00094099299999999999 2023-09-23T15:26:38.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828713+0000", 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7970999999999999e-05 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828752+0000", 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9622000000000001e-05 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.829 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:149034 8.4 8:2206fd28:::benchmark_data_smithi017_485823_object58897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:27.150027+0000", 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.52366337399999, 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.685963561, 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 149034 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150027+0000", 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150027+0000", 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150027+0000", 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150031+0000", 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4519999999999999e-06 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150043+0000", 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2119e-05 2023-09-23T15:26:38.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150049+0000", 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3299999999999998e-06 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150108+0000", 2023-09-23T15:26:38.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9460999999999999e-05 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.150119+0000", 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1501000000000001e-05 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815231+0000", 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66511148099999995 2023-09-23T15:26:38.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815350+0000", 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011916799999999999 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815468+0000", 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011788799999999999 2023-09-23T15:26:38.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826265+0000", 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010796744 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828042+0000", 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001777083 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.835895+0000", 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0078533549999999994 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.835949+0000", 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4073999999999998e-05 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.835991+0000", 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1754e-05 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:149038 8.4 8:2a1ddaa5:::benchmark_data_smithi017_485823_object58901:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:26:38.839 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:27.157169+0000", 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.51652208799999, 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.67892008199999998, 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 149038 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:38.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157169+0000", 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157169+0000", 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157168+0000", 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157174+0000", 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.66e-06 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157183+0000", 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4800000000000007e-06 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157189+0000", 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8379999999999998e-06 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157249+0000", 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9728000000000001e-05 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.157258+0000", 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8799999999999997e-06 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:38.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.817954+0000", 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66069687600000004 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.818031+0000", 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6157999999999998e-05 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:26:38.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.818110+0000", 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9481999999999996e-05 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826366+0000", 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082557589999999997 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.828094+0000", 2023-09-23T15:26:38.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017282630000000001 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.836023+0000", 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0079289289999999995 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.836058+0000", 2023-09-23T15:26:38.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4669000000000003e-05 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.836089+0000", 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1118999999999998e-05 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:38.848 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:26:38.849 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:38.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:38.850 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:38.850 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:38.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:39.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 15 70593 70578 81.6352 46.5625 0.0136774 0.0122431 2023-09-23T15:26:39.041 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:39.243 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:39.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:39.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:39.312 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:26:39.351 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:39.384 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:39.537 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:39.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:39.545 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/713971743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.545 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/713971743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.545 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:39.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/713971743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:26:39.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:39.685 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:39.916 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:39.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:39.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:39.932 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:26:40.007 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 71351 71335 81.0114 47.3125 0.00331017 0.0123272 2023-09-23T15:26:40.024 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:40.100 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:40.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 5849/202677 objects degraded (2.886%); 1650/202677 objects misplaced (0.814%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.308 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:40 smithi027 ceph-mon[195233]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 5849/202677 objects degraded (2.886%); 1650/202677 objects misplaced (0.814%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:40.423 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[222878]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: pgmap v1051: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.1 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 56 MiB/s wr, 889 op/s; 5849/202677 objects degraded (2.886%); 1650/202677 objects misplaced (0.814%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.424 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:26:40.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T15:26:40.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.425 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:40 smithi017 ceph-mon[215646]: failed to encode map e355 with expected crc 2023-09-23T15:26:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:26:40.608 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:40.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:40.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:40.633 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:40.660 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:26:40.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:40.801 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:41.013 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 72077 72061 80.3739 45.375 0.00681646 0.0124366 2023-09-23T15:26:41.256 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:41.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:41.373 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:41.412 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:41.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:41 smithi027 ceph-mon[195233]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:41.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:41 smithi017 ceph-mon[215646]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:41 smithi017 ceph-mon[222878]: from='client.36282 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:26:42.008 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 72672 72656 79.6166 37.1875 0.00558902 0.0125554 2023-09-23T15:26:42.153 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:42.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:42.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:42.219 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1052,"stamp":"2023-09-23T15:26:39.300910+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4430501811,"num_objects":67559,"num_object_clones":0,"num_object_copies":202677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5849,"num_objects_misplaced":1650,"num_objects_unfound":0,"num_objects_dirty":67559,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":68438,"num_write_kb":4334709,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1340,"num_bytes_recovered":85339028,"num_keys_recovered":30,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71944,"ondisk_log_size":71944,"up":327,"acting":324,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":16165468,"kb_used_data":12992868,"kb_used_omap":44,"kb_used_meta":3172499,"kb_avail":640050596,"statfs":{"total":671965249536,"available":655411810304,"internally_reserved":0,"allocated":13304696832,"data_stored":13295017635,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45125,"internal_metadata":3248639931},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":325779456,"num_objects":4971,"num_object_clones":0,"num_object_copies":14913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-275,"num_objects_unfound":0,"num_objects_dirty":4971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4971,"num_write_kb":318144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":275,"num_bytes_recovered":18022400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.502545"},"pg_stats":[{"pgid":"3.1a","version":"68'1","reported_seq":69,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189759+0000","last_change":"2023-09-23T15:26:11.368279+0000","last_active":"2023-09-23T15:26:29.189759+0000","last_peered":"2023-09-23T15:26:29.189759+0000","last_clean":"2023-09-23T15:26:29.189759+0000","last_became_active":"2023-09-23T15:26:11.367801+0000","last_became_peered":"2023-09-23T15:26:11.367801+0000","last_unstale":"2023-09-23T15:26:29.189759+0000","last_undegraded":"2023-09-23T15:26:29.189759+0000","last_fullsized":"2023-09-23T15:26:29.189759+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:26.099110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":276,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188124+0000","last_change":"2023-09-23T15:26:11.385221+0000","last_active":"2023-09-23T15:26:29.188124+0000","last_peered":"2023-09-23T15:26:29.188124+0000","last_clean":"2023-09-23T15:26:29.188124+0000","last_became_active":"2023-09-23T15:26:11.384389+0000","last_became_peered":"2023-09-23T15:26:11.384389+0000","last_unstale":"2023-09-23T15:26:29.188124+0000","last_undegraded":"2023-09-23T15:26:29.188124+0000","last_fullsized":"2023-09-23T15:26:29.188124+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:30.414963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":268,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189299+0000","last_change":"2023-09-23T15:26:11.368570+0000","last_active":"2023-09-23T15:26:29.189299+0000","last_peered":"2023-09-23T15:26:29.189299+0000","last_clean":"2023-09-23T15:26:29.189299+0000","last_became_active":"2023-09-23T15:26:11.368201+0000","last_became_peered":"2023-09-23T15:26:11.368201+0000","last_unstale":"2023-09-23T15:26:29.189299+0000","last_undegraded":"2023-09-23T15:26:29.189299+0000","last_fullsized":"2023-09-23T15:26:29.189299+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:38.276306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":333,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.216022+0000","last_change":"2023-09-23T15:21:19.258732+0000","last_active":"2023-09-23T15:26:29.216022+0000","last_peered":"2023-09-23T15:26:29.216022+0000","last_clean":"2023-09-23T15:26:29.216022+0000","last_became_active":"2023-09-23T15:21:19.258255+0000","last_became_peered":"2023-09-23T15:21:19.258255+0000","last_unstale":"2023-09-23T15:26:29.216022+0000","last_undegraded":"2023-09-23T15:26:29.216022+0000","last_fullsized":"2023-09-23T15:26:29.216022+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:49.776810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":193,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210466+0000","last_change":"2023-09-23T15:25:42.528502+0000","last_active":"2023-09-23T15:26:29.210466+0000","last_peered":"2023-09-23T15:26:29.210466+0000","last_clean":"2023-09-23T15:26:29.210466+0000","last_became_active":"2023-09-23T15:25:42.528034+0000","last_became_peered":"2023-09-23T15:25:42.528034+0000","last_unstale":"2023-09-23T15:26:29.210466+0000","last_undegraded":"2023-09-23T15:26:29.210466+0000","last_fullsized":"2023-09-23T15:26:29.210466+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:59.419787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":304,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186470+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:26:29.186470+0000","last_peered":"2023-09-23T15:26:29.186470+0000","last_clean":"2023-09-23T15:26:29.186470+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:26:29.186470+0000","last_undegraded":"2023-09-23T15:26:29.186470+0000","last_fullsized":"2023-09-23T15:26:29.186470+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":303,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210808+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:26:29.210808+0000","last_peered":"2023-09-23T15:26:29.210808+0000","last_clean":"2023-09-23T15:26:29.210808+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:26:29.210808+0000","last_undegraded":"2023-09-23T15:26:29.210808+0000","last_fullsized":"2023-09-23T15:26:29.210808+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":315,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189362+0000","last_change":"2023-09-23T15:25:42.529084+0000","last_active":"2023-09-23T15:26:29.189362+0000","last_peered":"2023-09-23T15:26:29.189362+0000","last_clean":"2023-09-23T15:26:29.189362+0000","last_became_active":"2023-09-23T15:25:42.528357+0000","last_became_peered":"2023-09-23T15:25:42.528357+0000","last_unstale":"2023-09-23T15:26:29.189362+0000","last_undegraded":"2023-09-23T15:26:29.189362+0000","last_fullsized":"2023-09-23T15:26:29.189362+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:58:21.317285+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":730,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.194553+0000","last_change":"2023-09-23T15:26:14.399298+0000","last_active":"2023-09-23T15:26:29.194553+0000","last_peered":"2023-09-23T15:26:29.194553+0000","last_clean":"2023-09-23T15:26:29.194553+0000","last_became_active":"2023-09-23T15:26:14.398931+0000","last_became_peered":"2023-09-23T15:26:14.398931+0000","last_unstale":"2023-09-23T15:26:29.194553+0000","last_undegraded":"2023-09-23T15:26:29.194553+0000","last_fullsized":"2023-09-23T15:26:29.194553+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:28.275467+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":354,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188087+0000","last_change":"2023-09-23T15:21:19.253639+0000","last_active":"2023-09-23T15:26:29.188087+0000","last_peered":"2023-09-23T15:26:29.188087+0000","last_clean":"2023-09-23T15:26:29.188087+0000","last_became_active":"2023-09-23T15:21:19.253283+0000","last_became_peered":"2023-09-23T15:21:19.253283+0000","last_unstale":"2023-09-23T15:26:29.188087+0000","last_undegraded":"2023-09-23T15:26:29.188087+0000","last_fullsized":"2023-09-23T15:26:29.188087+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:31.946809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":271,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192669+0000","last_change":"2023-09-23T15:25:42.526106+0000","last_active":"2023-09-23T15:26:29.192669+0000","last_peered":"2023-09-23T15:26:29.192669+0000","last_clean":"2023-09-23T15:26:29.192669+0000","last_became_active":"2023-09-23T15:25:42.525734+0000","last_became_peered":"2023-09-23T15:25:42.525734+0000","last_unstale":"2023-09-23T15:26:29.192669+0000","last_undegraded":"2023-09-23T15:26:29.192669+0000","last_fullsized":"2023-09-23T15:26:29.192669+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:09.036516+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":114,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193429+0000","last_change":"2023-09-23T15:26:11.403429+0000","last_active":"2023-09-23T15:26:29.193429+0000","last_peered":"2023-09-23T15:26:29.193429+0000","last_clean":"2023-09-23T15:26:29.193429+0000","last_became_active":"2023-09-23T15:26:11.383762+0000","last_became_peered":"2023-09-23T15:26:11.383762+0000","last_unstale":"2023-09-23T15:26:29.193429+0000","last_undegraded":"2023-09-23T15:26:29.193429+0000","last_fullsized":"2023-09-23T15:26:29.193429+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:39.355290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":245,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189999+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:26:29.189999+0000","last_peered":"2023-09-23T15:26:29.189999+0000","last_clean":"2023-09-23T15:26:29.189999+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:26:29.189999+0000","last_undegraded":"2023-09-23T15:26:29.189999+0000","last_fullsized":"2023-09-23T15:26:29.189999+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":60,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193289+0000","last_change":"2023-09-23T15:26:11.396846+0000","last_active":"2023-09-23T15:26:29.193289+0000","last_peered":"2023-09-23T15:26:29.193289+0000","last_clean":"2023-09-23T15:26:29.193289+0000","last_became_active":"2023-09-23T15:26:11.380259+0000","last_became_peered":"2023-09-23T15:26:11.380259+0000","last_unstale":"2023-09-23T15:26:29.193289+0000","last_undegraded":"2023-09-23T15:26:29.193289+0000","last_fullsized":"2023-09-23T15:26:29.193289+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:32.176239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":241,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187099+0000","last_change":"2023-09-23T15:26:11.374626+0000","last_active":"2023-09-23T15:26:29.187099+0000","last_peered":"2023-09-23T15:26:29.187099+0000","last_clean":"2023-09-23T15:26:29.187099+0000","last_became_active":"2023-09-23T15:26:11.374231+0000","last_became_peered":"2023-09-23T15:26:11.374231+0000","last_unstale":"2023-09-23T15:26:29.187099+0000","last_undegraded":"2023-09-23T15:26:29.187099+0000","last_fullsized":"2023-09-23T15:26:29.187099+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:17:23.765320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":664,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213768+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:26:29.213768+0000","last_peered":"2023-09-23T15:26:29.213768+0000","last_clean":"2023-09-23T15:26:29.213768+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:26:29.213768+0000","last_undegraded":"2023-09-23T15:26:29.213768+0000","last_fullsized":"2023-09-23T15:26:29.213768+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":118,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193029+0000","last_change":"2023-09-23T15:26:11.388830+0000","last_active":"2023-09-23T15:26:29.193029+0000","last_peered":"2023-09-23T15:26:29.193029+0000","last_clean":"2023-09-23T15:26:29.193029+0000","last_became_active":"2023-09-23T15:26:11.379056+0000","last_became_peered":"2023-09-23T15:26:11.379056+0000","last_unstale":"2023-09-23T15:26:29.193029+0000","last_undegraded":"2023-09-23T15:26:29.193029+0000","last_fullsized":"2023-09-23T15:26:29.193029+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:26:42.632096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":102,"num_keys_recovered":30,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":303,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.200622+0000","last_change":"2023-09-23T15:25:42.527397+0000","last_active":"2023-09-23T15:26:29.200622+0000","last_peered":"2023-09-23T15:26:29.200622+0000","last_clean":"2023-09-23T15:26:29.200622+0000","last_became_active":"2023-09-23T15:25:42.527091+0000","last_became_peered":"2023-09-23T15:25:42.527091+0000","last_unstale":"2023-09-23T15:26:29.200622+0000","last_undegraded":"2023-09-23T15:26:29.200622+0000","last_fullsized":"2023-09-23T15:26:29.200622+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:03:58.238273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":712,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187536+0000","last_change":"2023-09-23T15:22:27.296160+0000","last_active":"2023-09-23T15:26:29.187536+0000","last_peered":"2023-09-23T15:26:29.187536+0000","last_clean":"2023-09-23T15:26:29.187536+0000","last_became_active":"2023-09-23T15:22:27.295773+0000","last_became_peered":"2023-09-23T15:22:27.295773+0000","last_unstale":"2023-09-23T15:26:29.187536+0000","last_undegraded":"2023-09-23T15:26:29.187536+0000","last_fullsized":"2023-09-23T15:26:29.187536+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:12:38.759261+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":332,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210459+0000","last_change":"2023-09-23T15:21:19.251494+0000","last_active":"2023-09-23T15:26:29.210459+0000","last_peered":"2023-09-23T15:26:29.210459+0000","last_clean":"2023-09-23T15:26:29.210459+0000","last_became_active":"2023-09-23T15:21:19.250791+0000","last_became_peered":"2023-09-23T15:21:19.250791+0000","last_unstale":"2023-09-23T15:26:29.210459+0000","last_undegraded":"2023-09-23T15:26:29.210459+0000","last_fullsized":"2023-09-23T15:26:29.210459+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:14.503898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":339,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188083+0000","last_change":"2023-09-23T15:21:19.250104+0000","last_active":"2023-09-23T15:26:29.188083+0000","last_peered":"2023-09-23T15:26:29.188083+0000","last_clean":"2023-09-23T15:26:29.188083+0000","last_became_active":"2023-09-23T15:21:19.249789+0000","last_became_peered":"2023-09-23T15:21:19.249789+0000","last_unstale":"2023-09-23T15:26:29.188083+0000","last_undegraded":"2023-09-23T15:26:29.188083+0000","last_fullsized":"2023-09-23T15:26:29.188083+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:16:06.559638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":662,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189884+0000","last_change":"2023-09-23T15:26:14.410996+0000","last_active":"2023-09-23T15:26:29.189884+0000","last_peered":"2023-09-23T15:26:29.189884+0000","last_clean":"2023-09-23T15:26:29.189884+0000","last_became_active":"2023-09-23T15:26:14.410628+0000","last_became_peered":"2023-09-23T15:26:14.410628+0000","last_unstale":"2023-09-23T15:26:29.189884+0000","last_undegraded":"2023-09-23T15:26:29.189884+0000","last_fullsized":"2023-09-23T15:26:29.189884+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:54.330177+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":112,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192607+0000","last_change":"2023-09-23T15:26:11.381231+0000","last_active":"2023-09-23T15:26:29.192607+0000","last_peered":"2023-09-23T15:26:29.192607+0000","last_clean":"2023-09-23T15:26:29.192607+0000","last_became_active":"2023-09-23T15:26:11.380888+0000","last_became_peered":"2023-09-23T15:26:11.380888+0000","last_unstale":"2023-09-23T15:26:29.192607+0000","last_undegraded":"2023-09-23T15:26:29.192607+0000","last_fullsized":"2023-09-23T15:26:29.192607+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:12.014493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":321,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.216087+0000","last_change":"2023-09-23T15:21:19.258549+0000","last_active":"2023-09-23T15:26:29.216087+0000","last_peered":"2023-09-23T15:26:29.216087+0000","last_clean":"2023-09-23T15:26:29.216087+0000","last_became_active":"2023-09-23T15:21:19.257805+0000","last_became_peered":"2023-09-23T15:21:19.257805+0000","last_unstale":"2023-09-23T15:26:29.216087+0000","last_undegraded":"2023-09-23T15:26:29.216087+0000","last_fullsized":"2023-09-23T15:26:29.216087+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:15.752388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":670,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187960+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:26:29.187960+0000","last_peered":"2023-09-23T15:26:29.187960+0000","last_clean":"2023-09-23T15:26:29.187960+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:26:29.187960+0000","last_undegraded":"2023-09-23T15:26:29.187960+0000","last_fullsized":"2023-09-23T15:26:29.187960+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":290,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186182+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:26:29.186182+0000","last_peered":"2023-09-23T15:26:29.186182+0000","last_clean":"2023-09-23T15:26:29.186182+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:26:29.186182+0000","last_undegraded":"2023-09-23T15:26:29.186182+0000","last_fullsized":"2023-09-23T15:26:29.186182+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":304,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193541+0000","last_change":"2023-09-23T15:25:42.532898+0000","last_active":"2023-09-23T15:26:29.193541+0000","last_peered":"2023-09-23T15:26:29.193541+0000","last_clean":"2023-09-23T15:26:29.193541+0000","last_became_active":"2023-09-23T15:25:42.532572+0000","last_became_peered":"2023-09-23T15:25:42.532572+0000","last_unstale":"2023-09-23T15:26:29.193541+0000","last_undegraded":"2023-09-23T15:26:29.193541+0000","last_fullsized":"2023-09-23T15:26:29.193541+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:55.632166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.5","version":"354'4196","reported_seq":4220,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.298118+0000","last_change":"2023-09-23T15:25:35.709660+0000","last_active":"2023-09-23T15:26:34.298118+0000","last_peered":"2023-09-23T15:26:34.298118+0000","last_clean":"2023-09-23T15:26:34.298118+0000","last_became_active":"2023-09-23T15:25:35.709292+0000","last_became_peered":"2023-09-23T15:25:35.709292+0000","last_unstale":"2023-09-23T15:26:34.298118+0000","last_undegraded":"2023-09-23T15:26:34.298118+0000","last_fullsized":"2023-09-23T15:26:34.298118+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4196,"log_dups_size":0,"ondisk_log_size":4196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:09.665343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274989056,"num_objects":4196,"num_object_clones":0,"num_object_copies":12588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4196,"num_write_kb":268544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":636,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193661+0000","last_change":"2023-09-23T15:26:14.396785+0000","last_active":"2023-09-23T15:26:29.193661+0000","last_peered":"2023-09-23T15:26:29.193661+0000","last_clean":"2023-09-23T15:26:29.193661+0000","last_became_active":"2023-09-23T15:26:14.396456+0000","last_became_peered":"2023-09-23T15:26:14.396456+0000","last_unstale":"2023-09-23T15:26:29.193661+0000","last_undegraded":"2023-09-23T15:26:29.193661+0000","last_fullsized":"2023-09-23T15:26:29.193661+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:47.610498+0000","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":108,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193425+0000","last_change":"2023-09-23T15:26:11.400287+0000","last_active":"2023-09-23T15:26:29.193425+0000","last_peered":"2023-09-23T15:26:29.193425+0000","last_clean":"2023-09-23T15:26:29.193425+0000","last_became_active":"2023-09-23T15:26:11.380840+0000","last_became_peered":"2023-09-23T15:26:11.380840+0000","last_unstale":"2023-09-23T15:26:29.193425+0000","last_undegraded":"2023-09-23T15:26:29.193425+0000","last_fullsized":"2023-09-23T15:26:29.193425+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:09.995914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":346,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187438+0000","last_change":"2023-09-23T15:22:27.296165+0000","last_active":"2023-09-23T15:26:29.187438+0000","last_peered":"2023-09-23T15:26:29.187438+0000","last_clean":"2023-09-23T15:26:29.187438+0000","last_became_active":"2023-09-23T15:22:27.295825+0000","last_became_peered":"2023-09-23T15:22:27.295825+0000","last_unstale":"2023-09-23T15:26:29.187438+0000","last_undegraded":"2023-09-23T15:26:29.187438+0000","last_fullsized":"2023-09-23T15:26:29.187438+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:06:12.660586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"354'4400","reported_seq":4432,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:26:37.202515+0000","last_change":"2023-09-23T15:26:12.489819+0000","last_active":"2023-09-23T15:26:37.202515+0000","last_peered":"2023-09-23T15:26:37.202515+0000","last_clean":"2023-09-23T15:26:10.351458+0000","last_became_active":"2023-09-23T15:26:12.482684+0000","last_became_peered":"2023-09-23T15:26:12.482684+0000","last_unstale":"2023-09-23T15:26:37.202515+0000","last_undegraded":"2023-09-23T15:26:12.376377+0000","last_fullsized":"2023-09-23T15:26:12.360595+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4400,"log_dups_size":0,"ondisk_log_size":4400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:02.300238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288358400,"num_objects":4400,"num_object_clones":0,"num_object_copies":13200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4400,"num_write_kb":281600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3","objects":1478},{"shards":"1,3,4","objects":2922}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":547,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210945+0000","last_change":"2023-09-23T15:25:48.080098+0000","last_active":"2023-09-23T15:26:29.210945+0000","last_peered":"2023-09-23T15:26:29.210945+0000","last_clean":"2023-09-23T15:26:29.210945+0000","last_became_active":"2023-09-23T15:25:42.528109+0000","last_became_peered":"2023-09-23T15:25:42.528109+0000","last_unstale":"2023-09-23T15:26:29.210945+0000","last_undegraded":"2023-09-23T15:26:29.210945+0000","last_fullsized":"2023-09-23T15:26:29.210945+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:48.079935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:48.079935+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:30.039456+0000","scrub_duration":0.0050498640000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":108,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193979+0000","last_change":"2023-09-23T15:26:11.405379+0000","last_active":"2023-09-23T15:26:29.193979+0000","last_peered":"2023-09-23T15:26:29.193979+0000","last_clean":"2023-09-23T15:26:29.193979+0000","last_became_active":"2023-09-23T15:26:11.384740+0000","last_became_peered":"2023-09-23T15:26:11.384740+0000","last_unstale":"2023-09-23T15:26:29.193979+0000","last_undegraded":"2023-09-23T15:26:29.193979+0000","last_fullsized":"2023-09-23T15:26:29.193979+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:07.211920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":243,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189207+0000","last_change":"2023-09-23T15:26:11.367504+0000","last_active":"2023-09-23T15:26:29.189207+0000","last_peered":"2023-09-23T15:26:29.189207+0000","last_clean":"2023-09-23T15:26:29.189207+0000","last_became_active":"2023-09-23T15:26:11.367129+0000","last_became_peered":"2023-09-23T15:26:11.367129+0000","last_unstale":"2023-09-23T15:26:29.189207+0000","last_undegraded":"2023-09-23T15:26:29.189207+0000","last_fullsized":"2023-09-23T15:26:29.189207+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:17.725798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.7","version":"354'4129","reported_seq":4152,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.922311+0000","last_change":"2023-09-23T15:25:35.702721+0000","last_active":"2023-09-23T15:26:34.922311+0000","last_peered":"2023-09-23T15:26:34.922311+0000","last_clean":"2023-09-23T15:26:34.922311+0000","last_became_active":"2023-09-23T15:25:35.702416+0000","last_became_peered":"2023-09-23T15:25:35.702416+0000","last_unstale":"2023-09-23T15:26:34.922311+0000","last_undegraded":"2023-09-23T15:26:34.922311+0000","last_fullsized":"2023-09-23T15:26:34.922311+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:06.939333+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270598144,"num_objects":4129,"num_object_clones":0,"num_object_copies":12387,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4129,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4129,"num_write_kb":264256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":581,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186597+0000","last_change":"2023-09-23T15:26:14.412884+0000","last_active":"2023-09-23T15:26:29.186597+0000","last_peered":"2023-09-23T15:26:29.186597+0000","last_clean":"2023-09-23T15:26:29.186597+0000","last_became_active":"2023-09-23T15:26:14.412536+0000","last_became_peered":"2023-09-23T15:26:14.412536+0000","last_unstale":"2023-09-23T15:26:29.186597+0000","last_undegraded":"2023-09-23T15:26:29.186597+0000","last_fullsized":"2023-09-23T15:26:29.186597+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:47.772813+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":94,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193313+0000","last_change":"2023-09-23T15:22:27.435395+0000","last_active":"2023-09-23T15:26:29.193313+0000","last_peered":"2023-09-23T15:26:29.193313+0000","last_clean":"2023-09-23T15:26:29.193313+0000","last_became_active":"2023-09-23T15:22:27.434397+0000","last_became_peered":"2023-09-23T15:22:27.434397+0000","last_unstale":"2023-09-23T15:26:29.193313+0000","last_undegraded":"2023-09-23T15:26:29.193313+0000","last_fullsized":"2023-09-23T15:26:29.193313+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:06.537621+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":317,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186236+0000","last_change":"2023-09-23T15:22:27.299592+0000","last_active":"2023-09-23T15:26:29.186236+0000","last_peered":"2023-09-23T15:26:29.186236+0000","last_clean":"2023-09-23T15:26:29.186236+0000","last_became_active":"2023-09-23T15:22:27.299279+0000","last_became_peered":"2023-09-23T15:22:27.299279+0000","last_unstale":"2023-09-23T15:26:29.186236+0000","last_undegraded":"2023-09-23T15:26:29.186236+0000","last_fullsized":"2023-09-23T15:26:29.186236+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:50.754327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"354'4191","reported_seq":4214,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.957590+0000","last_change":"2023-09-23T15:25:35.723494+0000","last_active":"2023-09-23T15:26:34.957590+0000","last_peered":"2023-09-23T15:26:34.957590+0000","last_clean":"2023-09-23T15:26:34.957590+0000","last_became_active":"2023-09-23T15:25:35.723097+0000","last_became_peered":"2023-09-23T15:25:35.723097+0000","last_unstale":"2023-09-23T15:26:34.957590+0000","last_undegraded":"2023-09-23T15:26:34.957590+0000","last_fullsized":"2023-09-23T15:26:34.957590+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4191,"log_dups_size":0,"ondisk_log_size":4191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:58.863529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":274661376,"num_objects":4191,"num_object_clones":0,"num_object_copies":12573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4191,"num_write_kb":268224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":562,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.194067+0000","last_change":"2023-09-23T15:26:14.408758+0000","last_active":"2023-09-23T15:26:29.194067+0000","last_peered":"2023-09-23T15:26:29.194067+0000","last_clean":"2023-09-23T15:26:29.194067+0000","last_became_active":"2023-09-23T15:26:14.408423+0000","last_became_peered":"2023-09-23T15:26:14.408423+0000","last_unstale":"2023-09-23T15:26:29.194067+0000","last_undegraded":"2023-09-23T15:26:29.194067+0000","last_fullsized":"2023-09-23T15:26:29.194067+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:42.833633+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:42.833633+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:40:53.130509+0000","scrub_duration":0.003114112,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187844+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:26:29.187844+0000","last_peered":"2023-09-23T15:26:29.187844+0000","last_clean":"2023-09-23T15:26:29.187844+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:26:29.187844+0000","last_undegraded":"2023-09-23T15:26:29.187844+0000","last_fullsized":"2023-09-23T15:26:29.187844+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":345,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213622+0000","last_change":"2023-09-23T15:26:11.360512+0000","last_active":"2023-09-23T15:26:29.213622+0000","last_peered":"2023-09-23T15:26:29.213622+0000","last_clean":"2023-09-23T15:26:29.213622+0000","last_became_active":"2023-09-23T15:26:11.360114+0000","last_became_peered":"2023-09-23T15:26:11.360114+0000","last_unstale":"2023-09-23T15:26:29.213622+0000","last_undegraded":"2023-09-23T15:26:29.213622+0000","last_fullsized":"2023-09-23T15:26:29.213622+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:17:29.368629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,3],"acting":[0,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"354'4236","reported_seq":4260,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.484746+0000","last_change":"2023-09-23T15:25:42.524482+0000","last_active":"2023-09-23T15:26:34.484746+0000","last_peered":"2023-09-23T15:26:34.484746+0000","last_clean":"2023-09-23T15:26:34.484746+0000","last_became_active":"2023-09-23T15:25:42.524107+0000","last_became_peered":"2023-09-23T15:25:42.524107+0000","last_unstale":"2023-09-23T15:26:34.484746+0000","last_undegraded":"2023-09-23T15:26:34.484746+0000","last_fullsized":"2023-09-23T15:26:34.484746+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4236,"log_dups_size":0,"ondisk_log_size":4236,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:18.831761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277610496,"num_objects":4236,"num_object_clones":0,"num_object_copies":12708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4236,"num_write_kb":271104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":590,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192826+0000","last_change":"2023-09-23T15:21:22.258976+0000","last_active":"2023-09-23T15:26:29.192826+0000","last_peered":"2023-09-23T15:26:29.192826+0000","last_clean":"2023-09-23T15:26:29.192826+0000","last_became_active":"2023-09-23T15:21:22.258349+0000","last_became_peered":"2023-09-23T15:21:22.258349+0000","last_unstale":"2023-09-23T15:26:29.192826+0000","last_undegraded":"2023-09-23T15:26:29.192826+0000","last_fullsized":"2023-09-23T15:26:29.192826+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:36.256399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:36.256399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:51.912203+0000","scrub_duration":0.0063027459999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":54,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213720+0000","last_change":"2023-09-23T15:26:11.374595+0000","last_active":"2023-09-23T15:26:29.213720+0000","last_peered":"2023-09-23T15:26:29.213720+0000","last_clean":"2023-09-23T15:26:29.213720+0000","last_became_active":"2023-09-23T15:26:11.360257+0000","last_became_peered":"2023-09-23T15:26:11.360257+0000","last_unstale":"2023-09-23T15:26:29.213720+0000","last_undegraded":"2023-09-23T15:26:29.213720+0000","last_fullsized":"2023-09-23T15:26:29.213720+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:33.506816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":82,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186464+0000","last_change":"2023-09-23T15:26:11.371620+0000","last_active":"2023-09-23T15:26:29.186464+0000","last_peered":"2023-09-23T15:26:29.186464+0000","last_clean":"2023-09-23T15:26:29.186464+0000","last_became_active":"2023-09-23T15:26:11.371004+0000","last_became_peered":"2023-09-23T15:26:11.371004+0000","last_unstale":"2023-09-23T15:26:29.186464+0000","last_undegraded":"2023-09-23T15:26:29.186464+0000","last_fullsized":"2023-09-23T15:26:29.186464+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:05:41.106442+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.0","version":"354'4431","reported_seq":4455,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:37.192395+0000","last_change":"2023-09-23T15:25:35.722919+0000","last_active":"2023-09-23T15:26:37.192395+0000","last_peered":"2023-09-23T15:26:37.192395+0000","last_clean":"2023-09-23T15:26:37.192395+0000","last_became_active":"2023-09-23T15:25:35.722667+0000","last_became_peered":"2023-09-23T15:25:35.722667+0000","last_unstale":"2023-09-23T15:26:37.192395+0000","last_undegraded":"2023-09-23T15:26:37.192395+0000","last_fullsized":"2023-09-23T15:26:37.192395+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:51.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290390016,"num_objects":4431,"num_object_clones":0,"num_object_copies":13293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4431,"num_write_kb":283584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":658,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187561+0000","last_change":"2023-09-23T15:16:46.272397+0000","last_active":"2023-09-23T15:26:29.187561+0000","last_peered":"2023-09-23T15:26:29.187561+0000","last_clean":"2023-09-23T15:26:29.187561+0000","last_became_active":"2023-09-23T15:16:46.271876+0000","last_became_peered":"2023-09-23T15:16:46.271876+0000","last_unstale":"2023-09-23T15:26:29.187561+0000","last_undegraded":"2023-09-23T15:26:29.187561+0000","last_fullsized":"2023-09-23T15:26:29.187561+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:12:40.331327+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:07.587391+0000","scrub_duration":0.02087499,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":311,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186290+0000","last_change":"2023-09-23T15:26:11.369439+0000","last_active":"2023-09-23T15:26:29.186290+0000","last_peered":"2023-09-23T15:26:29.186290+0000","last_clean":"2023-09-23T15:26:29.186290+0000","last_became_active":"2023-09-23T15:26:11.369036+0000","last_became_peered":"2023-09-23T15:26:11.369036+0000","last_unstale":"2023-09-23T15:26:29.186290+0000","last_undegraded":"2023-09-23T15:26:29.186290+0000","last_fullsized":"2023-09-23T15:26:29.186290+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:54.756776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":242,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.195736+0000","last_change":"2023-09-23T15:26:11.370435+0000","last_active":"2023-09-23T15:26:29.195736+0000","last_peered":"2023-09-23T15:26:29.195736+0000","last_clean":"2023-09-23T15:26:29.195736+0000","last_became_active":"2023-09-23T15:26:11.370002+0000","last_became_peered":"2023-09-23T15:26:11.370002+0000","last_unstale":"2023-09-23T15:26:29.195736+0000","last_undegraded":"2023-09-23T15:26:29.195736+0000","last_fullsized":"2023-09-23T15:26:29.195736+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:28.143906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.3","version":"354'4156","reported_seq":4179,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.630252+0000","last_change":"2023-09-23T15:25:35.712664+0000","last_active":"2023-09-23T15:26:34.630252+0000","last_peered":"2023-09-23T15:26:34.630252+0000","last_clean":"2023-09-23T15:26:34.630252+0000","last_became_active":"2023-09-23T15:25:35.712363+0000","last_became_peered":"2023-09-23T15:25:35.712363+0000","last_unstale":"2023-09-23T15:26:34.630252+0000","last_undegraded":"2023-09-23T15:26:34.630252+0000","last_fullsized":"2023-09-23T15:26:34.630252+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4156,"log_dups_size":0,"ondisk_log_size":4156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:33:38.768683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272367616,"num_objects":4156,"num_object_clones":0,"num_object_copies":12468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4156,"num_write_kb":265984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":611,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189798+0000","last_change":"2023-09-23T15:16:50.492722+0000","last_active":"2023-09-23T15:26:29.189798+0000","last_peered":"2023-09-23T15:26:29.189798+0000","last_clean":"2023-09-23T15:26:29.189798+0000","last_became_active":"2023-09-23T15:16:46.266932+0000","last_became_peered":"2023-09-23T15:16:46.266932+0000","last_unstale":"2023-09-23T15:26:29.189798+0000","last_undegraded":"2023-09-23T15:26:29.189798+0000","last_fullsized":"2023-09-23T15:26:29.189798+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:32.835003+0000","scrub_duration":0.0017666369999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":238,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.194393+0000","last_change":"2023-09-23T15:26:11.371002+0000","last_active":"2023-09-23T15:26:29.194393+0000","last_peered":"2023-09-23T15:26:29.194393+0000","last_clean":"2023-09-23T15:26:29.194393+0000","last_became_active":"2023-09-23T15:26:11.370662+0000","last_became_peered":"2023-09-23T15:26:11.370662+0000","last_unstale":"2023-09-23T15:26:29.194393+0000","last_undegraded":"2023-09-23T15:26:29.194393+0000","last_fullsized":"2023-09-23T15:26:29.194393+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:02.511171+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":241,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186404+0000","last_change":"2023-09-23T15:26:11.375798+0000","last_active":"2023-09-23T15:26:29.186404+0000","last_peered":"2023-09-23T15:26:29.186404+0000","last_clean":"2023-09-23T15:26:29.186404+0000","last_became_active":"2023-09-23T15:26:11.375443+0000","last_became_peered":"2023-09-23T15:26:11.375443+0000","last_unstale":"2023-09-23T15:26:29.186404+0000","last_undegraded":"2023-09-23T15:26:29.186404+0000","last_fullsized":"2023-09-23T15:26:29.186404+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:07.695507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.2","version":"354'4316","reported_seq":4338,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:37.156132+0000","last_change":"2023-09-23T15:25:42.527999+0000","last_active":"2023-09-23T15:26:37.156132+0000","last_peered":"2023-09-23T15:26:37.156132+0000","last_clean":"2023-09-23T15:26:37.156132+0000","last_became_active":"2023-09-23T15:25:42.527725+0000","last_became_peered":"2023-09-23T15:25:42.527725+0000","last_unstale":"2023-09-23T15:26:37.156132+0000","last_undegraded":"2023-09-23T15:26:37.156132+0000","last_fullsized":"2023-09-23T15:26:37.156132+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4316,"log_dups_size":0,"ondisk_log_size":4316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:59.569058+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282853376,"num_objects":4316,"num_object_clones":0,"num_object_copies":12948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4316,"num_write_kb":276224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":670,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187868+0000","last_change":"2023-09-23T15:22:26.307688+0000","last_active":"2023-09-23T15:26:29.187868+0000","last_peered":"2023-09-23T15:26:29.187868+0000","last_clean":"2023-09-23T15:26:29.187868+0000","last_became_active":"2023-09-23T15:22:26.307093+0000","last_became_peered":"2023-09-23T15:22:26.307093+0000","last_unstale":"2023-09-23T15:26:29.187868+0000","last_undegraded":"2023-09-23T15:26:29.187868+0000","last_fullsized":"2023-09-23T15:26:29.187868+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:30.287148+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:30.287148+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:04.682127+0000","scrub_duration":0.067208851999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":313,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189336+0000","last_change":"2023-09-23T15:25:42.529276+0000","last_active":"2023-09-23T15:26:29.189336+0000","last_peered":"2023-09-23T15:26:29.189336+0000","last_clean":"2023-09-23T15:26:29.189336+0000","last_became_active":"2023-09-23T15:25:42.528783+0000","last_became_peered":"2023-09-23T15:25:42.528783+0000","last_unstale":"2023-09-23T15:26:29.189336+0000","last_undegraded":"2023-09-23T15:26:29.189336+0000","last_fullsized":"2023-09-23T15:26:29.189336+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:22.381897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":272,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.191001+0000","last_change":"2023-09-23T15:26:11.380590+0000","last_active":"2023-09-23T15:26:29.191001+0000","last_peered":"2023-09-23T15:26:29.191001+0000","last_clean":"2023-09-23T15:26:29.191001+0000","last_became_active":"2023-09-23T15:26:11.380152+0000","last_became_peered":"2023-09-23T15:26:11.380152+0000","last_unstale":"2023-09-23T15:26:29.191001+0000","last_undegraded":"2023-09-23T15:26:29.191001+0000","last_fullsized":"2023-09-23T15:26:29.191001+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:27:44.803182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"354'4301","reported_seq":4324,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.955327+0000","last_change":"2023-09-23T15:25:35.716554+0000","last_active":"2023-09-23T15:26:34.955327+0000","last_peered":"2023-09-23T15:26:34.955327+0000","last_clean":"2023-09-23T15:26:34.955327+0000","last_became_active":"2023-09-23T15:25:35.716257+0000","last_became_peered":"2023-09-23T15:25:35.716257+0000","last_unstale":"2023-09-23T15:26:34.955327+0000","last_undegraded":"2023-09-23T15:26:34.955327+0000","last_fullsized":"2023-09-23T15:26:34.955327+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4301,"log_dups_size":0,"ondisk_log_size":4301,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:37.096030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281870336,"num_objects":4301,"num_object_clones":0,"num_object_copies":12903,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4301,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4301,"num_write_kb":275264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":551,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186590+0000","last_change":"2023-09-23T15:22:27.308873+0000","last_active":"2023-09-23T15:26:29.186590+0000","last_peered":"2023-09-23T15:26:29.186590+0000","last_clean":"2023-09-23T15:26:29.186590+0000","last_became_active":"2023-09-23T15:22:27.308608+0000","last_became_peered":"2023-09-23T15:22:27.308608+0000","last_unstale":"2023-09-23T15:26:29.186590+0000","last_undegraded":"2023-09-23T15:26:29.186590+0000","last_fullsized":"2023-09-23T15:26:29.186590+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:24:02.102298+0000","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":554,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.200890+0000","last_change":"2023-09-23T15:21:19.271066+0000","last_active":"2023-09-23T15:26:29.200890+0000","last_peered":"2023-09-23T15:26:29.200890+0000","last_clean":"2023-09-23T15:26:29.200890+0000","last_became_active":"2023-09-23T15:21:19.255056+0000","last_became_peered":"2023-09-23T15:21:19.255056+0000","last_unstale":"2023-09-23T15:26:29.200890+0000","last_undegraded":"2023-09-23T15:26:29.200890+0000","last_fullsized":"2023-09-23T15:26:29.200890+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:26:39.564911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":330,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188047+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:26:29.188047+0000","last_peered":"2023-09-23T15:26:29.188047+0000","last_clean":"2023-09-23T15:26:29.188047+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:26:29.188047+0000","last_undegraded":"2023-09-23T15:26:29.188047+0000","last_fullsized":"2023-09-23T15:26:29.188047+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":308,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:31.203251+0000","last_change":"2023-09-23T15:26:11.374600+0000","last_active":"2023-09-23T15:26:31.203251+0000","last_peered":"2023-09-23T15:26:31.203251+0000","last_clean":"2023-09-23T15:26:31.203251+0000","last_became_active":"2023-09-23T15:26:11.374199+0000","last_became_peered":"2023-09-23T15:26:11.374199+0000","last_unstale":"2023-09-23T15:26:31.203251+0000","last_undegraded":"2023-09-23T15:26:31.203251+0000","last_fullsized":"2023-09-23T15:26:31.203251+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:42:52.247541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"354'4156","reported_seq":4180,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.276123+0000","last_change":"2023-09-23T15:25:35.708965+0000","last_active":"2023-09-23T15:26:34.276123+0000","last_peered":"2023-09-23T15:26:34.276123+0000","last_clean":"2023-09-23T15:26:34.276123+0000","last_became_active":"2023-09-23T15:25:35.708632+0000","last_became_peered":"2023-09-23T15:25:35.708632+0000","last_unstale":"2023-09-23T15:26:34.276123+0000","last_undegraded":"2023-09-23T15:26:34.276123+0000","last_fullsized":"2023-09-23T15:26:34.276123+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4156,"log_dups_size":0,"ondisk_log_size":4156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:15.667562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272367616,"num_objects":4156,"num_object_clones":0,"num_object_copies":12468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4156,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4156,"num_write_kb":265984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":651,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187606+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:26:29.187606+0000","last_peered":"2023-09-23T15:26:29.187606+0000","last_clean":"2023-09-23T15:26:29.187606+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:26:29.187606+0000","last_undegraded":"2023-09-23T15:26:29.187606+0000","last_fullsized":"2023-09-23T15:26:29.187606+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":889,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188152+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:26:29.188152+0000","last_peered":"2023-09-23T15:26:29.188152+0000","last_clean":"2023-09-23T15:26:29.188152+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:26:29.188152+0000","last_undegraded":"2023-09-23T15:26:29.188152+0000","last_fullsized":"2023-09-23T15:26:29.188152+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":122,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213969+0000","last_change":"2023-09-23T15:26:11.365938+0000","last_active":"2023-09-23T15:26:29.213969+0000","last_peered":"2023-09-23T15:26:29.213969+0000","last_clean":"2023-09-23T15:26:29.213969+0000","last_became_active":"2023-09-23T15:26:11.365355+0000","last_became_peered":"2023-09-23T15:26:11.365355+0000","last_unstale":"2023-09-23T15:26:29.213969+0000","last_undegraded":"2023-09-23T15:26:29.213969+0000","last_fullsized":"2023-09-23T15:26:29.213969+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:55:40.583184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":251,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.200550+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:26:29.200550+0000","last_peered":"2023-09-23T15:26:29.200550+0000","last_clean":"2023-09-23T15:26:29.200550+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:26:29.200550+0000","last_undegraded":"2023-09-23T15:26:29.200550+0000","last_fullsized":"2023-09-23T15:26:29.200550+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"354'4199","reported_seq":4222,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.940557+0000","last_change":"2023-09-23T15:25:35.715315+0000","last_active":"2023-09-23T15:26:34.940557+0000","last_peered":"2023-09-23T15:26:34.940557+0000","last_clean":"2023-09-23T15:26:34.940557+0000","last_became_active":"2023-09-23T15:25:35.715056+0000","last_became_peered":"2023-09-23T15:25:35.715056+0000","last_unstale":"2023-09-23T15:26:34.940557+0000","last_undegraded":"2023-09-23T15:26:34.940557+0000","last_fullsized":"2023-09-23T15:26:34.940557+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4199,"log_dups_size":0,"ondisk_log_size":4199,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:18.608733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275185664,"num_objects":4199,"num_object_clones":0,"num_object_copies":12597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4199,"num_write_kb":268736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":729,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192883+0000","last_change":"2023-09-23T15:26:14.401428+0000","last_active":"2023-09-23T15:26:29.192883+0000","last_peered":"2023-09-23T15:26:29.192883+0000","last_clean":"2023-09-23T15:26:29.192883+0000","last_became_active":"2023-09-23T15:26:14.401034+0000","last_became_peered":"2023-09-23T15:26:14.401034+0000","last_unstale":"2023-09-23T15:26:29.192883+0000","last_undegraded":"2023-09-23T15:26:29.192883+0000","last_fullsized":"2023-09-23T15:26:29.192883+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:48:11.721227+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,2],"acting":[7,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":972,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:31.203058+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:26:31.203058+0000","last_peered":"2023-09-23T15:26:31.203058+0000","last_clean":"2023-09-23T15:26:31.203058+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:26:31.203058+0000","last_undegraded":"2023-09-23T15:26:31.203058+0000","last_fullsized":"2023-09-23T15:26:31.203058+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":246,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189424+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:26:29.189424+0000","last_peered":"2023-09-23T15:26:29.189424+0000","last_clean":"2023-09-23T15:26:29.189424+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:26:29.189424+0000","last_undegraded":"2023-09-23T15:26:29.189424+0000","last_fullsized":"2023-09-23T15:26:29.189424+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":255,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.190917+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:26:29.190917+0000","last_peered":"2023-09-23T15:26:29.190917+0000","last_clean":"2023-09-23T15:26:29.190917+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:26:29.190917+0000","last_undegraded":"2023-09-23T15:26:29.190917+0000","last_fullsized":"2023-09-23T15:26:29.190917+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"354'4143","reported_seq":4166,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.306012+0000","last_change":"2023-09-23T15:25:42.528955+0000","last_active":"2023-09-23T15:26:34.306012+0000","last_peered":"2023-09-23T15:26:34.306012+0000","last_clean":"2023-09-23T15:26:34.306012+0000","last_became_active":"2023-09-23T15:25:42.528662+0000","last_became_peered":"2023-09-23T15:25:42.528662+0000","last_unstale":"2023-09-23T15:26:34.306012+0000","last_undegraded":"2023-09-23T15:26:34.306012+0000","last_fullsized":"2023-09-23T15:26:34.306012+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:56.711464+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271515648,"num_objects":4143,"num_object_clones":0,"num_object_copies":12429,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4143,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4143,"num_write_kb":265152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":683,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187830+0000","last_change":"2023-09-23T15:21:22.265907+0000","last_active":"2023-09-23T15:26:29.187830+0000","last_peered":"2023-09-23T15:26:29.187830+0000","last_clean":"2023-09-23T15:26:29.187830+0000","last_became_active":"2023-09-23T15:21:22.265418+0000","last_became_peered":"2023-09-23T15:21:22.265418+0000","last_unstale":"2023-09-23T15:26:29.187830+0000","last_undegraded":"2023-09-23T15:26:29.187830+0000","last_fullsized":"2023-09-23T15:26:29.187830+0000","mapping_epoch":290,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":291,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:56.553288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":673,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192742+0000","last_change":"2023-09-23T15:22:27.299564+0000","last_active":"2023-09-23T15:26:29.192742+0000","last_peered":"2023-09-23T15:26:29.192742+0000","last_clean":"2023-09-23T15:26:29.192742+0000","last_became_active":"2023-09-23T15:22:27.299201+0000","last_became_peered":"2023-09-23T15:22:27.299201+0000","last_unstale":"2023-09-23T15:26:29.192742+0000","last_undegraded":"2023-09-23T15:26:29.192742+0000","last_fullsized":"2023-09-23T15:26:29.192742+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:31:09.080707+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":251,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.188053+0000","last_change":"2023-09-23T15:25:42.527275+0000","last_active":"2023-09-23T15:26:29.188053+0000","last_peered":"2023-09-23T15:26:29.188053+0000","last_clean":"2023-09-23T15:26:29.188053+0000","last_became_active":"2023-09-23T15:25:42.526943+0000","last_became_peered":"2023-09-23T15:25:42.526943+0000","last_unstale":"2023-09-23T15:26:29.188053+0000","last_undegraded":"2023-09-23T15:26:29.188053+0000","last_fullsized":"2023-09-23T15:26:29.188053+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:20:40.809855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":315,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186728+0000","last_change":"2023-09-23T15:26:11.382742+0000","last_active":"2023-09-23T15:26:29.186728+0000","last_peered":"2023-09-23T15:26:29.186728+0000","last_clean":"2023-09-23T15:26:29.186728+0000","last_became_active":"2023-09-23T15:26:11.382454+0000","last_became_peered":"2023-09-23T15:26:11.382454+0000","last_unstale":"2023-09-23T15:26:29.186728+0000","last_undegraded":"2023-09-23T15:26:29.186728+0000","last_fullsized":"2023-09-23T15:26:29.186728+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:58.937098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"354'4212","reported_seq":4236,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.283642+0000","last_change":"2023-09-23T15:25:35.708972+0000","last_active":"2023-09-23T15:26:34.283642+0000","last_peered":"2023-09-23T15:26:34.283642+0000","last_clean":"2023-09-23T15:26:34.283642+0000","last_became_active":"2023-09-23T15:25:35.708745+0000","last_became_peered":"2023-09-23T15:25:35.708745+0000","last_unstale":"2023-09-23T15:26:34.283642+0000","last_undegraded":"2023-09-23T15:26:34.283642+0000","last_fullsized":"2023-09-23T15:26:34.283642+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4212,"log_dups_size":0,"ondisk_log_size":4212,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:56.572163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276037632,"num_objects":4212,"num_object_clones":0,"num_object_copies":12636,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4212,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4212,"num_write_kb":269568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":678,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189642+0000","last_change":"2023-09-23T15:26:14.401665+0000","last_active":"2023-09-23T15:26:29.189642+0000","last_peered":"2023-09-23T15:26:29.189642+0000","last_clean":"2023-09-23T15:26:29.189642+0000","last_became_active":"2023-09-23T15:26:14.401311+0000","last_became_peered":"2023-09-23T15:26:14.401311+0000","last_unstale":"2023-09-23T15:26:29.189642+0000","last_undegraded":"2023-09-23T15:26:29.189642+0000","last_fullsized":"2023-09-23T15:26:29.189642+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:04.328053+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":98,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192666+0000","last_change":"2023-09-23T15:21:19.287464+0000","last_active":"2023-09-23T15:26:29.192666+0000","last_peered":"2023-09-23T15:26:29.192666+0000","last_clean":"2023-09-23T15:26:29.192666+0000","last_became_active":"2023-09-23T15:21:19.254650+0000","last_became_peered":"2023-09-23T15:21:19.254650+0000","last_unstale":"2023-09-23T15:26:29.192666+0000","last_undegraded":"2023-09-23T15:26:29.192666+0000","last_fullsized":"2023-09-23T15:26:29.192666+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:51.382496+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":331,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213695+0000","last_change":"2023-09-23T15:26:11.373443+0000","last_active":"2023-09-23T15:26:29.213695+0000","last_peered":"2023-09-23T15:26:29.213695+0000","last_clean":"2023-09-23T15:26:29.213695+0000","last_became_active":"2023-09-23T15:26:11.373168+0000","last_became_peered":"2023-09-23T15:26:11.373168+0000","last_unstale":"2023-09-23T15:26:29.213695+0000","last_undegraded":"2023-09-23T15:26:29.213695+0000","last_fullsized":"2023-09-23T15:26:29.213695+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:43.871002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":334,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.215825+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:26:29.215825+0000","last_peered":"2023-09-23T15:26:29.215825+0000","last_clean":"2023-09-23T15:26:29.215825+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:26:29.215825+0000","last_undegraded":"2023-09-23T15:26:29.215825+0000","last_fullsized":"2023-09-23T15:26:29.215825+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"354'4107","reported_seq":6473,"reported_epoch":354,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T15:26:34.654507+0000","last_change":"2023-09-23T15:26:12.521535+0000","last_active":"2023-09-23T15:26:34.654507+0000","last_peered":"2023-09-23T15:26:34.654507+0000","last_clean":"2023-09-23T15:26:10.351231+0000","last_became_active":"2023-09-23T15:26:12.469771+0000","last_became_peered":"2023-09-23T15:26:12.469771+0000","last_unstale":"2023-09-23T15:26:34.654507+0000","last_undegraded":"2023-09-23T15:26:34.654507+0000","last_fullsized":"2023-09-23T15:26:12.363885+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4107,"log_dups_size":0,"ondisk_log_size":4107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:26.913358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269156352,"num_objects":4107,"num_object_clones":0,"num_object_copies":12321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1650,"num_objects_unfound":0,"num_objects_dirty":4107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4107,"num_write_kb":262848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1166,"num_bytes_recovered":76414976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1650},{"shards":"1,7","objects":2457}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":642,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186401+0000","last_change":"2023-09-23T15:25:41.310902+0000","last_active":"2023-09-23T15:26:29.186401+0000","last_peered":"2023-09-23T15:26:29.186401+0000","last_clean":"2023-09-23T15:26:29.186401+0000","last_became_active":"2023-09-23T15:22:26.297445+0000","last_became_peered":"2023-09-23T15:22:26.297445+0000","last_unstale":"2023-09-23T15:26:29.186401+0000","last_undegraded":"2023-09-23T15:26:29.186401+0000","last_fullsized":"2023-09-23T15:26:29.186401+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:41.310801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:41.310801+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:40.274800+0000","scrub_duration":0.0019631739999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1105,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.193514+0000","last_change":"2023-09-23T15:26:14.405982+0000","last_active":"2023-09-23T15:26:29.193514+0000","last_peered":"2023-09-23T15:26:29.193514+0000","last_clean":"2023-09-23T15:26:29.193514+0000","last_became_active":"2023-09-23T15:26:14.405360+0000","last_became_peered":"2023-09-23T15:26:14.405360+0000","last_unstale":"2023-09-23T15:26:29.193514+0000","last_undegraded":"2023-09-23T15:26:29.193514+0000","last_fullsized":"2023-09-23T15:26:29.193514+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:09.602850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":231,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.200762+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:26:29.200762+0000","last_peered":"2023-09-23T15:26:29.200762+0000","last_clean":"2023-09-23T15:26:29.200762+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:26:29.200762+0000","last_undegraded":"2023-09-23T15:26:29.200762+0000","last_fullsized":"2023-09-23T15:26:29.200762+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":318,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213888+0000","last_change":"2023-09-23T15:21:19.255869+0000","last_active":"2023-09-23T15:26:29.213888+0000","last_peered":"2023-09-23T15:26:29.213888+0000","last_clean":"2023-09-23T15:26:29.213888+0000","last_became_active":"2023-09-23T15:21:19.255526+0000","last_became_peered":"2023-09-23T15:21:19.255526+0000","last_unstale":"2023-09-23T15:26:29.213888+0000","last_undegraded":"2023-09-23T15:26:29.213888+0000","last_fullsized":"2023-09-23T15:26:29.213888+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:01.315988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"354'4222","reported_seq":4245,"reported_epoch":354,"state":"active+clean","last_fresh":"2023-09-23T15:26:34.643571+0000","last_change":"2023-09-23T15:25:35.714862+0000","last_active":"2023-09-23T15:26:34.643571+0000","last_peered":"2023-09-23T15:26:34.643571+0000","last_clean":"2023-09-23T15:26:34.643571+0000","last_became_active":"2023-09-23T15:25:35.714552+0000","last_became_peered":"2023-09-23T15:25:35.714552+0000","last_unstale":"2023-09-23T15:26:34.643571+0000","last_undegraded":"2023-09-23T15:26:34.643571+0000","last_fullsized":"2023-09-23T15:26:34.643571+0000","mapping_epoch":339,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4222,"log_dups_size":0,"ondisk_log_size":4222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:35:31.672142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276692992,"num_objects":4222,"num_object_clones":0,"num_object_copies":12666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4222,"num_write_kb":270208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":614,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.195812+0000","last_change":"2023-09-23T15:26:14.400502+0000","last_active":"2023-09-23T15:26:29.195812+0000","last_peered":"2023-09-23T15:26:29.195812+0000","last_clean":"2023-09-23T15:26:29.195812+0000","last_became_active":"2023-09-23T15:26:14.400112+0000","last_became_peered":"2023-09-23T15:26:14.400112+0000","last_unstale":"2023-09-23T15:26:29.195812+0000","last_undegraded":"2023-09-23T15:26:29.195812+0000","last_fullsized":"2023-09-23T15:26:29.195812+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:15.108372+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:15.108372+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:28.174971+0000","scrub_duration":0.003823791,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,5],"acting":[2,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":110,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.195961+0000","last_change":"2023-09-23T15:21:19.260402+0000","last_active":"2023-09-23T15:26:29.195961+0000","last_peered":"2023-09-23T15:26:29.195961+0000","last_clean":"2023-09-23T15:26:29.195961+0000","last_became_active":"2023-09-23T15:21:19.250868+0000","last_became_peered":"2023-09-23T15:21:19.250868+0000","last_unstale":"2023-09-23T15:26:29.195961+0000","last_undegraded":"2023-09-23T15:26:29.195961+0000","last_fullsized":"2023-09-23T15:26:29.195961+0000","mapping_epoch":287,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":288,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:23.617366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":947,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.190294+0000","last_change":"2023-09-23T15:26:14.411224+0000","last_active":"2023-09-23T15:26:29.190294+0000","last_peered":"2023-09-23T15:26:29.190294+0000","last_clean":"2023-09-23T15:26:29.190294+0000","last_became_active":"2023-09-23T15:26:14.410592+0000","last_became_peered":"2023-09-23T15:26:14.410592+0000","last_unstale":"2023-09-23T15:26:29.190294+0000","last_undegraded":"2023-09-23T15:26:29.190294+0000","last_fullsized":"2023-09-23T15:26:29.190294+0000","mapping_epoch":350,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":351,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:13.276904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":294,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.192504+0000","last_change":"2023-09-23T15:25:42.526102+0000","last_active":"2023-09-23T15:26:29.192504+0000","last_peered":"2023-09-23T15:26:29.192504+0000","last_clean":"2023-09-23T15:26:29.192504+0000","last_became_active":"2023-09-23T15:25:42.525794+0000","last_became_peered":"2023-09-23T15:25:42.525794+0000","last_unstale":"2023-09-23T15:26:29.192504+0000","last_undegraded":"2023-09-23T15:26:29.192504+0000","last_fullsized":"2023-09-23T15:26:29.192504+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:37:10.472481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"354'4136","reported_seq":4168,"reported_epoch":354,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T15:26:34.575182+0000","last_change":"2023-09-23T15:26:12.496201+0000","last_active":"2023-09-23T15:26:34.575182+0000","last_peered":"2023-09-23T15:26:34.575182+0000","last_clean":"2023-09-23T15:26:10.342805+0000","last_became_active":"2023-09-23T15:26:12.482799+0000","last_became_peered":"2023-09-23T15:26:12.482799+0000","last_unstale":"2023-09-23T15:26:34.575182+0000","last_undegraded":"2023-09-23T15:26:12.388779+0000","last_fullsized":"2023-09-23T15:26:12.364534+0000","mapping_epoch":348,"log_start":"0'0","ondisk_log_start":"0'0","created":339,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:24.405535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271056896,"num_objects":4136,"num_object_clones":0,"num_object_copies":12408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2927,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4136,"num_write_kb":264704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3],"avail_no_missing":["7","3","4"],"object_location_counts":[{"shards":"3,4,7","objects":2927},{"shards":"3,7","objects":1209}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":534,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187743+0000","last_change":"2023-09-23T15:25:42.524359+0000","last_active":"2023-09-23T15:26:29.187743+0000","last_peered":"2023-09-23T15:26:29.187743+0000","last_clean":"2023-09-23T15:26:29.187743+0000","last_became_active":"2023-09-23T15:25:42.523974+0000","last_became_peered":"2023-09-23T15:25:42.523974+0000","last_unstale":"2023-09-23T15:26:29.187743+0000","last_undegraded":"2023-09-23T15:26:29.187743+0000","last_fullsized":"2023-09-23T15:26:29.187743+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:29.868140+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":790,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186486+0000","last_change":"2023-09-23T15:22:27.305419+0000","last_active":"2023-09-23T15:26:29.186486+0000","last_peered":"2023-09-23T15:26:29.186486+0000","last_clean":"2023-09-23T15:26:29.186486+0000","last_became_active":"2023-09-23T15:22:27.304905+0000","last_became_peered":"2023-09-23T15:22:27.304905+0000","last_unstale":"2023-09-23T15:26:29.186486+0000","last_undegraded":"2023-09-23T15:26:29.186486+0000","last_fullsized":"2023-09-23T15:26:29.186486+0000","mapping_epoch":305,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":306,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:38.674220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":374,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.215952+0000","last_change":"2023-09-23T15:26:11.392818+0000","last_active":"2023-09-23T15:26:29.215952+0000","last_peered":"2023-09-23T15:26:29.215952+0000","last_clean":"2023-09-23T15:26:29.215952+0000","last_became_active":"2023-09-23T15:26:11.381709+0000","last_became_peered":"2023-09-23T15:26:11.381709+0000","last_unstale":"2023-09-23T15:26:29.215952+0000","last_undegraded":"2023-09-23T15:26:29.215952+0000","last_fullsized":"2023-09-23T15:26:29.215952+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:25:17.533999+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":78,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187614+0000","last_change":"2023-09-23T15:26:11.381262+0000","last_active":"2023-09-23T15:26:29.187614+0000","last_peered":"2023-09-23T15:26:29.187614+0000","last_clean":"2023-09-23T15:26:29.187614+0000","last_became_active":"2023-09-23T15:26:11.381000+0000","last_became_peered":"2023-09-23T15:26:11.381000+0000","last_unstale":"2023-09-23T15:26:29.187614+0000","last_undegraded":"2023-09-23T15:26:29.187614+0000","last_fullsized":"2023-09-23T15:26:29.187614+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:44.385695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":346,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.213809+0000","last_change":"2023-09-23T15:26:11.382012+0000","last_active":"2023-09-23T15:26:29.213809+0000","last_peered":"2023-09-23T15:26:29.213809+0000","last_clean":"2023-09-23T15:26:29.213809+0000","last_became_active":"2023-09-23T15:26:11.381015+0000","last_became_peered":"2023-09-23T15:26:11.381015+0000","last_unstale":"2023-09-23T15:26:29.213809+0000","last_undegraded":"2023-09-23T15:26:29.213809+0000","last_fullsized":"2023-09-23T15:26:29.213809+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:23.252585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":358,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.194670+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:26:29.194670+0000","last_peered":"2023-09-23T15:26:29.194670+0000","last_clean":"2023-09-23T15:26:29.194670+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:26:29.194670+0000","last_undegraded":"2023-09-23T15:26:29.194670+0000","last_fullsized":"2023-09-23T15:26:29.194670+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":381,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187864+0000","last_change":"2023-09-23T15:26:11.390603+0000","last_active":"2023-09-23T15:26:29.187864+0000","last_peered":"2023-09-23T15:26:29.187864+0000","last_clean":"2023-09-23T15:26:29.187864+0000","last_became_active":"2023-09-23T15:26:11.380210+0000","last_became_peered":"2023-09-23T15:26:11.380210+0000","last_unstale":"2023-09-23T15:26:29.187864+0000","last_undegraded":"2023-09-23T15:26:29.187864+0000","last_fullsized":"2023-09-23T15:26:29.187864+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:35:11.388278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":188,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.189961+0000","last_change":"2023-09-23T15:26:11.374560+0000","last_active":"2023-09-23T15:26:29.189961+0000","last_peered":"2023-09-23T15:26:29.189961+0000","last_clean":"2023-09-23T15:26:29.189961+0000","last_became_active":"2023-09-23T15:26:11.374162+0000","last_became_peered":"2023-09-23T15:26:11.374162+0000","last_unstale":"2023-09-23T15:26:29.189961+0000","last_undegraded":"2023-09-23T15:26:29.189961+0000","last_fullsized":"2023-09-23T15:26:29.189961+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:29:14.653081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":361,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.187998+0000","last_change":"2023-09-23T15:26:11.398686+0000","last_active":"2023-09-23T15:26:29.187998+0000","last_peered":"2023-09-23T15:26:29.187998+0000","last_clean":"2023-09-23T15:26:29.187998+0000","last_became_active":"2023-09-23T15:26:11.384966+0000","last_became_peered":"2023-09-23T15:26:11.384966+0000","last_unstale":"2023-09-23T15:26:29.187998+0000","last_undegraded":"2023-09-23T15:26:29.187998+0000","last_fullsized":"2023-09-23T15:26:29.187998+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:18.728025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210589+0000","last_change":"2023-09-23T15:26:11.364508+0000","last_active":"2023-09-23T15:26:29.210589+0000","last_peered":"2023-09-23T15:26:29.210589+0000","last_clean":"2023-09-23T15:26:29.210589+0000","last_became_active":"2023-09-23T15:26:11.364120+0000","last_became_peered":"2023-09-23T15:26:11.364120+0000","last_unstale":"2023-09-23T15:26:29.210589+0000","last_undegraded":"2023-09-23T15:26:29.210589+0000","last_fullsized":"2023-09-23T15:26:29.210589+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:39:11.688978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":33,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186528+0000","last_change":"2023-09-23T15:25:42.524583+0000","last_active":"2023-09-23T15:26:29.186528+0000","last_peered":"2023-09-23T15:26:29.186528+0000","last_clean":"2023-09-23T15:26:29.186528+0000","last_became_active":"2023-09-23T15:25:42.524186+0000","last_became_peered":"2023-09-23T15:25:42.524186+0000","last_unstale":"2023-09-23T15:26:29.186528+0000","last_undegraded":"2023-09-23T15:26:29.186528+0000","last_fullsized":"2023-09-23T15:26:29.186528+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:16.880947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":107,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.194218+0000","last_change":"2023-09-23T15:25:42.530670+0000","last_active":"2023-09-23T15:26:29.194218+0000","last_peered":"2023-09-23T15:26:29.194218+0000","last_clean":"2023-09-23T15:26:29.194218+0000","last_became_active":"2023-09-23T15:25:42.529030+0000","last_became_peered":"2023-09-23T15:25:42.529030+0000","last_unstale":"2023-09-23T15:26:29.194218+0000","last_undegraded":"2023-09-23T15:26:29.194218+0000","last_fullsized":"2023-09-23T15:26:29.194218+0000","mapping_epoch":343,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":344,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:17.984611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":307,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.210895+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:26:29.210895+0000","last_peered":"2023-09-23T15:26:29.210895+0000","last_clean":"2023-09-23T15:26:29.210895+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:26:29.210895+0000","last_undegraded":"2023-09-23T15:26:29.210895+0000","last_fullsized":"2023-09-23T15:26:29.210895+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":96,"reported_epoch":353,"state":"active+clean","last_fresh":"2023-09-23T15:26:29.186288+0000","last_change":"2023-09-23T15:26:11.368721+0000","last_active":"2023-09-23T15:26:29.186288+0000","last_peered":"2023-09-23T15:26:29.186288+0000","last_clean":"2023-09-23T15:26:29.186288+0000","last_became_active":"2023-09-23T15:26:11.368413+0000","last_became_peered":"2023-09-23T15:26:11.368413+0000","last_unstale":"2023-09-23T15:26:29.186288+0000","last_undegraded":"2023-09-23T15:26:29.186288+0000","last_fullsized":"2023-09-23T15:26:29.186288+0000","mapping_epoch":347,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":348,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:00.765945+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":10068,"num_keys_recovered":30,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":122880,"data_stored":21392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":40908,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3382,"ondisk_log_size":3382,"up":63,"acting":63,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":4425711616,"num_objects":67531,"num_object_clones":0,"num_object_copies":202593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5849,"num_objects_misplaced":1650,"num_objects_unfound":0,"num_objects_dirty":67531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":67531,"num_write_kb":4321984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1166,"num_bytes_recovered":76414976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13430685696,"data_stored":13430685696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":67531,"ondisk_log_size":67531,"up":48,"acting":45,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":130,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201662,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2658264,"kb_used_data":2462800,"kb_used_omap":0,"kb_used_meta":195456,"kb_avail":91086888,"statfs":{"total":95995035648,"available":93272973312,"internally_reserved":0,"allocated":2521907200,"data_stored":2520805995,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":200146944},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:26:27 2023","interfaces":[{"interface":"back","average":{"1min":6.9240000000000004,"5min":6.5869999999999997,"15min":4.2869999999999999},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":62.670999999999999,"5min":226.226,"15min":226.226},"last":2.831},{"interface":"front","average":{"1min":7.1440000000000001,"5min":8.1349999999999998,"15min":4.8979999999999997},"min":{"1min":0.26700000000000002,"5min":0.254,"15min":0.24399999999999999},"max":{"1min":62.823999999999998,"5min":226.619,"15min":226.619},"last":3.165}]},{"osd":1,"last update":"Sat Sep 23 15:26:11 2023","interfaces":[{"interface":"back","average":{"1min":3.3500000000000001,"5min":5.319,"15min":3.085},"min":{"1min":0.30299999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":56.746000000000002,"5min":311.16699999999997,"15min":311.16699999999997},"last":5.0970000000000004},{"interface":"front","average":{"1min":3.3029999999999999,"5min":5.5250000000000004,"15min":3.5470000000000002},"min":{"1min":0.28499999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":56.899999999999999,"5min":311.279,"15min":311.279},"last":5.0570000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:26:27 2023","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":2.0169999999999999,"15min":2.3119999999999998},"min":{"1min":0.30299999999999999,"5min":0.28799999999999998,"15min":0.25800000000000001},"max":{"1min":8.5899999999999999,"5min":8.8219999999999992,"15min":90.227999999999994},"last":4.8769999999999998},{"interface":"front","average":{"1min":1.0169999999999999,"5min":2.1070000000000002,"15min":2.7570000000000001},"min":{"1min":0.33900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.7719999999999998,"5min":10.308999999999999,"15min":90.197999999999993},"last":3.327}]},{"osd":3,"last update":"Sat Sep 23 15:26:04 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":2.1949999999999998,"15min":3.8220000000000001},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.28899999999999998},"max":{"1min":5.4710000000000001,"5min":41.588999999999999,"15min":146.166},"last":5.0099999999999998},{"interface":"front","average":{"1min":1.097,"5min":2.6509999999999998,"15min":3.9820000000000002},"min":{"1min":0.36099999999999999,"5min":0.34200000000000003,"15min":0.32400000000000001},"max":{"1min":5.7400000000000002,"5min":41.542000000000002,"15min":146.18299999999999},"last":4.9160000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:25:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2430000000000001,"5min":1.855,"15min":1.95},"min":{"1min":0.26600000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.867,"5min":16.771999999999998,"15min":120.587},"last":5.0289999999999999},{"interface":"front","average":{"1min":1.413,"5min":1.7270000000000001,"15min":2.0299999999999998},"min":{"1min":0.311,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":9.0649999999999995,"5min":18.023,"15min":121.056},"last":4.9569999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:25:48 2023","interfaces":[{"interface":"back","average":{"1min":0.73399999999999999,"5min":5.2850000000000001,"15min":3.2469999999999999},"min":{"1min":0.36599999999999999,"5min":0.29099999999999998,"15min":0.28999999999999998},"max":{"1min":4.2809999999999997,"5min":224.441,"15min":224.441},"last":4.399},{"interface":"front","average":{"1min":1.3959999999999999,"5min":5.9160000000000004,"15min":3.4790000000000001},"min":{"1min":0.317,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.6539999999999999,"5min":224.28700000000001,"15min":224.28700000000001},"last":2.8639999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:25:31 2023","interfaces":[{"interface":"back","average":{"1min":1.6719999999999999,"5min":2.3290000000000002,"15min":2.2639999999999998},"min":{"1min":0.34799999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.358000000000001,"5min":10.358000000000001,"15min":10.358000000000001},"last":3.218},{"interface":"front","average":{"1min":1.7609999999999999,"5min":2.472,"15min":2.5},"min":{"1min":0.38700000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":6.7050000000000001,"5min":10.336,"15min":10.336},"last":2.7890000000000001}]}]},{"osd":6,"up_from":304,"seq":1305670058035,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":921940,"kb_used_data":816268,"kb_used_omap":13,"kb_used_meta":105650,"kb_avail":92823212,"statfs":{"total":95995035648,"available":95050969088,"internally_reserved":0,"allocated":835858432,"data_stored":834761230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":108185955},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":2.996,"5min":4.7229999999999999,"15min":2.5609999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":46.740000000000002,"5min":184.53200000000001,"15min":184.53200000000001},"last":3.0430000000000001},{"interface":"front","average":{"1min":3.5760000000000001,"5min":4.149,"15min":3.3010000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":46.549999999999997,"5min":86.227999999999994,"15min":86.227999999999994},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.373,"5min":3.7650000000000001,"15min":3.0830000000000002},"min":{"1min":0.29899999999999999,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":6.4470000000000001,"5min":86.534000000000006,"15min":86.534000000000006},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":3.625,"15min":3.1160000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.4770000000000003,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.378}]},{"osd":2,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0189999999999999,"5min":1.6599999999999999,"15min":1.5700000000000001},"min":{"1min":0.38400000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7999999999999998,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.83699999999999997,"5min":1.8120000000000001,"15min":1.7549999999999999},"min":{"1min":0.35199999999999998,"5min":0.307,"15min":0.307},"max":{"1min":2.2330000000000001,"5min":9.1910000000000007,"15min":9.1910000000000007},"last":0.47899999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.0649999999999999,"5min":2.468,"15min":2.3420000000000001},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":4.6340000000000003,"5min":41.261000000000003,"15min":41.261000000000003},"last":0.309},{"interface":"front","average":{"1min":1.9690000000000001,"5min":2.7879999999999998,"15min":2.444},"min":{"1min":0.41299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":8.266,"5min":41.408000000000001,"15min":41.408000000000001},"last":0.63}]},{"osd":4,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":0.70699999999999996,"5min":1.137,"15min":1.0049999999999999},"min":{"1min":0.36599999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":4.5880000000000001,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.845,"15min":1.829},"min":{"1min":0.34000000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":4.8289999999999997,"5min":13.627000000000001,"15min":13.627000000000001},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.1030000000000002,"15min":1.845},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":2.1970000000000001,"5min":13.74,"15min":13.74},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.663,"15min":1.5429999999999999},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.8069999999999999,"5min":15.868,"15min":15.868},"last":0.41299999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:25:37 2023","interfaces":[{"interface":"back","average":{"1min":1.468,"5min":1.766,"15min":1.522},"min":{"1min":0.47099999999999997,"5min":0.373,"15min":0.373},"max":{"1min":6.7210000000000001,"5min":12.023999999999999,"15min":12.023999999999999},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.224,"5min":2.0430000000000001,"15min":1.8959999999999999},"min":{"1min":0.42299999999999999,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.8040000000000003,"5min":12.641999999999999,"15min":12.641999999999999},"last":0.53500000000000003}]}]},{"osd":1,"up_from":183,"seq":785979015336,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3068992,"kb_used_data":2558328,"kb_used_omap":6,"kb_used_meta":510649,"kb_avail":90676160,"statfs":{"total":95995035648,"available":92852387840,"internally_reserved":0,"allocated":2619727872,"data_stored":2618022805,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":522904651},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.69399999999999995,"5min":2.6389999999999998,"15min":3.105},"min":{"1min":0.20100000000000001,"5min":0.19400000000000001,"15min":0.191},"max":{"1min":2.3159999999999998,"5min":60.817999999999998,"15min":60.817999999999998},"last":3.9159999999999999},{"interface":"front","average":{"1min":0.82899999999999996,"5min":3.5600000000000001,"15min":3.8479999999999999},"min":{"1min":0.24399999999999999,"5min":0.221,"15min":0.21099999999999999},"max":{"1min":3.3519999999999999,"5min":63.594999999999999,"15min":63.594999999999999},"last":2.9129999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":2.3109999999999999,"15min":3.8500000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":1.5209999999999999,"5min":12.429,"15min":109.375},"last":3.4350000000000001},{"interface":"front","average":{"1min":0.67100000000000004,"5min":1.913,"15min":3.2189999999999999},"min":{"1min":0.28699999999999998,"5min":0.219,"15min":0.219},"max":{"1min":2.2839999999999998,"5min":12.259,"15min":105.16},"last":3.9489999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.83299999999999996,"5min":2.6619999999999999,"15min":6.4939999999999998},"min":{"1min":0.33300000000000002,"5min":0.30299999999999999,"15min":0.23899999999999999},"max":{"1min":3.0619999999999998,"5min":64.540999999999997,"15min":211.405},"last":3.8860000000000001},{"interface":"front","average":{"1min":1.155,"5min":2.9569999999999999,"15min":6.9320000000000004},"min":{"1min":0.28299999999999997,"5min":0.246,"15min":0.246},"max":{"1min":3.3980000000000001,"5min":67.465000000000003,"15min":210.988},"last":2.9489999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":2.1139999999999999,"15min":3.2639999999999998},"min":{"1min":0.34399999999999997,"5min":0.32100000000000001,"15min":0.29799999999999999},"max":{"1min":4.0519999999999996,"5min":13.379,"15min":80.570999999999998},"last":3.4060000000000001},{"interface":"front","average":{"1min":0.75800000000000001,"5min":2.0880000000000001,"15min":3.2210000000000001},"min":{"1min":0.41699999999999998,"5min":0.313,"15min":0.308},"max":{"1min":2.1190000000000002,"5min":12.279999999999999,"15min":80.75},"last":3.8439999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":3.8500000000000001,"15min":3.2730000000000001},"min":{"1min":0.314,"5min":0.314,"15min":0.27400000000000002},"max":{"1min":5.29,"5min":115.401,"15min":115.401},"last":2.9710000000000001},{"interface":"front","average":{"1min":1.173,"5min":3.7599999999999998,"15min":3.319},"min":{"1min":0.35899999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":4.4269999999999996,"5min":115.169,"15min":115.169},"last":3.706}]},{"osd":6,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.8169999999999999,"5min":3.1000000000000001,"15min":3.0800000000000001},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.7199999999999998,"5min":19.553999999999998,"15min":19.553999999999998},"last":3.4580000000000002},{"interface":"front","average":{"1min":2.242,"5min":2.2999999999999998,"15min":2.3380000000000001},"min":{"1min":0.41299999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":10.029999999999999,"5min":15.605,"15min":15.605},"last":2.9950000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:26:12 2023","interfaces":[{"interface":"back","average":{"1min":0.80100000000000005,"5min":2.7240000000000002,"15min":2.8410000000000002},"min":{"1min":0.46100000000000002,"5min":0.371,"15min":0.371},"max":{"1min":2.73,"5min":19.702999999999999,"15min":19.702999999999999},"last":27.481999999999999},{"interface":"front","average":{"1min":0.86799999999999999,"5min":2.3719999999999999,"15min":2.4889999999999999},"min":{"1min":0.47799999999999998,"5min":0.377,"15min":0.377},"max":{"1min":2.157,"5min":20.722000000000001,"15min":20.722000000000001},"last":27.364000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893385,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2192728,"kb_used_data":1551952,"kb_used_omap":8,"kb_used_meta":640759,"kb_avail":91552424,"statfs":{"total":95995035648,"available":93749682176,"internally_reserved":0,"allocated":1589198848,"data_stored":1587645443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":656137812},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:26:05 2023","interfaces":[{"interface":"back","average":{"1min":8.2409999999999997,"5min":4.9989999999999997,"15min":4.1130000000000004},"min":{"1min":0.28100000000000003,"5min":0.20599999999999999,"15min":0.18099999999999999},"max":{"1min":110.3,"5min":110.3,"15min":110.3},"last":2.7650000000000001},{"interface":"front","average":{"1min":8.2119999999999997,"5min":4.806,"15min":3.915},"min":{"1min":0.28699999999999998,"5min":0.21299999999999999,"15min":0.20999999999999999},"max":{"1min":110.35599999999999,"5min":110.35599999999999,"15min":110.35599999999999},"last":0.223}]},{"osd":2,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":2.2469999999999999,"15min":3.919},"min":{"1min":0.28599999999999998,"5min":0.20200000000000001,"15min":0.189},"max":{"1min":4.5590000000000002,"5min":30.510999999999999,"15min":103.81},"last":1.4810000000000001},{"interface":"front","average":{"1min":0.998,"5min":2.3399999999999999,"15min":3.835},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.2359999999999998,"5min":30.396999999999998,"15min":108.19499999999999},"last":1.421}]},{"osd":3,"last update":"Sat Sep 23 15:25:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":4.4379999999999997,"15min":5.8899999999999997},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.20499999999999999},"max":{"1min":8.5399999999999991,"5min":197.78899999999999,"15min":197.78899999999999},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.597,"5min":4.4870000000000001,"15min":6.3410000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.25700000000000001},"max":{"1min":10.923,"5min":200.58799999999999,"15min":200.58799999999999},"last":0.68700000000000006}]},{"osd":4,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":0.82699999999999996,"5min":2.0459999999999998,"15min":2.0459999999999998},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.28000000000000003},"max":{"1min":3.1920000000000002,"5min":15.358000000000001,"15min":31.341999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.974,"15min":2.3330000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.27800000000000002},"max":{"1min":3.6480000000000001,"5min":9.8249999999999993,"15min":99.168999999999997},"last":0.34799999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":2.3879999999999999,"15min":2.4870000000000001},"min":{"1min":0.38700000000000001,"5min":0.313,"15min":0.28599999999999998},"max":{"1min":28.187999999999999,"5min":28.187999999999999,"15min":109.369},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.8319999999999999,"5min":2.7040000000000002,"15min":2.5099999999999998},"min":{"1min":0.40500000000000003,"5min":0.39000000000000001,"15min":0.312},"max":{"1min":28.056000000000001,"5min":28.056000000000001,"15min":109.387},"last":0.441}]},{"osd":6,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":2.3959999999999999,"15min":2.6720000000000002},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":3.274,"5min":11.989000000000001,"15min":11.989000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.5569999999999999,"5min":2.7669999999999999,"15min":3.1629999999999998},"min":{"1min":0.38500000000000001,"5min":0.371,"15min":0.371},"max":{"1min":6.0720000000000001,"5min":17.385000000000002,"15min":17.385000000000002},"last":0.38500000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.645,"5min":2.5499999999999998,"15min":2.6240000000000001},"min":{"1min":0.40999999999999998,"5min":0.374,"15min":0.36799999999999999},"max":{"1min":9.2550000000000008,"5min":17.712,"15min":99.549999999999997},"last":4.766},{"interface":"front","average":{"1min":1.2350000000000001,"5min":2.5019999999999998,"15min":2.1659999999999999},"min":{"1min":0.46400000000000002,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":3.9369999999999998,"5min":15.407999999999999,"15min":31.225999999999999},"last":0.93100000000000005}]}]},{"osd":2,"up_from":105,"seq":450971566333,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1618924,"kb_used_data":1008164,"kb_used_omap":11,"kb_used_meta":610740,"kb_avail":92126228,"statfs":{"total":95995035648,"available":94337257472,"internally_reserved":0,"allocated":1032359936,"data_stored":1030791152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":625398544},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:26:22 2023","interfaces":[{"interface":"back","average":{"1min":14.243,"5min":8.0700000000000003,"15min":5.9130000000000003},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":235.517,"5min":235.517,"15min":235.517},"last":0.30599999999999999},{"interface":"front","average":{"1min":14.269,"5min":8.2370000000000001,"15min":5.6150000000000002},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.23100000000000001},"max":{"1min":235.33799999999999,"5min":235.33799999999999,"15min":235.33799999999999},"last":0.32500000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":1.0469999999999999,"5min":3.6059999999999999,"15min":4.3109999999999999},"min":{"1min":0.32200000000000001,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":5.5300000000000002,"5min":71.819000000000003,"15min":84.641000000000005},"last":0.97499999999999998},{"interface":"front","average":{"1min":0.77400000000000002,"5min":3.8149999999999999,"15min":3.5750000000000002},"min":{"1min":0.24399999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":2.96,"5min":71.805000000000007,"15min":84.751999999999995},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:25:52 2023","interfaces":[{"interface":"back","average":{"1min":3.2509999999999999,"5min":4.6559999999999997,"15min":5.4500000000000002},"min":{"1min":0.314,"5min":0.23499999999999999,"15min":0.219},"max":{"1min":48.527000000000001,"5min":77.031999999999996,"15min":87.108000000000004},"last":1.0389999999999999},{"interface":"front","average":{"1min":3.7650000000000001,"5min":4.9699999999999998,"15min":5.532},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.20300000000000001},"max":{"1min":48.371000000000002,"5min":76.742000000000004,"15min":86.158000000000001},"last":1.0129999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:26:22 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":1.51,"15min":2.1909999999999998},"min":{"1min":0.33000000000000002,"5min":0.28000000000000003,"15min":0.27200000000000002},"max":{"1min":2.9489999999999998,"5min":16.712,"15min":84.608000000000004},"last":0.37},{"interface":"front","average":{"1min":0.83999999999999997,"5min":1.423,"15min":2.492},"min":{"1min":0.42799999999999999,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":2.9049999999999998,"5min":8.0519999999999996,"15min":87.019999999999996},"last":0.38700000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.2849999999999999,"5min":1.655,"15min":2.3570000000000002},"min":{"1min":0.38600000000000001,"5min":0.32500000000000001,"15min":0.28899999999999998},"max":{"1min":4.4790000000000001,"5min":7.6980000000000004,"15min":84.498000000000005},"last":4.0350000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.7430000000000001,"15min":2.2240000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":5.8529999999999998,"5min":10.052,"15min":84.664000000000001},"last":4.0540000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:26:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1519999999999999,"5min":2.0169999999999999,"15min":2.1949999999999998},"min":{"1min":0.41199999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":2.698,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":1.0529999999999999},{"interface":"front","average":{"1min":1.006,"5min":1.4419999999999999,"15min":1.8480000000000001},"min":{"1min":0.46600000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":2.948,"5min":7.6879999999999997,"15min":7.6879999999999997},"last":0.35399999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:26:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.839,"15min":2.4500000000000002},"min":{"1min":0.45100000000000001,"5min":0.376,"15min":0.34999999999999998},"max":{"1min":2.766,"5min":11.199999999999999,"15min":84.596000000000004},"last":6.9000000000000004},{"interface":"front","average":{"1min":1.3129999999999999,"5min":2.0459999999999998,"15min":2.4350000000000001},"min":{"1min":0.48399999999999999,"5min":0.38800000000000001,"15min":0.38200000000000001},"max":{"1min":5.8959999999999999,"5min":16.074999999999999,"15min":84.783000000000001},"last":4.1550000000000002}]}]},{"osd":3,"up_from":110,"seq":472446402806,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2844452,"kb_used_data":2171868,"kb_used_omap":3,"kb_used_meta":672572,"kb_avail":90900700,"statfs":{"total":95995035648,"available":93082316800,"internally_reserved":0,"allocated":2223992832,"data_stored":2222427952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":688713897},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.796,"5min":3.375,"15min":3.359},"min":{"1min":0.23200000000000001,"5min":0.183,"15min":0.183},"max":{"1min":13.557,"5min":108.318,"15min":108.318},"last":1.6659999999999999},{"interface":"front","average":{"1min":1.9350000000000001,"5min":3.798,"15min":3.6349999999999998},"min":{"1min":0.26900000000000002,"5min":0.21099999999999999,"15min":0.17599999999999999},"max":{"1min":13.445,"5min":108.889,"15min":108.889},"last":1.9379999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:26:11 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":3.6640000000000001,"15min":4.5750000000000002},"min":{"1min":0.25900000000000001,"5min":0.214,"15min":0.214},"max":{"1min":6.952,"5min":126.875,"15min":126.875},"last":1.5109999999999999},{"interface":"front","average":{"1min":1.0169999999999999,"5min":4.2939999999999996,"15min":4.798},"min":{"1min":0.312,"5min":0.25900000000000001,"15min":0.22900000000000001},"max":{"1min":4.0990000000000002,"5min":128.249,"15min":128.249},"last":2.242}]},{"osd":2,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":2.1499999999999999,"15min":3.2509999999999999},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.22},"max":{"1min":8.8460000000000001,"5min":15.048999999999999,"15min":63.009999999999998},"last":3.0510000000000002},{"interface":"front","average":{"1min":1.9239999999999999,"5min":2.5979999999999999,"15min":4.1230000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":13.047000000000001,"5min":36.582000000000001,"15min":119.449},"last":1.9039999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3460000000000001,"5min":1.7190000000000001,"15min":2.2250000000000001},"min":{"1min":0.36699999999999999,"5min":0.33000000000000002,"15min":0.27500000000000002},"max":{"1min":4.899,"5min":36.551000000000002,"15min":54.051000000000002},"last":1.679},{"interface":"front","average":{"1min":1.403,"5min":1.7749999999999999,"15min":2.141},"min":{"1min":0.36699999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0659999999999998,"5min":13.115,"15min":52.228999999999999},"last":1.748}]},{"osd":5,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":3.4670000000000001,"15min":2.7519999999999998},"min":{"1min":0.38900000000000001,"5min":0.31,"15min":0.28100000000000003},"max":{"1min":4.9450000000000003,"5min":148.619,"15min":148.619},"last":1.716},{"interface":"front","average":{"1min":1.3009999999999999,"5min":3.2749999999999999,"15min":2.6339999999999999},"min":{"1min":0.43099999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.1829999999999998,"5min":148.56999999999999,"15min":148.56999999999999},"last":1.542}]},{"osd":6,"last update":"Sat Sep 23 15:26:30 2023","interfaces":[{"interface":"back","average":{"1min":6.5529999999999999,"5min":3.2000000000000002,"15min":2.5499999999999998},"min":{"1min":0.33800000000000002,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":84.665000000000006,"5min":84.665000000000006,"15min":84.665000000000006},"last":1.5680000000000001},{"interface":"front","average":{"1min":6.4349999999999996,"5min":3.5600000000000001,"15min":3.6200000000000001},"min":{"1min":0.41699999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":84.897999999999996,"5min":84.897999999999996,"15min":84.897999999999996},"last":1.7749999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:25:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3959999999999999,"5min":3.1520000000000001,"15min":2.6179999999999999},"min":{"1min":0.38500000000000001,"5min":0.35099999999999998,"15min":0.34899999999999998},"max":{"1min":5.7329999999999997,"5min":145.25,"15min":145.25},"last":1.639},{"interface":"front","average":{"1min":0.89500000000000002,"5min":3.1150000000000002,"15min":2.5070000000000001},"min":{"1min":0.41199999999999998,"5min":0.38700000000000001,"15min":0.34799999999999998},"max":{"1min":3.9769999999999999,"5min":145.00899999999999,"15min":145.00899999999999},"last":1.948}]}]},{"osd":4,"up_from":0,"seq":133143986571,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":39,"seq":167503724934,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2860168,"kb_used_data":2423488,"kb_used_omap":0,"kb_used_meta":436672,"kb_avail":90884984,"statfs":{"total":95995035648,"available":93066223616,"internally_reserved":0,"allocated":2481651712,"data_stored":2480563058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":447152128},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:26:23 2023","interfaces":[{"interface":"back","average":{"1min":6.266,"5min":3.9569999999999999,"15min":3.6240000000000001},"min":{"1min":0.25900000000000001,"5min":0.217,"15min":0.217},"max":{"1min":50.460000000000001,"5min":54.939999999999998,"15min":124.31},"last":196.77000000000001},{"interface":"front","average":{"1min":6.2880000000000003,"5min":4.1639999999999997,"15min":3.6669999999999998},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":50.357999999999997,"5min":55.043999999999997,"15min":124.374},"last":196.91399999999999}]},{"osd":1,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":3.5960000000000001,"5min":4.165,"15min":3.4089999999999998},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":50.444000000000003,"5min":86.900999999999996,"15min":124.38800000000001},"last":193.66499999999999},{"interface":"front","average":{"1min":3.5470000000000002,"5min":2.9060000000000001,"15min":3.2010000000000001},"min":{"1min":0.28499999999999998,"5min":0.28299999999999997,"15min":0.25900000000000001},"max":{"1min":49.253,"5min":49.253,"15min":124.29000000000001},"last":193.52099999999999}]},{"osd":2,"last update":"Sat Sep 23 15:26:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1160000000000001,"5min":1.8500000000000001,"15min":3.0699999999999998},"min":{"1min":0.36899999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":4.1280000000000001,"5min":11.754,"15min":193.00999999999999},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.86299999999999999,"5min":2.1299999999999999,"15min":3.2890000000000001},"min":{"1min":0.33100000000000002,"5min":0.29299999999999998,"15min":0.27900000000000003},"max":{"1min":3.3650000000000002,"5min":10.542999999999999,"15min":193.15700000000001},"last":0.66400000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:25:51 2023","interfaces":[{"interface":"back","average":{"1min":6.3680000000000003,"5min":6.2699999999999996,"15min":6.3899999999999997},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.27500000000000002},"max":{"1min":86.206999999999994,"5min":266.70299999999997,"15min":266.70299999999997},"last":0.629},{"interface":"front","average":{"1min":6.0259999999999998,"5min":7.0590000000000002,"15min":6.6239999999999997},"min":{"1min":0.34000000000000002,"5min":0.312,"15min":0.30199999999999999},"max":{"1min":86.266999999999996,"5min":266.80399999999997,"15min":266.80399999999997},"last":0.64100000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:26:13 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":1.732,"15min":1.869},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26200000000000001},"max":{"1min":1.5089999999999999,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.69799999999999995,"5min":1.7490000000000001,"15min":1.944},"min":{"1min":0.38100000000000001,"5min":0.29599999999999999,"15min":0.26000000000000001},"max":{"1min":3.202,"5min":10.516,"15min":138.49600000000001},"last":0.53900000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:26:32 2023","interfaces":[{"interface":"back","average":{"1min":1.008,"5min":2.044,"15min":2.2789999999999999},"min":{"1min":0.33400000000000002,"5min":0.308,"15min":0.308},"max":{"1min":3.9649999999999999,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":2.1400000000000001,"15min":2.3199999999999998},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.3420000000000001,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.65300000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:25:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.996,"15min":1.3009999999999999},"min":{"1min":0.312,"5min":0.30399999999999999,"15min":0.28999999999999998},"max":{"1min":8.3040000000000003,"5min":15.849,"15min":15.849},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.70499999999999996,"5min":1.7130000000000001,"15min":1.2410000000000001},"min":{"1min":0.34999999999999998,"5min":0.34499999999999997,"15min":0.311},"max":{"1min":2.2989999999999999,"5min":16.134,"15min":16.134},"last":0.67800000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1417412608,"data_stored":1417412608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2559246336,"data_stored":2559246336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":874708992,"data_stored":874708992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2227306496,"data_stored":2227306496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":567869440,"data_stored":567869440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2487156736,"data_stored":2487156736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":836567040,"data_stored":836567040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2460418048,"data_stored":2460418048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:26:42.226 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:26:42.226 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T15:26:42.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:42.360 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:42.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:42 smithi017 ceph-mon[215646]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 850 op/s; 5849/212265 objects degraded (2.756%); 1341/212265 objects misplaced (0.632%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:26:42.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/212265 objects degraded (2.756%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:42 smithi017 ceph-mon[222878]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 850 op/s; 5849/212265 objects degraded (2.756%); 1341/212265 objects misplaced (0.632%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:26:42.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/212265 objects degraded (2.756%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:42.593 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:42 smithi027 ceph-mon[195233]: pgmap v1053: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 53 MiB/s wr, 850 op/s; 5849/212265 objects degraded (2.756%); 1341/212265 objects misplaced (0.632%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T15:26:42.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/212265 objects degraded (2.756%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:42.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:42.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:42.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:42.707 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:42.732 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:42.755 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:42.775 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:42.810 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:42.846 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:42.883 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:73314 9.e 9:7b72513b:::benchmark_data_smithi017_543597_object73313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.884645+0000", 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.027880769999999999, 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027910292, 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.914 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 73314 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884645+0000", 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884645+0000", 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884644+0000", 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884671+0000", 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6650000000000001e-05 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884677+0000", 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.268e-06 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.884681+0000", 2023-09-23T15:26:42.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2300000000000002e-06 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.892669+0000", 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007987872 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.892824+0000", 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000154652 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.893000+0000", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000176219 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.895930+0000", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029294080000000001 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.898370+0000", 2023-09-23T15:26:42.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024408479999999998 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:73321 9.e 9:7a7f0348:::benchmark_data_smithi017_543597_object73320:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.901567+0000", 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.010958706, 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011081051, 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:42.920 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 73321 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901567+0000", 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901567+0000", 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901566+0000", 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901618+0000", 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2237000000000001e-05 2023-09-23T15:26:42.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901630+0000", 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2432999999999999e-05 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901637+0000", 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0759999999999997e-06 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901696+0000", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.817e-05 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.901886+0000", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000190903 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902049+0000", 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000162311 2023-09-23T15:26:42.924 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46157.0:73322 9.9 e355/339 9:9f1669fd:::benchmark_data_smithi017_543597_object73321:head v 355'4583, mlcod=355'4581)", 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.902250+0000", 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.010275133000000001, 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010443099000000001, 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902250+0000", 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902250+0000", 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902249+0000", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902305+0000", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5435e-05 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902315+0000", 2023-09-23T15:26:42.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0665999999999999e-05 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902321+0000", 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7139999999999998e-06 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902367+0000", 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6261000000000003e-05 2023-09-23T15:26:42.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.902388+0000", 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0769000000000001e-05 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46157.0:73325 9.1 e355/343 9:8661ff59:::benchmark_data_smithi017_543597_object73324:head v 355'4634, mlcod=355'4632)", 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.905035+0000", 2023-09-23T15:26:42.929 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0074901840000000004, 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076925439999999999, 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905035+0000", 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905035+0000", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905034+0000", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905065+0000", 2023-09-23T15:26:42.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0477e-05 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905077+0000", 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2303999999999999e-05 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905084+0000", 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9249999999999998e-06 2023-09-23T15:26:42.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905207+0000", 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012323499999999999 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905239+0000", 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1367e-05 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPushReply(9.e 355/348 [PushReplyOp(9:76ca7a76:::benchmark_data_smithi017_543597_object27848:head)])", 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.905117+0000", 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0074083910000000003, 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076452630000000002, 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.934 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905117+0000", 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905117+0000", 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905117+0000", 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905121+0000", 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9740000000000004e-06 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905148+0000", 2023-09-23T15:26:42.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7129999999999999e-05 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905153+0000", 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3079999999999998e-06 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:73310 9.e e355/348 ondisk, result = 0)", 2023-09-23T15:26:42.937 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.905162+0000", 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0073632810000000002, 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076268009999999999, 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905162+0000", 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905162+0000", 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905162+0000", 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905164+0000", 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5330000000000001e-06 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905170+0000", 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1950000000000002e-06 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905173+0000", 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9979999999999999e-06 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911642+0000", 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064691829999999999 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911652+0000", 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9860000000000006e-06 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.941 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:73314 9.e e355/348)", 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.905176+0000", 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0073498640000000002, 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076454440000000004, 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905176+0000", 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905176+0000", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905175+0000", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905177+0000", 2023-09-23T15:26:42.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.751e-06 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905182+0000", 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3839999999999999e-06 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905187+0000", 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3040000000000004e-06 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.944 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:73328 9.3 9:cf1b85e2:::benchmark_data_smithi017_543597_object73327:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e355)", 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.905709+0000", 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0068164200000000001, 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071440310000000003, 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:26:42.945 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 73328 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905709+0000", 2023-09-23T15:26:42.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905709+0000", 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905707+0000", 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:42.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905745+0000", 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8389999999999997e-05 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905758+0000", 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2405999999999999e-05 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905764+0000", 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5429999999999999e-06 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905813+0000", 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8868e-05 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.905954+0000", 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014072800000000001 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,3", 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.906100+0000", 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000145802 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46157.0:73326 9.1 e355/343 9:8db6eb0c:::benchmark_data_smithi017_543597_object73325:head v 355'4635, mlcod=355'4632)", 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.907253+0000", 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0052724310000000002, 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056472500000000004, 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907253+0000", 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907253+0000", 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907252+0000", 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907285+0000", 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3417999999999998e-05 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907298+0000", 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3512999999999999e-05 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.907309+0000", 2023-09-23T15:26:42.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0047999999999999e-05 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.908012+0000", 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070319800000000004 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.908035+0000", 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3125e-05 2023-09-23T15:26:42.954 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:73328 9.3 e355/339)", 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.911034+0000", 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0014915779999999999, 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018988239999999999, 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.955 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911034+0000", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911034+0000", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911033+0000", 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911039+0000", 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9619999999999998e-06 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911052+0000", 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2934e-05 2023-09-23T15:26:42.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911058+0000", 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5540000000000003e-06 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:73321 9.e e355/348)", 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.911461+0000", 2023-09-23T15:26:42.958 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0010640650000000001, 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014972200000000001, 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911461+0000", 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911461+0000", 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911460+0000", 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911473+0000", 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2943e-05 2023-09-23T15:26:42.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911491+0000", 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7274000000000001e-05 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.911497+0000", 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6869999999999997e-06 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.961 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46157.0:73334 9.9 e355/339 9:9409c2c5:::benchmark_data_smithi017_543597_object73333:head v 355'4584, mlcod=355'4581)", 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:42.912045+0000", 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00048004999999999999, 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00094072399999999999, 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912045+0000", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912045+0000", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912045+0000", 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912098+0000", 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3554999999999998e-05 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912111+0000", 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3081000000000001e-05 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:26:42.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912117+0000", 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9830000000000001e-06 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912571+0000", 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045388699999999999 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:42.912594+0000", 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3096999999999999e-05 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.965 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 12 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:26:42.966 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:42.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:42.967 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:42.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:42.970 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:42.978 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.978 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:26:42.979 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:26:43.008 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 73396 73380 79.0243 45.25 0.0172414 0.0126461 2023-09-23T15:26:43.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:43.170 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:43.287 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:26:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3478241769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:43.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:43.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3478241769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:43.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3478241769' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:43.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:43.791 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:43.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:43.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:43.844 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:43.867 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:26:43.901 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:43.973 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:44.008 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 74036 74020 78.3631 40 0.0121796 0.012755 2023-09-23T15:26:44.064 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:44.127 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:44.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:44.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:44.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:44.279 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:44.376 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:44.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:44.492 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:44.515 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:44 smithi017 ceph-mon[222878]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/214269 objects degraded (2.730%); 1341/214269 objects misplaced (0.626%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T15:26:44.524 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:44 smithi017 ceph-mon[215646]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/214269 objects degraded (2.730%); 1341/214269 objects misplaced (0.626%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T15:26:44.533 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:44 smithi027 ceph-mon[195233]: pgmap v1054: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 50 MiB/s wr, 797 op/s; 5849/214269 objects degraded (2.730%); 1341/214269 objects misplaced (0.626%); 3.6 MiB/s, 58 objects/s recovering 2023-09-23T15:26:44.546 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:44.609 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:44.625 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:44.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:44.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:44.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:44.770 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:44.961 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:44.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:44.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:45.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:26:45.009558+0000 min lat: 0.00111339 max lat: 2.1508 avg lat: 0.0128211 2023-09-23T15:26:45.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:26:45.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 74905 74889 77.9624 54.3125 0.00805106 0.0128211 2023-09-23T15:26:45.068 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:45.133 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:45.888 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:26:46.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 75715 75699 77.5144 50.625 0.0155417 0.0128953 2023-09-23T15:26:46.449 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:46.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:46.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:46.571 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:46.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:46 smithi027 ceph-mon[195233]: pgmap v1055: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 5849/220104 objects degraded (2.657%); 1035/220104 objects misplaced (0.470%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:46.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:46 smithi017 ceph-mon[215646]: pgmap v1055: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 5849/220104 objects degraded (2.657%); 1035/220104 objects misplaced (0.470%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:46.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:46 smithi017 ceph-mon[222878]: pgmap v1055: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 16 GiB used, 609 GiB / 626 GiB avail; 48 MiB/s wr, 763 op/s; 5849/220104 objects degraded (2.657%); 1035/220104 objects misplaced (0.470%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:46.695 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:46.856 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:46.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:46.908 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:46.961 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:47.009 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 76417 76401 76.972 43.875 0.0173469 0.0129872 2023-09-23T15:26:47.043 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:47.062 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:26:47.062 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:26:47.112 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:47.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:47.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:47.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:47.260 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:47.340 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:47.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:47.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:47.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:47.513 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:47 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:47 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/220104 objects degraded (2.657%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4003230836' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:47.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:47.615 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:47.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:47.619 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:47.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:47.759 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/220104 objects degraded (2.657%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4003230836' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:47.927 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:47.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/220104 objects degraded (2.657%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:47.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4003230836' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:47.928 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T15:26:48.012 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 77112 77096 76.438 43.4375 0.0168224 0.0130773 2023-09-23T15:26:48.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:26:48.042 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:48.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:48.151 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:48.194 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:48] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T15:26:48.248 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:48.280 INFO:teuthology.orchestra.run.smithi017.stderr:set pool 1 pg_num to 31 2023-09-23T15:26:48.482 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:26:48.802 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s; 5849/222687 objects degraded (2.627%); 1035/222687 objects misplaced (0.465%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:48.802 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T15:26:48.802 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T15:26:48.803 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2060086654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.803 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.803 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:48 smithi027 ceph-mon[195233]: failed to encode map e356 with expected crc 2023-09-23T15:26:48.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s; 5849/222687 objects degraded (2.627%); 1035/222687 objects misplaced (0.465%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:48.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T15:26:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T15:26:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2060086654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[222878]: failed to encode map e356 with expected crc 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: pgmap v1056: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 47 MiB/s wr, 747 op/s; 5849/222687 objects degraded (2.627%); 1035/222687 objects misplaced (0.465%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2060086654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:26:48.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:48 smithi017 ceph-mon[215646]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.013 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 77838 77822 75.9506 45.375 0.00491761 0.0131612 2023-09-23T15:26:49.268 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:26:49.617 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:49.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:49.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:49.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.821 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T15:26:49.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T15:26:49.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:49.833 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: failed to encode map e356 with expected crc 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T15:26:49.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:50.012 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 78617 78601 75.5313 48.6875 0.0129793 0.0132265 2023-09-23T15:26:50.111 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:50.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:50.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:50.219 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:50.325 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:50.429 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:26:50.429 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:26:50.613 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:50.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:50.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:50.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:50.793 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: pgmap v1058: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 5849/224850 objects degraded (2.601%); 1035/224850 objects misplaced (0.460%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T15:26:50.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:50 smithi027 ceph-mon[195233]: failed to encode map e358 with expected crc 2023-09-23T15:26:50.883 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:50.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: pgmap v1058: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 5849/224850 objects degraded (2.601%); 1035/224850 objects misplaced (0.460%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T15:26:50.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[215646]: failed to encode map e358 with expected crc 2023-09-23T15:26:50.892 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: pgmap v1058: 109 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 106 active+clean; 4.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 777 op/s; 5849/224850 objects degraded (2.601%); 1035/224850 objects misplaced (0.460%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: failed to encode map e357 with expected crc 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T15:26:50.892 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:50 smithi017 ceph-mon[222878]: failed to encode map e358 with expected crc 2023-09-23T15:26:50.993 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:51.006 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:26:51.015 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 16 79352 79336 75.0803 45.9375 0.0185312 0.0133131 2023-09-23T15:26:51.026 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:51.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:26:51.054 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:26:51.073 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:26:51.092 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:26:51.110 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:26:51.129 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:26:51.180 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:51.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:51.188 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:26:51.206 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.207 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:26:51.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:51.331 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: pgmap v1061: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s; 5849/234090 objects degraded (2.499%); 699/234090 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:51.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2666794198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: pgmap v1061: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s; 5849/234090 objects degraded (2.499%); 699/234090 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:51.583 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2666794198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:51.651 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:51.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:51.657 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:26:51.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:51.791 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:51.862 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:51.870 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: failed to encode map e358 with expected crc 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: pgmap v1061: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 10 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 49 MiB/s wr, 776 op/s; 5849/234090 objects degraded (2.499%); 699/234090 objects misplaced (0.299%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2666794198' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:26:51.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:52.015 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 80141 80125 74.6958 49.3125 0.0125609 0.013382 2023-09-23T15:26:52.081 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:52.200 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:52.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:52.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:52.309 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:52.400 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:52.513 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:52.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:52.573 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:52.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:26:52.672 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:52.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:26:52.743 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:26:52.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:52.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:26:52.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:52.964 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:53.011 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/234090 objects degraded (2.499%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:53.011 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/234090 objects degraded (2.499%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:53.015 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 80972 80956 74.3612 51.9375 0.0412885 0.0134432 2023-09-23T15:26:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/234090 objects degraded (2.499%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:53.205 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:53.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:53.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:53.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:53.352 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:53.440 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:53.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:53.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:53.547 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:53.572 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:53.616 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:53.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:53.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:53.723 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:53.749 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:53.836 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:53.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:53.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:54.016 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:53 smithi017 ceph-mon[215646]: pgmap v1062: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 9 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 5849/236175 objects degraded (2.477%); 699/236175 objects misplaced (0.296%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:54.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 81678 81662 73.9203 44.125 0.0132484 0.0135228 2023-09-23T15:26:54.023 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:54.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:53 smithi017 ceph-mon[222878]: pgmap v1062: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 9 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 5849/236175 objects degraded (2.477%); 699/236175 objects misplaced (0.296%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:53 smithi027 ceph-mon[195233]: pgmap v1062: 119 pgs: 1 active+recovering+undersized+remapped, 9 peering, 9 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.8 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 748 op/s; 5849/236175 objects degraded (2.477%); 699/236175 objects misplaced (0.296%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T15:26:54.129 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:54.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:54.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:54.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:26:54.260 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:26:54.284 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:54.298 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:54.363 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:26:54.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:54.370 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:26:54.471 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:54.517 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:54.576 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:54.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:54.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:54.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:54.772 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:54.822 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:26:54.822 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:26:54.822 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 2, 3, 4, 5, 7, 1, 6] 2023-09-23T15:26:54.823 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T15:26:54.823 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T15:26:55.005 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:55.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:55.014 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:55.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 82455 82439 73.5582 48.5625 0.0117995 0.0135891 2023-09-23T15:26:55.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:55.198 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:26:54 smithi027 systemd[1]: Stopping Ceph osd.7 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:26:55.213 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:55.218 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:26:55.218 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82591 9.4 9:2dad9a5e:::benchmark_data_smithi017_543597_object82590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.199267+0000", 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.014430563, 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014465240000000001, 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.219 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82591 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199267+0000", 2023-09-23T15:26:55.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199267+0000", 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199265+0000", 2023-09-23T15:26:55.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199293+0000", 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8053000000000001e-05 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199311+0000", 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7649e-05 2023-09-23T15:26:55.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199319+0000", 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7169999999999997e-06 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199372+0000", 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2979000000000002e-05 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199576+0000", 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020381899999999999 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.199729+0000", 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153194 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:55.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.205716+0000", 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059864999999999996 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.205761+0000", 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5058999999999997e-05 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46157.0:82575 9.4 e358/348)", 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.209317+0000", 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0043798350000000003, 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045169520000000003, 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209317+0000", 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209317+0000", 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209316+0000", 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209322+0000", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9900000000000002e-06 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209335+0000", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2636000000000001e-05 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209343+0000", 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4460000000000004e-06 2023-09-23T15:26:55.228 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46157.0:82584 9.4 e358/348)", 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.209355+0000", 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0043425989999999999, 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045280160000000002, 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "queued for pg", 2023-09-23T15:26:55.229 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209355+0000", 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209355+0000", 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209354+0000", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209357+0000", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4719999999999998e-06 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209362+0000", 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3090000000000003e-06 2023-09-23T15:26:55.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209366+0000", 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0099999999999997e-06 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82595 9.2 9:403ee57f:::benchmark_data_smithi017_543597_object82594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.209390+0000", 2023-09-23T15:26:55.232 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0043074230000000003, 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045494450000000001, 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82595 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209390+0000", 2023-09-23T15:26:55.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209390+0000", 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209388+0000", 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209438+0000", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0148999999999997e-05 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209449+0000", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0183999999999999e-05 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209455+0000", 2023-09-23T15:26:55.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2790000000000004e-06 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209522+0000", 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7341999999999993e-05 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209683+0000", 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016058500000000001 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.209835+0000", 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015229999999999999 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:82594 9.3 e358/339 9:c90bed5f:::benchmark_data_smithi017_543597_object82593:head v 358'5127, mlcod=358'5125)", 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.210525+0000", 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0031718779999999999, 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037830170000000001, 2023-09-23T15:26:55.237 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210525+0000", 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210525+0000", 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210525+0000", 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210570+0000", 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5899000000000002e-05 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210585+0000", 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4569999999999999e-05 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.210592+0000", 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6789999999999999e-06 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.211450+0000", 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00085863400000000005 2023-09-23T15:26:55.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.211473+0000", 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2819e-05 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82604 9.0 9:0b223b55:::benchmark_data_smithi017_543597_object82603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.212533+0000", 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.001164521, 2023-09-23T15:26:55.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001861265, 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82604 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212533+0000", 2023-09-23T15:26:55.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212533+0000", 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212531+0000", 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212605+0000", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4399000000000004e-05 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212621+0000", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5806000000000001e-05 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212631+0000", 2023-09-23T15:26:55.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5210000000000008e-06 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212688+0000", 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7460000000000002e-05 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212872+0000", 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000184017 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.213037+0000", 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000164771 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82605 9.2 9:4a56a8e3:::benchmark_data_smithi017_543597_object82604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.212649+0000", 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0010478880000000001, 2023-09-23T15:26:55.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018265250000000001, 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82605 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212649+0000", 2023-09-23T15:26:55.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212649+0000", 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212649+0000", 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212696+0000", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7613e-05 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212705+0000", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6270000000000006e-06 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212711+0000", 2023-09-23T15:26:55.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9320000000000001e-06 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.212878+0000", 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016668 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.213011+0000", 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013396 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.213148+0000", 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013698000000000001 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82575 9.4 9:240819da:::benchmark_data_smithi017_543597_object82574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.185031+0000", 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.028666513000000001, 2023-09-23T15:26:55.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029527478999999999, 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82575 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185031+0000", 2023-09-23T15:26:55.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185031+0000", 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185029+0000", 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185067+0000", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8056999999999998e-05 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185083+0000", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6609999999999999e-05 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185093+0000", 2023-09-23T15:26:55.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0135999999999999e-05 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185189+0000", 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5861999999999998e-05 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185396+0000", 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020659899999999999 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185587+0000", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000191118 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.198341+0000", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012754041000000001 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.202363+0000", 2023-09-23T15:26:55.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040217179999999996 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:82584 9.4 9:27903554:::benchmark_data_smithi017_543597_object82583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.185241+0000", 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.028455975000000001, 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.029407679999999999, 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:26:55.257 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 82584 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185241+0000", 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:26:55.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185241+0000", 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185240+0000", 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185264+0000", 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3439999999999999e-05 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185272+0000", 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2020000000000004e-06 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.185279+0000", 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7780000000000002e-06 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:26:55.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.188401+0000", 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031223269999999998 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.188559+0000", 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000158221 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.188750+0000", 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019056300000000001 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.198349+0000", 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095990339999999993 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.202424+0000", 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040744850000000001 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.262 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:26:55.263 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:26:55.263 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 9 2023-09-23T15:26:55.263 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:26:55.360 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:55.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:55.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:55.472 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:55.510 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:26:55 smithi027 conmon[141165]: 2023-09-23T15:26:55.197+0000 7f30b34f4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:26:55.510 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:26:55 smithi027 conmon[141165]: 2023-09-23T15:26:55.197+0000 7f30b34f4700 -1 osd.7 358 *** Got signal Terminated *** 2023-09-23T15:26:55.510 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:26:55 smithi027 conmon[141165]: 2023-09-23T15:26:55.197+0000 7f30b34f4700 -1 osd.7 358 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:26:55.523 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:55.606 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:55.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:55.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:55.708 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:55.736 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:55.781 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:55.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:55.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:55.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:55.895 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:55 smithi017 ceph-mon[215646]: pgmap v1063: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s; 5849/242619 objects degraded (2.411%); 359/242619 objects misplaced (0.148%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T15:26:55.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:55 smithi017 ceph-mon[215646]: osd.7 marked itself down and dead 2023-09-23T15:26:55.923 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:55.971 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:55.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:55.974 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:56.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 83124 83108 73.1112 41.8125 0.0371142 0.0136501 2023-09-23T15:26:56.075 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:26:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:55 smithi027 ceph-mon[195233]: pgmap v1063: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s; 5849/242619 objects degraded (2.411%); 359/242619 objects misplaced (0.148%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T15:26:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:55 smithi027 ceph-mon[195233]: osd.7 marked itself down and dead 2023-09-23T15:26:56.102 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:56.145 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:26:56.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:56.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:26:56.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:55 smithi017 ceph-mon[222878]: pgmap v1063: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 2 unknown, 2 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 904 op/s; 5849/242619 objects degraded (2.411%); 359/242619 objects misplaced (0.148%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T15:26:56.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:55 smithi017 ceph-mon[222878]: osd.7 marked itself down and dead 2023-09-23T15:26:56.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:56.279 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:56.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:26:56.734 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:56.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:56.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:56.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:56.848 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:56.849 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:26:56 smithi027 bash[340781]: 8566abdd88d9ee36984faf79456419fb63bda7cef4153cb4e79554b9986e8dab 2023-09-23T15:26:56.878 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:56.912 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: osdmap e359: 8 total, 7 up, 7 in 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: osdmap e359: 8 total, 7 up, 7 in 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:56 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:26:56.990 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:57.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:57.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 83145 83129 72.1146 1.3125 0.0532333 0.0137663 2023-09-23T15:26:57.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:57.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:57.283 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:57.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:57.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:57.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T15:26:57.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: osdmap e359: 8 total, 7 up, 7 in 2023-09-23T15:26:57.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:56 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:26:57.389 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:26:57.420 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:57.469 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:26:57.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:57.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:26:57.572 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:57.603 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:57.654 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:57.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:57.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:57.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:26:57.786 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:57.834 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:26:57.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:57.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:26:57.921 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:26:57.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:26:57.967 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:58.012 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:26:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:58.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:26:58.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 83252 83236 71.2188 6.6875 0.00483742 0.01376 2023-09-23T15:26:58.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:26:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:26:58] "GET /metrics HTTP/1.1" 200 36386 "" "Prometheus/2.43.0" 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: pgmap v1065: 119 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 22 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 5849/245433 objects degraded (2.383%); 359/245433 objects misplaced (0.146%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: osdmap e360: 8 total, 7 up, 7 in 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 5849/245433 objects degraded (2.383%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: pgmap v1065: 119 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 22 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 5849/245433 objects degraded (2.383%); 359/245433 objects misplaced (0.146%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: osdmap e360: 8 total, 7 up, 7 in 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 5849/245433 objects degraded (2.383%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:58.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:58.145 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:58.198 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:58.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:58.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:58.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:26:58.330 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:58.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: pgmap v1065: 119 pgs: 1 stale+active+recovering+undersized+remapped, 1 stale+active+recovery_wait+undersized+degraded+remapped, 22 stale+active+clean, 1 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 908 op/s; 5849/245433 objects degraded (2.383%); 359/245433 objects misplaced (0.146%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: osdmap e360: 8 total, 7 up, 7 in 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: failed to encode map e360 with expected crc 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 5849/245433 objects degraded (2.383%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:26:58.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:26:59.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 83272 83256 70.2738 1.25 0.00342964 0.0138696 2023-09-23T15:26:59.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:59.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: osdmap e361: 8 total, 7 up, 7 in 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: 1.5 scrub starts 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[215646]: 1.5 scrub ok 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: osdmap e361: 8 total, 7 up, 7 in 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: 1.5 scrub starts 2023-09-23T15:26:59.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:26:58 smithi017 ceph-mon[222878]: 1.5 scrub ok 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1566470130' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: osdmap e361: 8 total, 7 up, 7 in 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: failed to encode map e361 with expected crc 2023-09-23T15:26:59.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:26:59.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:26:59.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: 1.5 scrub starts 2023-09-23T15:26:59.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:26:58 smithi027 ceph-mon[195233]: 1.5 scrub ok 2023-09-23T15:26:59.492 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:26:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:59.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:26:59.516 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:26:59.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:59.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T15:26:59.628 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:59.652 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:26:59.652 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T15:26:59.733 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:59.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:59.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:26:59.838 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:26:59.876 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:26:59.940 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:26:59.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:26:59.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:00.018 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 83272 83256 69.3373 0 - 0.0138696 2023-09-23T15:27:00.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:00.078 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:00.287 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:00.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:00.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:00.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[215646]: pgmap v1068: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 15 stale+active+clean, 5 active+undersized+degraded, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 13290/247635 objects degraded (5.367%); 2797/247635 objects misplaced (1.129%); 5.1 MiB/s, 89 objects/s recovering 2023-09-23T15:27:00.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[215646]: 1.a scrub starts 2023-09-23T15:27:00.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[215646]: 1.a scrub ok 2023-09-23T15:27:00.420 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2497204456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[222878]: pgmap v1068: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 15 stale+active+clean, 5 active+undersized+degraded, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 13290/247635 objects degraded (5.367%); 2797/247635 objects misplaced (1.129%); 5.1 MiB/s, 89 objects/s recovering 2023-09-23T15:27:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[222878]: 1.a scrub starts 2023-09-23T15:27:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[222878]: 1.a scrub ok 2023-09-23T15:27:00.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2497204456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:00.423 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:00.475 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:00.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:00.580 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:00 smithi027 ceph-mon[195233]: pgmap v1068: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 15 stale+active+clean, 5 active+undersized+degraded, 90 active+clean; 5.0 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 40 MiB/s wr, 636 op/s; 13290/247635 objects degraded (5.367%); 2797/247635 objects misplaced (1.129%); 5.1 MiB/s, 89 objects/s recovering 2023-09-23T15:27:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:00 smithi027 ceph-mon[195233]: 1.a scrub starts 2023-09-23T15:27:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:00 smithi027 ceph-mon[195233]: 1.a scrub ok 2023-09-23T15:27:00.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2497204456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:00.609 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:00.660 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:00.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:00.766 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:00.797 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:00.846 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:00.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:00.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:00.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:00.978 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 83272 83256 68.4255 0 - 0.0138696 2023-09-23T15:27:01.029 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:01.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:01.161 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.207 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:01.312 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:01.339 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.384 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:01.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.389 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:01.489 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:01.495 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:27:01.524 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.588 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:01.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:01.693 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:01.719 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.765 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:01.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:01.897 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:01.941 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:01.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:01.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:02.016 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:02.018 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:02.018 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:27:02.018 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:27:02.018 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:27:02.018 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:02.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 83272 83256 67.5373 0 - 0.0138696 2023-09-23T15:27:02.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:02.085 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:02.193 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:02.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:02.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:02.298 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:02.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[215646]: pgmap v1069: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 MiB/s wr, 328 op/s; 28776/248532 objects degraded (11.578%); 2797/248532 objects misplaced (1.125%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T15:27:02.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[215646]: 1.15 scrub starts 2023-09-23T15:27:02.325 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[215646]: 1.15 scrub ok 2023-09-23T15:27:02.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[222878]: pgmap v1069: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 MiB/s wr, 328 op/s; 28776/248532 objects degraded (11.578%); 2797/248532 objects misplaced (1.125%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T15:27:02.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[222878]: 1.15 scrub starts 2023-09-23T15:27:02.326 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:02 smithi017 ceph-mon[222878]: 1.15 scrub ok 2023-09-23T15:27:02.328 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:02.383 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:02.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:02.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:02.487 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:02.519 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:02.572 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:02.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:02.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:02 smithi027 ceph-mon[195233]: pgmap v1069: 119 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 21 MiB/s wr, 328 op/s; 28776/248532 objects degraded (11.578%); 2797/248532 objects misplaced (1.125%); 1.6 MiB/s, 32 objects/s recovering 2023-09-23T15:27:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:02 smithi027 ceph-mon[195233]: 1.15 scrub starts 2023-09-23T15:27:02.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:02 smithi027 ceph-mon[195233]: 1.15 scrub ok 2023-09-23T15:27:02.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:02.703 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:02.749 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:02.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:02.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:02.855 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:02.882 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:02.927 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:02.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:02.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:03.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 83272 83256 66.6719 0 - 0.0138696 2023-09-23T15:27:03.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:03.063 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:03.316 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:03.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[215646]: 1.8 scrub starts 2023-09-23T15:27:03.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[215646]: 1.8 scrub ok 2023-09-23T15:27:03.318 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 28776/248532 objects degraded (11.578%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:03.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[222878]: 1.8 scrub starts 2023-09-23T15:27:03.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[222878]: 1.8 scrub ok 2023-09-23T15:27:03.319 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:03 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 28776/248532 objects degraded (11.578%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:03.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:03.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:03.348 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:27:03.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:03.462 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:03 smithi027 ceph-mon[195233]: 1.8 scrub starts 2023-09-23T15:27:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:03 smithi027 ceph-mon[195233]: 1.8 scrub ok 2023-09-23T15:27:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:03 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 28776/248532 objects degraded (11.578%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:03.671 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:03.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:03.676 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:03.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:03.809 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:04.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 83272 83256 65.8283 0 - 0.0138696 2023-09-23T15:27:04.446 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:04.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:04.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:04.550 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:04.588 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:04 smithi017 ceph-mon[222878]: pgmap v1070: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 7.6 MiB/s, 127 objects/s recovering 2023-09-23T15:27:04.589 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:04 smithi017 ceph-mon[215646]: pgmap v1070: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 7.6 MiB/s, 127 objects/s recovering 2023-09-23T15:27:04.593 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:04 smithi027 ceph-mon[195233]: pgmap v1070: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 9.4 MiB/s wr, 151 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 7.6 MiB/s, 127 objects/s recovering 2023-09-23T15:27:04.613 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:04.653 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [7] live_osds: [0, 2, 3, 4, 5, 1, 6] 2023-09-23T15:27:04.653 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:27:04.653 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T15:27:04.653 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 1.000000 2023-09-23T15:27:04.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T15:27:04.691 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:04.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:04.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:04.795 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:04.822 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:04.867 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:04.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:04.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:04.974 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:05.001 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:05.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:27:05.020087+0000 min lat: 0.00111339 max lat: 2.16391 avg lat: 0.0138696 2023-09-23T15:27:05.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:27:05.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 83272 83256 65.0058 0 - 0.0138696 2023-09-23T15:27:05.045 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:05.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:05.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:05.177 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:05.222 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:05.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:05.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:05.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:05.370 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:05.473 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:05.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:05.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:05.577 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:05.604 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:05.651 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:05.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:05.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:05.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:05.785 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:05.837 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:05.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:05.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:05.941 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:05.970 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:06.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 83272 83256 64.2037 0 - 0.0138696 2023-09-23T15:27:06.020 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:06.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:06.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:06.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:06.154 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:06.252 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:27:06.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:06 smithi017 ceph-mon[222878]: pgmap v1071: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 6.6 MiB/s, 110 objects/s recovering 2023-09-23T15:27:06.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:06.276 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:06 smithi017 ceph-mon[215646]: pgmap v1071: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 6.6 MiB/s, 110 objects/s recovering 2023-09-23T15:27:06.276 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:06.475 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:06.504 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:06.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:06.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:06 smithi027 ceph-mon[195233]: pgmap v1071: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 8.2 MiB/s wr, 130 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 6.6 MiB/s, 110 objects/s recovering 2023-09-23T15:27:06.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:06.611 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:06.637 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:06.683 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:06.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:06.788 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:06.815 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:06.862 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:06.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:06.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:06.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:06.994 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:07.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 83272 83256 63.4211 0 - 0.0138696 2023-09-23T15:27:07.041 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:07.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:07.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:07.147 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:07.174 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:07.219 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:07.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:07.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:07.324 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:27:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:07.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: osdmap e362: 8 total, 7 up, 7 in 2023-09-23T15:27:07.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 36636/248574 objects degraded (14.738%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:07.357 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: osdmap e362: 8 total, 7 up, 7 in 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.567 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 36636/248574 objects degraded (14.738%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1781939989' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: osdmap e362: 8 total, 7 up, 7 in 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: failed to encode map e362 with expected crc 2023-09-23T15:27:07.568 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 36636/248574 objects degraded (14.738%), 16 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:07.575 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:07.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:07.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:07.707 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:07.751 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:07.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:07.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:07.893 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165868 7.d e271/223 7:b3afc17b:::benchmark_data_smithi017_422824_object71998:head v 271'10544, mlcod=271'10542)", 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476582+0000", 2023-09-23T15:27:08.012 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 406.53295787899998, 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27746355, 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476582+0000", 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.013 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476582+0000", 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476582+0000", 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476588+0000", 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6220000000000003e-06 2023-09-23T15:27:08.014 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476599+0000", 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.046e-05 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476605+0000", 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6390000000000003e-06 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.015 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478296+0000", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001691166 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478316+0000", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9196000000000001e-05 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754009+0000", 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27569312699999998 2023-09-23T15:27:08.016 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754046+0000", 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6924000000000001e-05 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45479.0:165871 7.5 e271/223 7:a7f75625:::benchmark_data_smithi017_422824_object72001:head v 271'10293, mlcod=271'10289)", 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.480536+0000", 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 406.52900377899999, 2023-09-23T15:27:08.017 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27361333700000001, 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480536+0000", 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480536+0000", 2023-09-23T15:27:08.018 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480535+0000", 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480544+0000", 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3149999999999992e-06 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.019 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480555+0000", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1138000000000001e-05 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.480562+0000", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2930000000000003e-06 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751262+0000", 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27069967900000003 2023-09-23T15:27:08.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751289+0000", 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6951e-05 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754127+0000", 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028384859999999999 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754150+0000", 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2289999999999998e-05 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165873 7.b 7:dddf4601:::benchmark_data_smithi017_422824_object72003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.482147+0000", 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 406.52739349500001, 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27301487800000002, 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.022 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165873 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482147+0000", 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482147+0000", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482145+0000", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482152+0000", 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9469999999999997e-06 2023-09-23T15:27:08.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751205+0000", 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.269053343 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751227+0000", 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1301000000000002e-05 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751286+0000", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9052999999999999e-05 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751440+0000", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015447700000000001 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.751546+0000", 2023-09-23T15:27:08.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010531000000000001 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.753837+0000", 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022915639999999998 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.755048+0000", 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012104380000000001 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.755085+0000", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7449e-05 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.755128+0000", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2879000000000001e-05 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.755161+0000", 2023-09-23T15:27:08.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3494999999999997e-05 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:45140 8.7 8:e0333dbe:::benchmark_data_smithi017_485823_object45139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:51.039290+0000", 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 316.97025028799999, 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29361414200000002, 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.029 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 45140 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.039290+0000", 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.039290+0000", 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.039288+0000", 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.039326+0000", 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8142000000000002e-05 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273208+0000", 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.233882531 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273223+0000", 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4252e-05 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273364+0000", 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141428 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273573+0000", 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020856999999999999 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273722+0000", 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014971099999999999 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.331080+0000", 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.057357938999999997 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.332786+0000", 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017056409999999999 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.332828+0000", 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1894000000000002e-05 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.332904+0000", 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5949000000000001e-05 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:45138 8.5 e295/288 8:a64e88ac:::benchmark_data_smithi017_485823_object45137:head v 295'2724, mlcod=295'2722)", 2023-09-23T15:27:08.035 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:51.041041+0000", 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 316.96849890099998, 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.289964901, 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.041041+0000", 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.041041+0000", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.041040+0000", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.041080+0000", 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9399e-05 2023-09-23T15:27:08.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273216+0000", 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23213619499999999 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273226+0000", 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8600000000000005e-06 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273283+0000", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6944999999999998e-05 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.273314+0000", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1192000000000002e-05 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.330958+0000", 2023-09-23T15:27:08.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.057644046999999997 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:51.331006+0000", 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8019999999999998e-05 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.040 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:86742 8.1 8:8e1928b6:::benchmark_data_smithi017_485823_object86741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:45.125519+0000", 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 262.88402128799999, 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27500539899999998, 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 86742 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125519+0000", 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125519+0000", 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125517+0000", 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125542+0000", 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4853000000000001e-05 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125557+0000", 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5214999999999999e-05 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125565+0000", 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7910000000000005e-06 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125615+0000", 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0112999999999999e-05 2023-09-23T15:27:08.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125737+0000", 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121995 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.125854+0000", 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116729 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.129178+0000", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003324352 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.129527+0000", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00034862199999999998 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400443+0000", 2023-09-23T15:27:08.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.270916457 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400486+0000", 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3276999999999999e-05 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400524+0000", 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7833999999999997e-05 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.047 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:86750 8.1 8:826bfe54:::benchmark_data_smithi017_485823_object86749:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e309)", 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:22:45.134445+0000", 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 262.87509522800002, 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26619590599999998, 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.048 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 86750 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134445+0000", 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134445+0000", 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134444+0000", 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134466+0000", 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2401000000000001e-05 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134485+0000", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8542e-05 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134494+0000", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2199999999999998e-06 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134541+0000", 2023-09-23T15:27:08.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7373999999999997e-05 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134661+0000", 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012018 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.134768+0000", 2023-09-23T15:27:08.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000106251 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.137709+0000", 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0029415470000000001 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.139312+0000", 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016025049999999999 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.053 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400563+0000", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26125187999999999 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400607+0000", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3080000000000001e-05 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:22:45.400641+0000", 2023-09-23T15:27:08.054 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4125999999999998e-05 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:150653 8.7 8:e4867400:::benchmark_data_smithi017_485823_object60516:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:29.944419+0000", 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.06512102600001, 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27127222200000001, 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.055 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150653 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944419+0000", 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.056 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944419+0000", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944418+0000", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944423+0000", 2023-09-23T15:27:08.057 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6999999999999999e-06 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944435+0000", 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1922e-05 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944442+0000", 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3470000000000001e-06 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.058 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.946619+0000", 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002176552 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.946752+0000", 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013332900000000001 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,6", 2023-09-23T15:27:08.059 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.946891+0000", 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013904799999999999 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951521+0000", 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046300530000000003 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.959314+0000", 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077931279999999999 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.060 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213594+0000", 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25428015300000001 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213648+0000", 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3328999999999997e-05 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.061 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.215691+0000", 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020433719999999999 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:150655 8.1 8:8d4f1b7a:::benchmark_data_smithi017_485823_object60518:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:29.944466+0000", 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.06507440600001, 2023-09-23T15:27:08.062 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27511296600000001, 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150655 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944466+0000", 2023-09-23T15:27:08.063 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944466+0000", 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944465+0000", 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.064 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944468+0000", 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1889999999999999e-06 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944475+0000", 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1130000000000003e-06 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.065 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.944481+0000", 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9170000000000003e-06 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.948265+0000", 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003783547 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.948370+0000", 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105538 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.066 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.948492+0000", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000121241 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.955256+0000", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067647849999999997 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.962552+0000", 2023-09-23T15:27:08.067 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072958959999999996 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219491+0000", 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25693880600000002 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219541+0000", 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9821000000000003e-05 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.068 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219579+0000", 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7611999999999998e-05 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:150657 8.1 8:819bffb1:::benchmark_data_smithi017_485823_object60520:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:29.951522+0000", 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.05801807899999, 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26816077599999999, 2023-09-23T15:27:08.069 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150657 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951522+0000", 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.070 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951522+0000", 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951520+0000", 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.071 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951527+0000", 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.17e-06 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951544+0000", 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6901e-05 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951552+0000", 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1850000000000004e-06 2023-09-23T15:27:08.072 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951705+0000", 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153332 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951808+0000", 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102771 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.073 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,2", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.951914+0000", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105547 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.958767+0000", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068530020000000004 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.962591+0000", 2023-09-23T15:27:08.074 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038239319999999999 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219618+0000", 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.25702733100000003 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219652+0000", 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3943999999999999e-05 2023-09-23T15:27:08.075 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.219683+0000", 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0732999999999999e-05 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:150672 8.3 8:cafb91ab:::benchmark_data_smithi017_485823_object60535:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T15:27:08.076 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:29.967850+0000", 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.041690475, 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.296743072, 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150672 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.077 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967850+0000", 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967850+0000", 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967848+0000", 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.078 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967854+0000", 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5799999999999999e-06 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967864+0000", 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9219999999999992e-06 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.079 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967870+0000", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2689999999999996e-06 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.967948+0000", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8269999999999994e-05 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213197+0000", 2023-09-23T15:27:08.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.245248671 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213379+0000", 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000181879 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.224370+0000", 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0109917 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.224449+0000", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9000999999999998e-05 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.262894+0000", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.038444063000000001 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.262954+0000", 2023-09-23T15:27:08.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0786999999999999e-05 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.264593+0000", 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001638299 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:150674 8.3 8:c68cf90a:::benchmark_data_smithi017_485823_object60537:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e328)", 2023-09-23T15:27:08.083 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:29.969299+0000", 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 158.04024116400001, 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29637957399999998, 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 150674 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.969299+0000", 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.969299+0000", 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.969298+0000", 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:29.969305+0000", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0940000000000002e-06 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213108+0000", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24380357 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.213129+0000", 2023-09-23T15:27:08.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0593000000000001e-05 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.216772+0000", 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003642597 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.216938+0000", 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166706 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,6", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.217092+0000", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000153664 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.224459+0000", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00736661 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.237285+0000", 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012826519999999999 2023-09-23T15:27:08.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.264643+0000", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.027357945000000002 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.264678+0000", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4808999999999998e-05 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:30.265678+0000", 2023-09-23T15:27:08.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010005669999999999 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83106 9.e 9:7e487db8:::benchmark_data_smithi017_543597_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.093016673999999, 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1221425209999998, 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.090 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83106 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916522+0000", 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916551+0000", 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9121000000000001e-05 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916561+0000", 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4059999999999994e-06 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916570+0000", 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5149999999999995e-06 2023-09-23T15:27:08.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918608+0000", 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020381800000000001 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918619+0000", 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1222e-05 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913525+0000", 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490589500000004 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913537+0000", 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1749000000000001e-05 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.028480+0000", 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1149425470000001 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.029082+0000", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060205899999999995 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038593+0000", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095110260000000005 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038666+0000", 2023-09-23T15:27:08.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3088000000000002e-05 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83114 9.d 9:bfb40ed3:::benchmark_data_smithi017_543597_object83113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.092964079, 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024477069, 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83114 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916575+0000", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916600+0000", 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4606000000000001e-05 2023-09-23T15:27:08.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916607+0000", 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916613+0000", 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2219999999999999e-06 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916703+0000", 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9252000000000005e-05 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916711+0000", 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9039999999999998e-06 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.931237+0000", 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0145257830000001 2023-09-23T15:27:08.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933189+0000", 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001951745 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933360+0000", 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017091799999999999 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940451+0000", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070909079999999999 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940940+0000", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048871099999999996 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940977+0000", 2023-09-23T15:27:08.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7722999999999998e-05 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.941053+0000", 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5729000000000006e-05 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83118 9.e 9:7615419c:::benchmark_data_smithi017_543597_object83117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:08.104 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.092914092000001, 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1299481519999999, 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83118 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916625+0000", 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:08.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916651+0000", 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5911e-05 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916659+0000", 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916665+0000", 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9460000000000003e-06 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918662+0000", 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019976350000000002 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918669+0000", 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2910000000000002e-06 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913570+0000", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490066300000002 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913580+0000", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.030629+0000", 2023-09-23T15:27:08.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170488599999999 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032411+0000", 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001782063 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032533+0000", 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012223299999999999 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038742+0000", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006208613 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038809+0000", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6854999999999996e-05 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045301+0000", 2023-09-23T15:27:08.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064921479999999997 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045369+0000", 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8693000000000001e-05 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046574+0000", 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012047169999999999 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.112 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83121 9.e 9:71f6c9a6:::benchmark_data_smithi017_543597_object83120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.092784344, 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1305874610000002, 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.113 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83121 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916755+0000", 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916784+0000", 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9522e-05 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916793+0000", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1000000000000004e-06 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916799+0000", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7530000000000004e-06 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918708+0000", 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001908734 2023-09-23T15:27:08.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918717+0000", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6629999999999999e-06 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914427+0000", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99571021100000001 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914437+0000", 2023-09-23T15:27:08.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0592e-05 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034594+0000", 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12015687 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034760+0000", 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001661 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034983+0000", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022256399999999999 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038750+0000", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037670210000000002 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038859+0000", 2023-09-23T15:27:08.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010865 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046631+0000", 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077723879999999999 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046672+0000", 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0759000000000001e-05 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047343+0000", 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067126700000000002 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83124 9.3 9:c55b3f52:::benchmark_data_smithi017_543597_object83123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 12.090538819000001, 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0418616510000001, 2023-09-23T15:27:08.121 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83124 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919000+0000", 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919028+0000", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8895e-05 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919041+0000", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2996000000000001e-05 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919051+0000", 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5729999999999997e-06 2023-09-23T15:27:08.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919101+0000", 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9864999999999998e-05 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919108+0000", 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2760000000000003e-06 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.944052+0000", 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024943395 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946310+0000", 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022579900000000001 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946472+0000", 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016254900000000001 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.958448+0000", 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975497999999999 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960747+0000", 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002299592 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960794+0000", 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6409000000000002e-05 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960863+0000", 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9227999999999995e-05 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83137 9.e 9:7d999fd1:::benchmark_data_smithi017_543597_object83136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:27:08.128 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 11.01794265, 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056604109, 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83137 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991596+0000", 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991621+0000", 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4907000000000001e-05 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991634+0000", 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2556e-05 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991642+0000", 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1340000000000004e-06 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991689+0000", 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7781000000000002e-05 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991696+0000", 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5239999999999997e-06 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038894+0000", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.047198039 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039062+0000", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016794100000000001 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039246+0000", 2023-09-23T15:27:08.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018399900000000001 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045231+0000", 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059852339999999999 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046596+0000", 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001365125 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047421+0000", 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082434100000000003 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047483+0000", 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2686000000000003e-05 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048201+0000", 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071811100000000001 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83228 9.e 9:7f83ce07:::benchmark_data_smithi017_543597_object83227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 10.853393434999999, 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89863112300000003, 2023-09-23T15:27:08.136 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83228 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156145+0000", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156194+0000", 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8671000000000002e-05 2023-09-23T15:27:08.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156210+0000", 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6716999999999999e-05 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156219+0000", 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156278+0000", 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8897999999999998e-05 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156292+0000", 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042118+0000", 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88582658800000003 2023-09-23T15:27:08.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042277+0000", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015909299999999999 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042470+0000", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193022 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048224+0000", 2023-09-23T15:27:08.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057530630000000001 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048291+0000", 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7751999999999998e-05 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.053961+0000", 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056697969999999999 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054020+0000", 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9419999999999997e-05 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054778+0000", 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075720099999999999 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83232 9.e 9:72608dfb:::benchmark_data_smithi017_543597_object83231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 10.841531797, 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88775156300000002, 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83232 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.144 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168006+0000", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168035+0000", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8824e-05 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168053+0000", 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8185e-05 2023-09-23T15:27:08.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168065+0000", 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1391e-05 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168131+0000", 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:08.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168140+0000", 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7129999999999994e-06 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048345+0000", 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88020536000000005 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048556+0000", 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021095900000000001 2023-09-23T15:27:08.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048793+0000", 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023668699999999999 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054856+0000", 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060630129999999999 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054871+0000", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851000000000001e-05 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054915+0000", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4753999999999999e-05 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054975+0000", 2023-09-23T15:27:08.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9562999999999999e-05 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078479899999999998 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:08.151 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:08.152 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 83272 83256 62.6573 0 - 0.0138696 2023-09-23T15:27:08.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:08] "GET /metrics HTTP/1.1" 200 36396 "" "Prometheus/2.43.0" 2023-09-23T15:27:08.209 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:08.236 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:08.262 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:08.287 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:08.311 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:08.336 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:08.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:27:08.357 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T15:27:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:08 smithi027 ceph-mon[195233]: 1.1a scrub starts 2023-09-23T15:27:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:08 smithi027 ceph-mon[195233]: 1.1a scrub ok 2023-09-23T15:27:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:08 smithi027 ceph-mon[195233]: pgmap v1073: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:27:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:08 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:27:08.668 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:08.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[215646]: 1.1a scrub starts 2023-09-23T15:27:08.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[215646]: 1.1a scrub ok 2023-09-23T15:27:08.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[215646]: pgmap v1073: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:27:08.672 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:27:08.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[222878]: 1.1a scrub starts 2023-09-23T15:27:08.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[222878]: 1.1a scrub ok 2023-09-23T15:27:08.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[222878]: pgmap v1073: 119 pgs: 24 active+undersized, 1 stale+active+recovery_wait+undersized+degraded+remapped, 6 stale+active+clean, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded, 73 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.2 MiB/s wr, 35 op/s; 36636/248574 objects degraded (14.738%); 273/248574 objects misplaced (0.110%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:27:08.672 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:08 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:27:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:08.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:08.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:08.801 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:08.851 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:08.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:08.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:08.986 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:09.019 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 83388 83372 61.998 0.725 0.00374573 0.0156476 2023-09-23T15:27:09.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[215646]: 1.18 scrub starts 2023-09-23T15:27:09.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[215646]: 1.18 scrub ok 2023-09-23T15:27:09.253 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:27:09.254 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[222878]: 1.18 scrub starts 2023-09-23T15:27:09.254 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[222878]: 1.18 scrub ok 2023-09-23T15:27:09.254 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:09 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:27:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:09 smithi027 ceph-mon[195233]: 1.18 scrub starts 2023-09-23T15:27:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:09 smithi027 ceph-mon[195233]: 1.18 scrub ok 2023-09-23T15:27:09.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:09 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:27:09.635 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:09.636 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:27:09.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:09.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:09.673 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:27:09.739 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:09.783 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:09.880 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:09.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:09.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:09.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:10.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 83559 83543 61.3947 10.6875 0.00346965 0.0160308 2023-09-23T15:27:10.020 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:10.277 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:10.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:10.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:10.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:10.420 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:27:10.478 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:27:10.532 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:10 smithi017 ceph-mon[215646]: pgmap v1074: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.7 MiB/s wr, 74 op/s; 49324/249870 objects degraded (19.740%); 3200/249870 objects misplaced (1.281%); 10 MiB/s, 164 objects/s recovering 2023-09-23T15:27:10.532 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:10 smithi017 ceph-mon[222878]: pgmap v1074: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.7 MiB/s wr, 74 op/s; 49324/249870 objects degraded (19.740%); 3200/249870 objects misplaced (1.281%); 10 MiB/s, 164 objects/s recovering 2023-09-23T15:27:10.592 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:10.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:10.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:10 smithi027 ceph-mon[195233]: pgmap v1074: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.7 MiB/s wr, 74 op/s; 49324/249870 objects degraded (19.740%); 3200/249870 objects misplaced (1.281%); 10 MiB/s, 164 objects/s recovering 2023-09-23T15:27:10.655 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:10.697 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:10.751 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:10.752 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:10.865 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:10.869 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:10.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:10.997 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:11.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 83722 83706 60.7995 10.1875 0.00414141 0.0161597 2023-09-23T15:27:11.042 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:11.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:11.048 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:11.148 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:11.176 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:11.222 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:11.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:11.281 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T15:27:11.329 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:11.364 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:11.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:11 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:27:11.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:27:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:11 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:27:11.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:27:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:11 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:27:11.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:27:12.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 83784 83768 60.1455 3.875 0.00323809 0.0162014 2023-09-23T15:27:12.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: pgmap v1075: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 49373/250104 objects degraded (19.741%); 3200/250104 objects misplaced (1.279%); 10 MiB/s, 165 objects/s recovering 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: osdmap e363: 8 total, 7 up, 7 in 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 49373/250104 objects degraded (19.741%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[222878]: osdmap e364: 8 total, 7 up, 7 in 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: pgmap v1075: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 49373/250104 objects degraded (19.741%); 3200/250104 objects misplaced (1.279%); 10 MiB/s, 165 objects/s recovering 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: osdmap e363: 8 total, 7 up, 7 in 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.507 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 49373/250104 objects degraded (19.741%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:12.508 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:12 smithi017 ceph-mon[215646]: osdmap e364: 8 total, 7 up, 7 in 2023-09-23T15:27:12.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: pgmap v1075: 119 pgs: 1 activating+degraded+remapped, 33 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.3 MiB/s wr, 52 op/s; 49373/250104 objects degraded (19.741%); 3200/250104 objects misplaced (1.279%); 10 MiB/s, 165 objects/s recovering 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/779765230' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: osdmap e363: 8 total, 7 up, 7 in 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: failed to encode map e363 with expected crc 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 49373/250104 objects degraded (19.741%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:12.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:12 smithi027 ceph-mon[195233]: osdmap e364: 8 total, 7 up, 7 in 2023-09-23T15:27:13.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 83788 83772 59.4652 0.25 0.00367131 0.0162136 2023-09-23T15:27:13.084 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:13.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:13.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:13.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:13.219 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: failed to encode map e365 with expected crc 2023-09-23T15:27:13.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[215646]: osdmap e365: 8 total, 7 up, 7 in 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: failed to encode map e365 with expected crc 2023-09-23T15:27:13.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:13 smithi017 ceph-mon[222878]: osdmap e365: 8 total, 7 up, 7 in 2023-09-23T15:27:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: failed to encode map e364 with expected crc 2023-09-23T15:27:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: failed to encode map e365 with expected crc 2023-09-23T15:27:13.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:13 smithi027 ceph-mon[195233]: osdmap e365: 8 total, 7 up, 7 in 2023-09-23T15:27:13.677 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:13.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:13.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:13.781 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:13.814 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:13.865 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:13.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:13.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:13.999 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:14.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 83788 83772 58.7973 0 - 0.0162136 2023-09-23T15:27:14.028 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:14.045 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:14.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:14.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:14.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:14.179 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:14.223 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:14.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:14.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:14.324 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:27:14.324 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:27:14.330 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[215646]: pgmap v1078: 119 pgs: 1 active+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.0 MiB/s wr, 95 op/s; 41551/250461 objects degraded (16.590%); 10712/250461 objects misplaced (4.277%); 20 MiB/s, 312 objects/s recovering 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[215646]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[215646]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[222878]: pgmap v1078: 119 pgs: 1 active+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.0 MiB/s wr, 95 op/s; 41551/250461 objects degraded (16.590%); 10712/250461 objects misplaced (4.277%); 20 MiB/s, 312 objects/s recovering 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[222878]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.341 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:14 smithi017 ceph-mon[222878]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.366 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:14.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:14 smithi027 ceph-mon[195233]: pgmap v1078: 119 pgs: 1 active+undersized+degraded+remapped, 1 activating+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 6.0 MiB/s wr, 95 op/s; 41551/250461 objects degraded (16.590%); 10712/250461 objects misplaced (4.277%); 20 MiB/s, 312 objects/s recovering 2023-09-23T15:27:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:14 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:27:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:14 smithi027 ceph-mon[195233]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:14 smithi027 ceph-mon[195233]: failed to encode map e365 with expected crc 2023-09-23T15:27:14.921 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:14.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:14.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:15.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 90 16 83788 83772 58.1443 0 - 0.0162136 2023-09-23T15:27:15.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:15.058 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:15.109 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:15.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:15.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:15.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:15.244 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:15.278 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:27:15.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:15 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:27:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1442662109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.539 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:27:15.540 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1442662109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.540 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:27:15.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1442662109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.540 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:15.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:27:15.901 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:15.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:15.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:16.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:16.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 91 16 83788 83772 57.5056 0 - 0.0162136 2023-09-23T15:27:16.043 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:16.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:16.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:16.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:16.256 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:16.291 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:16.394 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: pgmap v1080: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.4 MiB/s wr, 54 op/s; 36450/250857 objects degraded (14.530%); 7785/250857 objects misplaced (3.103%); 11 MiB/s, 181 objects/s recovering 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: osdmap e366: 8 total, 7 up, 7 in 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[215646]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.396 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: pgmap v1080: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.4 MiB/s wr, 54 op/s; 36450/250857 objects degraded (14.530%); 7785/250857 objects misplaced (3.103%); 11 MiB/s, 181 objects/s recovering 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: osdmap e366: 8 total, 7 up, 7 in 2023-09-23T15:27:16.397 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:16 smithi017 ceph-mon[222878]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:16.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:16.438 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:16.489 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:16.499 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:16.510 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:16.529 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:16.532 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:16.553 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:16.580 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: pgmap v1080: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.4 MiB/s wr, 54 op/s; 36450/250857 objects degraded (14.530%); 7785/250857 objects misplaced (3.103%); 11 MiB/s, 181 objects/s recovering 2023-09-23T15:27:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: osdmap e366: 8 total, 7 up, 7 in 2023-09-23T15:27:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:16 smithi027 ceph-mon[195233]: failed to encode map e366 with expected crc 2023-09-23T15:27:16.601 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:16.621 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:27:16.622 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T15:27:16.637 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:16.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:16.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:16.743 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:16.773 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:17.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 92 16 83788 83772 56.8808 0 - 0.0162136 2023-09-23T15:27:17.274 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1805302707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:17.275 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 36450/250857 objects degraded (14.530%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:17.411 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:17.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:17.414 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:17.515 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:17.540 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1805302707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:17.541 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 36450/250857 objects degraded (14.530%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:17.544 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:17.558 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:27:17.559 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:27:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1805302707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:17.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 36450/250857 objects degraded (14.530%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:17.752 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:17.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:17.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:17.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:17.887 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 93 16 83788 83772 56.2695 0 - 0.0162136 2023-09-23T15:27:18.021 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:18.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.025 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:18.033 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [7] live_osds: [0, 2, 3, 4, 5, 1, 6] 2023-09-23T15:27:18.033 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:27:18.033 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T15:27:18.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T15:27:18.092 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:18] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-09-23T15:27:18.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:27:18.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:18.188 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.254 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:18.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:18.347 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:18.347 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:27:18.347 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:27:18.347 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:27:18.347 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:18.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:18.374 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:18 smithi017 ceph-mon[215646]: pgmap v1082: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.9 MiB/s wr, 77 op/s; 36531/251400 objects degraded (14.531%); 7785/251400 objects misplaced (3.097%); 12 MiB/s, 196 objects/s recovering 2023-09-23T15:27:18.374 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:18 smithi017 ceph-mon[222878]: pgmap v1082: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.9 MiB/s wr, 77 op/s; 36531/251400 objects degraded (14.531%); 7785/251400 objects misplaced (3.097%); 12 MiB/s, 196 objects/s recovering 2023-09-23T15:27:18.392 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.445 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:18.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:18.551 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:18.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:18 smithi027 ceph-mon[195233]: pgmap v1082: 119 pgs: 1 remapped+peering, 1 active+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 4.9 MiB/s wr, 77 op/s; 36531/251400 objects degraded (14.531%); 7785/251400 objects misplaced (3.097%); 12 MiB/s, 196 objects/s recovering 2023-09-23T15:27:18.579 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.626 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:18.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:18.760 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.803 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:18.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:18.908 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:18.935 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:18.979 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:18.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:18.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:19.000 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:19.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 94 16 83788 83772 55.6711 0 - 0.0162136 2023-09-23T15:27:19.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:27:19.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:19.114 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:19.156 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:19.223 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:19.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:19.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:19.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:19.358 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:19.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:27:19.814 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:19.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:19.817 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:19.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:19.951 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.003 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:20.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.010 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:20.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 95 16 83788 83772 55.0854 0 - 0.0162136 2023-09-23T15:27:20.111 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:20.141 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.244 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:20.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:20.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:20.381 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.433 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:20.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:20 smithi017 ceph-mon[215646]: pgmap v1083: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.0 MiB/s wr, 47 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 973 KiB/s, 15 objects/s recovering 2023-09-23T15:27:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:20 smithi017 ceph-mon[222878]: pgmap v1083: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.0 MiB/s wr, 47 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 973 KiB/s, 15 objects/s recovering 2023-09-23T15:27:20.543 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:20.570 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:20 smithi027 ceph-mon[195233]: pgmap v1083: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 3.0 MiB/s wr, 47 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 973 KiB/s, 15 objects/s recovering 2023-09-23T15:27:20.573 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:20.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:20.660 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:20.727 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:20.755 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.802 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:20.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:20.818 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:20.910 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:20.938 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:20.981 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:20.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:20.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:21.020 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 96 16 83788 83772 54.5118 0 - 0.0162136 2023-09-23T15:27:21.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:21.123 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:22.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 97 16 83788 83772 53.9501 0 - 0.0162136 2023-09-23T15:27:22.049 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:22.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:22.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:22.153 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:22.186 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:22.237 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:22.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:22.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:22.294 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.4. 2023-09-23T15:27:22.294 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:27:22.344 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:22.371 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:22.415 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:22.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:22.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:22 smithi017 conmon[215622]: 2023-09-23T15:27:22.278+0000 7f618a1a9700 -1 mon.a@0(leader).osd e366 definitely_dead 0 2023-09-23T15:27:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[215646]: pgmap v1084: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.5 MiB/s wr, 39 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 800 KiB/s, 12 objects/s recovering 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 36532/251406 objects degraded (14.531%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[222878]: pgmap v1084: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.5 MiB/s wr, 39 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 800 KiB/s, 12 objects/s recovering 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 36532/251406 objects degraded (14.531%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:27:22.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:22.551 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:22 smithi027 ceph-mon[195233]: pgmap v1084: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 2.5 MiB/s wr, 39 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 800 KiB/s, 12 objects/s recovering 2023-09-23T15:27:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 36532/251406 objects degraded (14.531%), 18 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T15:27:23.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 98 16 83788 83772 53.3995 0 - 0.0162136 2023-09-23T15:27:23.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:27:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: osdmap e367: 8 total, 7 up, 8 in 2023-09-23T15:27:23.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:23 smithi027 ceph-mon[195233]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.616 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: osdmap e367: 8 total, 7 up, 8 in 2023-09-23T15:27:23.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[215646]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1848161498' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3107716509' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: osdmap e367: 8 total, 7 up, 8 in 2023-09-23T15:27:23.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:23 smithi017 ceph-mon[222878]: failed to encode map e367 with expected crc 2023-09-23T15:27:23.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:23.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:23.721 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:23.749 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:23.796 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:23.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:23.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:23.904 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:23.911 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T15:27:23.934 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:24.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 99 16 83788 83772 52.8604 0 - 0.0162136 2023-09-23T15:27:24.248 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:24.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:24.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:24.331 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:24.331 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:27:24.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161416 7.4 7:23fd1d4d:::benchmark_data_smithi017_422824_object67546:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.89087571900001, 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2737346510000001, 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.332 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161416 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422524+0000", 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422522+0000", 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422531+0000", 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.473e-06 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422551+0000", 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0579000000000001e-05 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422561+0000", 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4590000000000004e-06 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422648+0000", 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7039000000000005e-05 2023-09-23T15:27:24.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422805+0000", 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000157534 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.422938+0000", 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013334499999999999 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.426632+0000", 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036934110000000002 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.446635+0000", 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.020002678999999999 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.446646+0000", 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1630999999999999e-05 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689234+0000", 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2425875829999999 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689345+0000", 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111606 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696168+0000", 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068226500000000004 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696258+0000", 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0245000000000001e-05 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161431 7.4 7:211b85c0:::benchmark_data_smithi017_422824_object67561:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:27:24.339 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.87467663799998, 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2578971299999999, 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161431 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438723+0000", 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438721+0000", 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438728+0000", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2910000000000004e-06 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438740+0000", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2554e-05 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438749+0000", 2023-09-23T15:27:24.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6279999999999994e-06 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.438939+0000", 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018970900000000001 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.439081+0000", 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014192400000000001 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4,6", 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.439213+0000", 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000132261 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.447521+0000", 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0083078590000000008 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.344 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.447530+0000", 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4390000000000006e-06 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689610+0000", 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2420803679999999 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689711+0000", 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010036500000000001 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696563+0000", 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0068518750000000003 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696620+0000", 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.715e-05 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161437 7.4 7:2481e772:::benchmark_data_smithi017_422824_object67567:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.863107372, 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2465200679999999, 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161437 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.347 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450292+0000", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450295+0000", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.0510000000000001e-06 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450301+0000", 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4320000000000004e-06 2023-09-23T15:27:24.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450309+0000", 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7349999999999996e-06 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450363+0000", 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4568999999999999e-05 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.450373+0000", 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689746+0000", 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2393721659999999 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.351 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689844+0000", 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7947000000000003e-05 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.689939+0000", 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5031999999999994e-05 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.695831+0000", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058926730000000002 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696129+0000", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00029764899999999998 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696672+0000", 2023-09-23T15:27:24.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00054306700000000005 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696721+0000", 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9468000000000001e-05 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696812+0000", 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0595999999999998e-05 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.354 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161441 7.4 7:2b51fbee:::benchmark_data_smithi017_422824_object67571:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.85245805199997, 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2360482610000001, 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.355 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161441 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460941+0000", 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460940+0000", 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460947+0000", 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460965+0000", 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7609e-05 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.460973+0000", 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4910000000000005e-06 2023-09-23T15:27:24.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.461022+0000", 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9020000000000002e-05 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.461032+0000", 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8660000000000001e-06 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692245+0000", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2312128790000001 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692324+0000", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9361000000000001e-05 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.692410+0000", 2023-09-23T15:27:24.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5955e-05 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.695969+0000", 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003558912 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696133+0000", 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000163463 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696869+0000", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00073619500000000004 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696912+0000", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3534e-05 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696990+0000", 2023-09-23T15:27:24.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7205000000000004e-05 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161456 7.4 7:2e1a1183:::benchmark_data_smithi017_422824_object67586:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.83452901200002, 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2247869389999999, 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.363 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.364 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161456 2023-09-23T15:27:24.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.364 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478870+0000", 2023-09-23T15:27:24.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478869+0000", 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478876+0000", 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.109e-06 2023-09-23T15:27:24.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478888+0000", 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2215000000000001e-05 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478895+0000", 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3579999999999999e-06 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478940+0000", 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5683999999999998e-05 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.478951+0000", 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0312000000000001e-05 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694102+0000", 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2151512950000001 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694202+0000", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000100055 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.694274+0000", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2175000000000006e-05 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696135+0000", 2023-09-23T15:27:24.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018612909999999999 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.696295+0000", 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015990700000000001 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703512+0000", 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072162390000000002 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703565+0000", 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3285000000000002e-05 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703657+0000", 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2373000000000005e-05 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161460 7.4 7:2562121b:::benchmark_data_smithi017_422824_object67590:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.82708298099999, 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.226105368, 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161460 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.372 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486316+0000", 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486314+0000", 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486321+0000", 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8519999999999998e-06 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486333+0000", 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2021000000000001e-05 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486341+0000", 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7870000000000002e-06 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486430+0000", 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.8721999999999995e-05 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.486440+0000", 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.8989999999999996e-06 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697044+0000", 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.210603841 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697146+0000", 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000102636 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.697263+0000", 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011694299999999999 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706589+0000", 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0093257740000000002 2023-09-23T15:27:24.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706638+0000", 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9157999999999999e-05 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712250+0000", 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056120750000000002 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712305+0000", 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5049000000000001e-05 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712422+0000", 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116574 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161519 7.4 7:220f2d51:::benchmark_data_smithi017_422824_object67649:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.72774585399998, 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1269551660000001, 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161519 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.380 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585654+0000", 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585653+0000", 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585656+0000", 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1020000000000001e-06 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585663+0000", 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4749999999999998e-06 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585668+0000", 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6969999999999998e-06 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585721+0000", 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3004999999999998e-05 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.585731+0000", 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7520000000000008e-06 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700079+0000", 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.114347703 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700199+0000", 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120475 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.700331+0000", 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013203800000000001 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706596+0000", 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062643619999999999 2023-09-23T15:27:24.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706677+0000", 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1079000000000001e-05 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712483+0000", 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058062499999999998 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712525+0000", 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1771999999999999e-05 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712609+0000", 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3947000000000001e-05 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161524 7.4 7:2a8e46ee:::benchmark_data_smithi017_422824_object67654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.71663263200003, 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.116073053, 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161524 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.388 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596767+0000", 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596766+0000", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596771+0000", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.335e-06 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596777+0000", 2023-09-23T15:27:24.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9299999999999997e-06 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596783+0000", 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8300000000000001e-06 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596883+0000", 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9469999999999995e-05 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.596893+0000", 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0151e-05 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703705+0000", 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1068115599999999 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703843+0000", 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013830099999999999 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.703988+0000", 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000145492 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712149+0000", 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0081608110000000005 2023-09-23T15:27:24.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712160+0000", 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1076e-05 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712690+0000", 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00052939600000000001 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712745+0000", 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5758999999999999e-05 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712840+0000", 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4439999999999997e-05 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:161528 7.4 7:2096f0c5:::benchmark_data_smithi017_422824_object67658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e269)", 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 430.70782798400001, 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.114153634, 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 161528 2023-09-23T15:27:24.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.397 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605571+0000", 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605575+0000", 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9169999999999999e-06 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605582+0000", 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8940000000000004e-06 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605588+0000", 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.908e-06 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605644+0000", 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6863000000000001e-05 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:13.605654+0000", 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5279999999999992e-06 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706705+0000", 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1010514419999999 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706832+0000", 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012622100000000001 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,6", 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.706973+0000", 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014145499999999999 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712190+0000", 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0052171300000000004 2023-09-23T15:27:24.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.712441+0000", 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025033599999999998 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719593+0000", 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071525429999999999 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719634+0000", 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0735999999999998e-05 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:14.719725+0000", 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1137999999999994e-05 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.403 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50588 8.e 8:73911130:::benchmark_data_smithi017_485823_object50587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 328.10980119700002, 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.107695917, 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50588 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203597+0000", 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203626+0000", 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9794999999999998e-05 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203639+0000", 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2792999999999999e-05 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203648+0000", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0340000000000002e-06 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203778+0000", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129462 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203789+0000", 2023-09-23T15:27:24.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1683e-05 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123801+0000", 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92001208899999998 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123813+0000", 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1676e-05 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301009+0000", 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.177195422 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301190+0000", 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018111599999999999 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301356+0000", 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166479 2023-09-23T15:27:24.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307313+0000", 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059565829999999997 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307350+0000", 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6908e-05 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309727+0000", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023774629999999998 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309803+0000", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6243999999999997e-05 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311294+0000", 2023-09-23T15:27:24.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014907830000000001 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50599 8.e 8:730ce43d:::benchmark_data_smithi017_485823_object50598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 328.10157469900003, 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1058259499999998, 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.412 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50599 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211823+0000", 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211849+0000", 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6114000000000001e-05 2023-09-23T15:27:24.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211866+0000", 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6600999999999999e-05 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211874+0000", 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3229999999999998e-06 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211940+0000", 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6039999999999998e-05 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211958+0000", 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7448e-05 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123836+0000", 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91187878 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123844+0000", 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9489999999999996e-06 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.304978+0000", 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1811335549999999 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305174+0000", 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196164 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305380+0000", 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000206092 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311368+0000", 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059874079999999996 2023-09-23T15:27:24.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311447+0000", 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9140000000000005e-05 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315621+0000", 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041739070000000001 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315691+0000", 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0588999999999998e-05 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317651+0000", 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001959391 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50609 8.e 8:797f5f67:::benchmark_data_smithi017_485823_object50608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:27:24.420 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 328.09150351199997, 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.097560632, 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50609 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221894+0000", 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:24.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221919+0000", 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5066999999999999e-05 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221946+0000", 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7331e-05 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221959+0000", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.269e-05 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222024+0000", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4824000000000001e-05 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222038+0000", 2023-09-23T15:27:24.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3788000000000001e-05 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123868+0000", 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90182994900000002 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123876+0000", 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2979999999999992e-06 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307446+0000", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183570029 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307604+0000", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015765000000000001 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307772+0000", 2023-09-23T15:27:24.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168114 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311434+0000", 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036626810000000001 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311453+0000", 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9004e-05 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317944+0000", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064906199999999999 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317998+0000", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4296999999999999e-05 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319456+0000", 2023-09-23T15:27:24.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014581900000000001 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50596 8.2 8:4072d885:::benchmark_data_smithi017_485823_object50595:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 327.19245315400002, 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1658852479999999, 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.429 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50596 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.430 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120944+0000", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120979+0000", 2023-09-23T15:27:24.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4545000000000002e-05 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121002+0000", 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2994e-05 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121014+0000", 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.219e-05 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121135+0000", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012048399999999999 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121143+0000", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123756+0000", 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026131420000000002 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123766+0000", 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0566e-05 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275228+0000", 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1514619580000001 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275358+0000", 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129383 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275499+0000", 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141419 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279452+0000", 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039528280000000002 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279468+0000", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5583e-05 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285296+0000", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005828415 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285337+0000", 2023-09-23T15:27:24.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1216e-05 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.286831+0000", 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014941799999999999 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50597 8.2 8:4463d1ca:::benchmark_data_smithi017_485823_object50596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 327.19236604399998, 2023-09-23T15:27:24.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.173769987, 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50597 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:27:24.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121059+0000", 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6764999999999999e-05 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121068+0000", 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6510000000000007e-06 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121075+0000", 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6290000000000004e-06 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121210+0000", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135697 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121216+0000", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2360000000000004e-06 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126405+0000", 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051892259999999999 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126415+0000", 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0358e-05 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.277883+0000", 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.151468277 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278066+0000", 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018207400000000001 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278266+0000", 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020090099999999999 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279562+0000", 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001295825 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.281149+0000", 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00158647 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293587+0000", 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012437981000000001 2023-09-23T15:27:24.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293648+0000", 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0943000000000002e-05 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.294803+0000", 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011556100000000001 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83122 9.5 9:ab87d94d:::benchmark_data_smithi017_543597_object83121:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.913219+0000", 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 28.400179909999999, 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.089073672, 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83122 2023-09-23T15:27:24.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.913219+0000", 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.913219+0000", 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.913218+0000", 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.913246+0000", 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8558e-05 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.914712+0000", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001465769 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.914723+0000", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0779e-05 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.914779+0000", 2023-09-23T15:27:24.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6067999999999999e-05 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.914790+0000", 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0874e-05 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.964649+0000", 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.049859265 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.964809+0000", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016008999999999999 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.964987+0000", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017746 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.002089+0000", 2023-09-23T15:27:24.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.037102461000000003 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.002150+0000", 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0649999999999997e-05 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.002210+0000", 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0214000000000003e-05 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.002293+0000", 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3147999999999997e-05 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83123 9.5 9:ad660340:::benchmark_data_smithi017_543597_object83122:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919353+0000", 2023-09-23T15:27:24.453 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 28.394046014000001, 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0888389030000001, 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83123 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919353+0000", 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919353+0000", 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919352+0000", 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919373+0000", 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1092999999999999e-05 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919390+0000", 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6599000000000002e-05 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919397+0000", 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2880000000000004e-06 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919449+0000", 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1592999999999999e-05 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919460+0000", 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0934000000000001e-05 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.968546+0000", 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0490864339999999 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.968667+0000", 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012119400000000001 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.968820+0000", 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000152683 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.002347+0000", 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033527501000000001 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.008073+0000", 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057256499999999997 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.008119+0000", 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6300000000000001e-05 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.008192+0000", 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2875999999999998e-05 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83259 9.4 e361/348 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head v 361'5248, mlcod=344'117)", 2023-09-23T15:27:24.462 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.256976028, 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.141249403, 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.463 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056450+0000", 2023-09-23T15:27:24.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7029000000000002e-05 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056466+0000", 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6181e-05 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056472+0000", 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.772e-06 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.161146+0000", 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.104674147000001 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.161175+0000", 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.94e-05 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197621+0000", 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036445402000000002 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197673+0000", 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2212000000000001e-05 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.468 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83260 9.4 e361/348 9:295d7846:::benchmark_data_smithi017_543597_object83259:head v 361'5249, mlcod=344'117)", 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.254086773000001, 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.138397940000001, 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:27:24.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059311+0000", 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:24.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059337+0000", 2023-09-23T15:27:24.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6217e-05 2023-09-23T15:27:24.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059351+0000", 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3326000000000001e-05 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059359+0000", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6440000000000006e-06 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.165673+0000", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106313241000001 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.165706+0000", 2023-09-23T15:27:24.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2969000000000002e-05 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197678+0000", 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.031972689999999998 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197711+0000", 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2249999999999998e-05 2023-09-23T15:27:24.475 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83261 9.4 e361/348 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head v 361'5250, mlcod=344'117)", 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.250981982999999, 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.135328450999999, 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062416+0000", 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062444+0000", 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7557000000000001e-05 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062457+0000", 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3112000000000001e-05 2023-09-23T15:27:24.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062467+0000", 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9769999999999995e-06 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.169452+0000", 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106984581000001 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.169473+0000", 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1597999999999999e-05 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197713+0000", 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028239744000000001 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197746+0000", 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2818999999999999e-05 2023-09-23T15:27:24.480 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83265 9.4 e361/348 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head v 361'5251, mlcod=344'117)", 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 26.247315140000001, 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131697632, 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:24.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:24.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066083+0000", 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066111+0000", 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7898999999999998e-05 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066127+0000", 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5858e-05 2023-09-23T15:27:24.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066137+0000", 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0261999999999999e-05 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.172807+0000", 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106669544000001 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.172829+0000", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2413e-05 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197748+0000", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024919220999999998 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197782+0000", 2023-09-23T15:27:24.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3535999999999999e-05 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:24.486 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:24.487 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:24.487 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: pgmap v1086: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.4 MiB/s wr, 22 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 792 KiB/s, 12 objects/s recovering 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: osdmap e368: 8 total, 7 up, 8 in 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.487 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[215646]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: pgmap v1086: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.4 MiB/s wr, 22 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 792 KiB/s, 12 objects/s recovering 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: osdmap e368: 8 total, 7 up, 8 in 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.488 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:24 smithi017 ceph-mon[222878]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.489 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:24.489 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:24.493 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:24.517 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:24.517 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.517 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.517 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:27:24.518 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T15:27:24.523 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: pgmap v1086: 119 pgs: 1 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 33 active+undersized, 18 active+undersized+degraded, 65 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 1.4 MiB/s wr, 22 op/s; 36532/251406 objects degraded (14.531%); 7686/251406 objects misplaced (3.057%); 792 KiB/s, 12 objects/s recovering 2023-09-23T15:27:24.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: osdmap e368: 8 total, 7 up, 8 in 2023-09-23T15:27:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: failed to encode map e368 with expected crc 2023-09-23T15:27:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:24 smithi027 ceph-mon[195233]: failed to encode map e368 with expected crc 2023-09-23T15:27:25.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:27:25.022207+0000 min lat: 0.00111339 max lat: 12.6845 avg lat: 0.0162136 2023-09-23T15:27:25.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:27:25.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 100 16 83788 83772 52.332 0 - 0.0162136 2023-09-23T15:27:25.272 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:25.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:25.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:25.376 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:25.383 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:27:25.403 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:25.447 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:25.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:25.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:25.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: osdmap e369: 8 total, 7 up, 8 in 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.577 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1519757748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:25.579 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:25.624 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:25.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:25.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:25.651 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:27:25.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.663 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: osdmap e369: 8 total, 7 up, 8 in 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1519757748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: osdmap e369: 8 total, 7 up, 8 in 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: failed to encode map e369 with expected crc 2023-09-23T15:27:25.665 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1519757748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:25.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:25.760 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:25.815 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:25.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:25.818 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:25.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:25.946 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:25.992 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:25.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:25.997 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:26.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 101 16 83788 83772 51.814 0 - 0.0162136 2023-09-23T15:27:26.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:26.127 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:26.301 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:26.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:26.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:26.313 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: pgmap v1089: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T15:27:26.313 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.313 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.407 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:26.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: pgmap v1089: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T15:27:26.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.408 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[222878]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: pgmap v1089: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 1 keys/s, 0 objects/s recovering 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.409 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:26 smithi017 ceph-mon[215646]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T15:27:26.427 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:27:26.427 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:27:26.436 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:26.468 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:26.482 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:26.482 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:26.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:26.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:26.587 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:26.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: failed to encode map e370 with expected crc 2023-09-23T15:27:26.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:26 smithi027 ceph-mon[195233]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T15:27:26.615 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:26.661 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:26.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:26.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:26.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:26.798 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:26.855 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:26.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:26.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:26.961 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:26.998 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:27.021 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 102 16 83788 83772 51.3063 0 - 0.0162136 2023-09-23T15:27:27.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T15:27:27.316 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.317 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:27.355 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:27.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:27.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:27.492 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:27.545 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:27.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:27.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T15:27:27.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:27.649 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:27.689 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:27.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: failed to encode map e371 with expected crc 2023-09-23T15:27:27.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T15:27:27.736 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:27.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:27.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:27.842 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:27.869 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:27.914 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:27.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:27.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:28.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:28.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 103 16 83788 83772 50.8084 0 - 0.0162136 2023-09-23T15:27:28.055 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:28.316 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:28] "GET /metrics HTTP/1.1" 200 36394 "" "Prometheus/2.43.0" 2023-09-23T15:27:28.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[222878]: pgmap v1092: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T15:27:28.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:28.317 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:28.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:27:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:28 smithi027 ceph-mon[195233]: pgmap v1092: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T15:27:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:28.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[215646]: pgmap v1092: 119 pgs: 8 active+clean+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 24 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 63 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7694/251406 objects misplaced (3.060%); 0 B/s, 2 keys/s, 0 objects/s recovering 2023-09-23T15:27:28.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:28.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:28.912 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [7] live_osds: [0, 2, 3, 4, 5, 1, 6] 2023-09-23T15:27:28.912 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-09-23T15:27:28.912 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-09-23T15:27:28.912 DEBUG:teuthology.orchestra.run.smithi027:osd.7> sudo journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T15:27:28.915 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl start ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T15:27:28.942 INFO:journalctl@ceph.osd.7.smithi027.stdout:-- Logs begin at Sat 2023-09-23 14:30:18 UTC. -- 2023-09-23T15:27:28.975 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:28.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:28.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:29.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 104 16 83788 83772 50.32 0 - 0.0162136 2023-09-23T15:27:29.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:29.112 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:29.162 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:29.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:29.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:29.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:29.294 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:28 smithi027 systemd[1]: Starting Ceph osd.7 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:27:29.297 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:29.350 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:29.356 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:29.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:29.435 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:27:29.435 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:27:29.461 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:29.491 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:29.590 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:29.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:29.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:29.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:29.732 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:30.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 105 16 83788 83772 49.8409 0 - 0.0162136 2023-09-23T15:27:30.147 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:30.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:30.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:30.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:30.286 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:30.318 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T15:27:30.319 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:29 smithi027 bash[344185]: --> ceph-volume lvm activate successful for osd ID: 7 2023-09-23T15:27:30.333 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:27:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[215646]: pgmap v1093: 119 pgs: 11 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 23 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 4.6 MiB/s, 1 keys/s, 79 objects/s recovering 2023-09-23T15:27:30.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3924830477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:30.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[222878]: pgmap v1093: 119 pgs: 11 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 23 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 4.6 MiB/s, 1 keys/s, 79 objects/s recovering 2023-09-23T15:27:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3924830477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:30.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:30 smithi027 ceph-mon[195233]: pgmap v1093: 119 pgs: 11 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 23 active+undersized, 6 active+undersized+remapped, 13 active+undersized+degraded, 64 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 4.6 MiB/s, 1 keys/s, 79 objects/s recovering 2023-09-23T15:27:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3924830477' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:27:31.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 106 16 83788 83772 49.3709 0 - 0.0162136 2023-09-23T15:27:31.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-mon[195233]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:27:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-mon[195233]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T15:27:31.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-mon[195233]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:31.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[215646]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:27:31.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[215646]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T15:27:31.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[215646]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:31.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[222878]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:27:31.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[222878]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T15:27:31.636 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:31 smithi017 ceph-mon[222878]: failed to encode map e372 with expected crc 2023-09-23T15:27:31.654 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:31.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:31.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:31.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:27:31.680 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:27:31.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:31.790 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:31.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:31.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:31.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:31.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:31.978 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:31 smithi027 bash[344699]: cececd22c33f4a13a2aefed21c690cd18089ef8b0683ef6d1319a6ef1db39859 2023-09-23T15:27:31.978 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:31 smithi027 ceph-osd[344764]: -- 172.21.15.27:0/3797829365 <== mon.1 v2:172.21.15.27:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ed67b3ab60 con 0x55ed67b01400 2023-09-23T15:27:31.978 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:31 smithi027 systemd[1]: Started Ceph osd.7 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:27:31.981 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:32.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 107 16 83788 83772 48.9097 0 - 0.0162136 2023-09-23T15:27:32.033 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:32.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:32.078 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:27:32.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:32.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:32.283 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:32 smithi027 conmon[344740]: 2023-09-23T15:27:32.201+0000 7f3503ce2540 -1 Falling back to public interface 2023-09-23T15:27:32.326 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:32.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:32.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:32.419 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:32 smithi017 ceph-mon[215646]: pgmap v1095: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 5.0 MiB/s, 83 objects/s recovering 2023-09-23T15:27:32.419 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:32.419 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:32 smithi017 ceph-mon[222878]: pgmap v1095: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 5.0 MiB/s, 83 objects/s recovering 2023-09-23T15:27:32.419 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:32.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:32.467 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:32.527 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:32 smithi027 ceph-mon[195233]: pgmap v1095: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 7053/251406 objects misplaced (2.805%); 5.0 MiB/s, 83 objects/s recovering 2023-09-23T15:27:32.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 36523/251406 objects degraded (14.527%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:32.645 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:32.669 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:32.672 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:32.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:32.675 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:32.690 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:32.776 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:32.809 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:33.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 108 16 83788 83772 48.457 0 - 0.0162136 2023-09-23T15:27:33.120 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:33.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:33.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:33.254 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:33.332 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/656005185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/656005185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:33.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/656005185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:33.405 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:33.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:33.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:33.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:33.540 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:33.593 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:33.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:33.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:33.614 INFO:teuthology.orchestra.run.smithi027.stderr:no valid command found; 10 closest matches: 2023-09-23T15:27:33.614 INFO:teuthology.orchestra.run.smithi027.stderr:0 2023-09-23T15:27:33.614 INFO:teuthology.orchestra.run.smithi027.stderr:1 2023-09-23T15:27:33.614 INFO:teuthology.orchestra.run.smithi027.stderr:2 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:abort 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:assert 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:config diff 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:config diff get 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:config get 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:config help [] 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:config set ... 2023-09-23T15:27:33.615 INFO:teuthology.orchestra.run.smithi027.stderr:admin_socket: invalid command 2023-09-23T15:27:33.698 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:33.727 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:33.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:33.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:33.931 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:33.967 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:34.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 109 16 83788 83772 48.0126 0 - 0.0162136 2023-09-23T15:27:34.049 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:34.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:34.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:34.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:34.181 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:34.298 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T15:27:34.299 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-09-23T15:27:34.307 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:34.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:34.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:34.359 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:34.359 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:27:34.359 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:27:34.359 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:27:34.359 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:34.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:27:34.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:34 smithi017 ceph-mon[222878]: pgmap v1096: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 6253/251406 objects misplaced (2.487%); 11 MiB/s, 172 objects/s recovering 2023-09-23T15:27:34.385 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:34 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:27:34.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:34 smithi017 ceph-mon[215646]: pgmap v1096: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 6253/251406 objects misplaced (2.487%); 11 MiB/s, 172 objects/s recovering 2023-09-23T15:27:34.385 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:34 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:27:34.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:34.457 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:34.571 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:34.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:34.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:34.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:34 smithi027 ceph-mon[195233]: pgmap v1096: 119 pgs: 16 active+clean+remapped, 1 remapped+peering, 1 active+recovering+undersized+remapped, 21 active+undersized, 13 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 36523/251406 objects degraded (14.527%); 6253/251406 objects misplaced (2.487%); 11 MiB/s, 172 objects/s recovering 2023-09-23T15:27:34.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:34 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:27:34.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:34.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:34.701 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:34.708 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:34.734 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.875 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:27:34.876 INFO:teuthology.orchestra.run.smithi017.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T15:27:35.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 109.938 2023-09-23T15:27:35.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 83788 2023-09-23T15:27:35.022 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 47.6336 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 46.5215 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 188.75 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 0 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 762 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 744.35 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3020 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 0 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0209832 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.382993 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 38.8857 2023-09-23T15:27:35.023 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00111339 2023-09-23T15:27:35.026 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:27:35.212 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:35.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:35.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:35.318 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:35.396 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:35.841 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:27:36.042 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:36.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:36.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T15:27:36.160 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T15:27:36.183 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:36.240 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:36.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:36.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:36.320 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:36.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:36.424 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:36 smithi017 ceph-mon[222878]: pgmap v1097: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 9.4 MiB/s, 152 objects/s recovering 2023-09-23T15:27:36.427 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:36 smithi017 ceph-mon[215646]: pgmap v1097: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 9.4 MiB/s, 152 objects/s recovering 2023-09-23T15:27:36.434 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:36.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:36 smithi027 ceph-mon[195233]: pgmap v1097: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 9.4 MiB/s, 152 objects/s recovering 2023-09-23T15:27:36.650 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:36.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:36.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:36.769 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:36.873 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:36.914 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:37.033 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:37.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:37.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:37.150 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:37.235 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:37.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:37 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 41688/251406 objects degraded (16.582%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:37.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:37 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 41688/251406 objects degraded (16.582%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:37.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:37 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 41688/251406 objects degraded (16.582%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:37.893 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:37.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:38.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:38.043 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:38.108 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:38.116 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:38.116 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:38.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:27:38.188 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:38] "GET /metrics HTTP/1.1" 200 36379 "" "Prometheus/2.43.0" 2023-09-23T15:27:38.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:38.253 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:38.315 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:38.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:38.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:38.420 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:38.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:38 smithi017 ceph-mon[222878]: pgmap v1098: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 8.0 MiB/s, 130 objects/s recovering 2023-09-23T15:27:38.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:38 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:27:38.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:38 smithi017 ceph-mon[215646]: pgmap v1098: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 8.0 MiB/s, 130 objects/s recovering 2023-09-23T15:27:38.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:38 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:27:38.525 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:38.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:38 smithi027 ceph-mon[195233]: pgmap v1098: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 41688/251406 objects degraded (16.582%); 8530/251406 objects misplaced (3.393%); 8.0 MiB/s, 130 objects/s recovering 2023-09-23T15:27:38.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:38 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:27:38.705 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:38.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:38.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:38.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:38.933 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:39.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:38 smithi027 conmon[344740]: 2023-09-23T15:27:38.761+0000 7f3503ce2540 -1 osd.7 358 log_to_monitors true 2023-09-23T15:27:39.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:27:39.541 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:39.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:39.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:39 smithi027 ceph-mon[195233]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:39 smithi027 ceph-mon[195233]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.600 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:27:39 smithi027 conmon[344740]: 2023-09-23T15:27:39.406+0000 7f34f4ac2700 -1 osd.7 358 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T15:27:39.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:39.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:39 smithi017 ceph-mon[215646]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:39 smithi017 ceph-mon[215646]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:39 smithi017 ceph-mon[222878]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:39 smithi017 ceph-mon[222878]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T15:27:39.747 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:27:39.776 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:39.906 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:39.919 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:39.945 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:40.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:40.054 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:40.111 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:40.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:40.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:40.327 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:40.430 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:27:40.463 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: pgmap v1099: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 21 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 90 op/s; 41677/248691 objects degraded (16.759%); 7453/248691 objects misplaced (2.997%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: pgmap v1099: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 21 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 90 op/s; 41677/248691 objects degraded (16.759%); 7453/248691 objects misplaced (2.997%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T15:27:40.464 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.465 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:40 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64529 7.e 7:7d16ebd8:::benchmark_data_smithi017_422824_object64528:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:27:40.571 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 595.72079372099995, 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.273188133, 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64529 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847675+0000", 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847696+0000", 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0910000000000001e-05 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847703+0000", 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5499999999999997e-06 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847709+0000", 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4909999999999996e-06 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847787+0000", 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8174000000000001e-05 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.847961+0000", 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017452599999999999 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.848087+0000", 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012575800000000001 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.850201+0000", 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021134719999999999 2023-09-23T15:27:40.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120707+0000", 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.270506674 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120747+0000", 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9394999999999998e-05 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120796+0000", 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9567000000000002e-05 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120863+0000", 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7117000000000001e-05 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64532 7.e 7:731a4c04:::benchmark_data_smithi017_422824_object64531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 595.71739466899999, 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28321638999999998, 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64532 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.579 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851074+0000", 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851073+0000", 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851107+0000", 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4767000000000001e-05 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851120+0000", 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2638e-05 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851128+0000", 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1519999999999992e-06 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851174+0000", 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6133e-05 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118070+0000", 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26689595700000002 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118233+0000", 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016251300000000001 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.125753+0000", 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075200090000000002 2023-09-23T15:27:40.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.128398+0000", 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026455459999999999 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134191+0000", 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057925729999999996 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134247+0000", 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6369999999999997e-05 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.134291+0000", 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3470999999999998e-05 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.585 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64534 7.5 7:a537e92b:::benchmark_data_smithi017_422824_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 595.71732711799996, 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28027905600000003, 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64534 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851142+0000", 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851141+0000", 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851169+0000", 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7999000000000001e-05 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851177+0000", 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5970000000000003e-06 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851182+0000", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4040000000000003e-06 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117946+0000", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.266763213 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118119+0000", 2023-09-23T15:27:40.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017326 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.118260+0000", 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000140767 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129302+0000", 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01104228 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129468+0000", 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166195 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131318+0000", 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018502740000000001 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131369+0000", 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0327000000000001e-05 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.131421+0000", 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2238999999999999e-05 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:64539 7.5 7:aff3ef34:::benchmark_data_smithi017_422824_object64538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T15:27:40.592 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 595.71727178000003, 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28619797000000002, 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 64539 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851197+0000", 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:44.851217+0000", 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0457000000000001e-05 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117948+0000", 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26673075899999998 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.117957+0000", 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9560000000000003e-06 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120513+0000", 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002556034 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120654+0000", 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014076799999999999 2023-09-23T15:27:40.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.120798+0000", 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000144394 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129322+0000", 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0085240860000000002 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.129497+0000", 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000174737 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137299+0000", 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0078019229999999997 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137345+0000", 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6551000000000002e-05 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:17:45.137395+0000", 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9852999999999999e-05 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126249 7.8 7:1dd9f547:::benchmark_data_smithi017_422824_object32379:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 508.79697836399998, 2023-09-23T15:27:40.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30324335800000002, 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126249 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:27:40.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771491+0000", 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771489+0000", 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771499+0000", 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0219e-05 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771514+0000", 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4487999999999999e-05 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771523+0000", 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771607+0000", 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3652999999999993e-05 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771729+0000", 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012218299999999999 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771846+0000", 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011697900000000001 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.779901+0000", 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0080551970000000001 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.782538+0000", 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002637393 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074627+0000", 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.29208838199999998 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074687+0000", 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0074000000000001e-05 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074734+0000", 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7129000000000002e-05 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:126250 7.8 7:12de4e94:::benchmark_data_smithi017_422824_object32380:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e250)", 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 508.79692137500001, 2023-09-23T15:27:40.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.30340214700000001, 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 126250 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771548+0000", 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771547+0000", 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771551+0000", 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.873e-06 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771562+0000", 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1371e-05 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.771569+0000", 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:27:40.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775770+0000", 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042005970000000004 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.775901+0000", 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000131066 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.776027+0000", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012558299999999999 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.781218+0000", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051909429999999999 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:11.788400+0000", 2023-09-23T15:27:40.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007182127 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074879+0000", 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28647872800000002 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074918+0000", 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9526000000000001e-05 2023-09-23T15:27:40.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:12.074950+0000", 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1764999999999998e-05 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147540 7.5 7:a064fe8e:::benchmark_data_smithi017_422824_object53670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.43976513600001, 2023-09-23T15:27:40.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35421415899999997, 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147540 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128704+0000", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128703+0000", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128709+0000", 2023-09-23T15:27:40.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2330000000000003e-06 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128731+0000", 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2753000000000001e-05 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128739+0000", 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6799999999999993e-06 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128795+0000", 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5846e-05 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.128906+0000", 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111094 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:27:40.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.129011+0000", 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010522 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.132677+0000", 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036654330000000001 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.134043+0000", 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013660079999999999 2023-09-23T15:27:40.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482827+0000", 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.348784551 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482880+0000", 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2193e-05 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.482918+0000", 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8566999999999999e-05 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147555 7.e 7:7b3c5bac:::benchmark_data_smithi017_422824_object53685:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.42997660700001, 2023-09-23T15:27:40.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35786704400000002, 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147555 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:27:40.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138492+0000", 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138491+0000", 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138497+0000", 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7359999999999998e-06 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138506+0000", 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4890000000000001e-06 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138512+0000", 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.344e-06 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.138557+0000", 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5674999999999999e-05 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428127+0000", 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28956913299999998 2023-09-23T15:27:40.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130603 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432003+0000", 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037461870000000002 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432016+0000", 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2327000000000001e-05 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496277+0000", 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.064261756000000003 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496320+0000", 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2994999999999997e-05 2023-09-23T15:27:40.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496359+0000", 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9004000000000001e-05 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:147557 7.5 7:a3342b39:::benchmark_data_smithi017_422824_object53687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 472.42754327300003, 2023-09-23T15:27:40.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35537888099999998, 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 147557 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140926+0000", 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140925+0000", 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.140930+0000", 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1170000000000003e-06 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428068+0000", 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.28713744600000002 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428083+0000", 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5126000000000001e-05 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428149+0000", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428257+0000", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010804899999999999 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.428331+0000", 2023-09-23T15:27:40.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4103000000000005e-05 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432563+0000", 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042319059999999997 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.432991+0000", 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00042866099999999999 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496219+0000", 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.063227189000000003 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496265+0000", 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6489000000000002e-05 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:48.496305+0000", 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9665000000000001e-05 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:148579 7.8 7:14985ae7:::benchmark_data_smithi017_422824_object54709:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 470.42568906299999, 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27762552400000001, 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.634 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 148579 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142780+0000", 2023-09-23T15:27:40.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142779+0000", 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142785+0000", 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7640000000000002e-06 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142795+0000", 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.042e-05 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.142816+0000", 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0897e-05 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143029+0000", 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021310799999999999 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.637 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143147+0000", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000117836 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4,5", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.143269+0000", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001225 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.151514+0000", 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0082451160000000003 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156566+0000", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005051303 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.156883+0000", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000317378 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420224+0000", 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.263340776 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420275+0000", 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.058e-05 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:19:50.420406+0000", 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013101000000000001 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.640 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165850 7.8 7:1eb76cbb:::benchmark_data_smithi017_422824_object71980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 439.11294991099999, 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.360395309, 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165850 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455519+0000", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455522+0000", 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.613e-06 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455530+0000", 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2649999999999996e-06 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455537+0000", 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1770000000000003e-06 2023-09-23T15:27:40.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455589+0000", 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2590999999999998e-05 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455703+0000", 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011421000000000001 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.455815+0000", 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000112071 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.460827+0000", 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050111419999999997 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.468359+0000", 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0075321540000000001 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815826+0000", 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34746724800000001 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815875+0000", 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8959999999999999e-05 2023-09-23T15:27:40.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.815914+0000", 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9490999999999998e-05 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165867 7.8 7:1ac8c744:::benchmark_data_smithi017_422824_object71997:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 439.09492965200002, 2023-09-23T15:27:40.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35163571799999999, 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165867 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:27:40.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473539+0000", 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473542+0000", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1200000000000002e-06 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473549+0000", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2429999999999999e-06 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473556+0000", 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1999999999999997e-06 2023-09-23T15:27:40.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473610+0000", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3355e-05 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.473737+0000", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126804 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.474340+0000", 2023-09-23T15:27:40.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060306200000000002 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482320+0000", 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079798620000000008 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482361+0000", 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1823000000000003e-05 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825087+0000", 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34272576999999999 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825135+0000", 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7395000000000003e-05 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.825175+0000", 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0583000000000001e-05 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.653 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165870 7.8 7:198dc808:::benchmark_data_smithi017_422824_object72000:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 439.092046751, 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.35318146500000003, 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.654 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165870 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476422+0000", 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476421+0000", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476428+0000", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3429999999999998e-06 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476446+0000", 2023-09-23T15:27:40.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7997000000000001e-05 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476455+0000", 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478546+0000", 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002091192 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478670+0000", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012399199999999999 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,5", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.478787+0000", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116907 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482437+0000", 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036504319999999999 2023-09-23T15:27:40.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.483321+0000", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00088376899999999996 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829506+0000", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.34618508100000001 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829557+0000", 2023-09-23T15:27:40.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0427999999999999e-05 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.829604+0000", 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7024000000000002e-05 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:165871 7.5 7:a7f75625:::benchmark_data_smithi017_422824_object72001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e271)", 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:27:40.660 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 439.09193998000001, 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27785160399999997, 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 165871 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476529+0000", 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476528+0000", 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476533+0000", 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.318e-06 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476542+0000", 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7059999999999999e-06 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.663 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.476547+0000", 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6260000000000003e-06 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477473+0000", 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00092625100000000001 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477603+0000", 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129806 2023-09-23T15:27:40.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.477726+0000", 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012271 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.481919+0000", 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041935080000000003 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.482075+0000", 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015598599999999999 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754297+0000", 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27222164999999998 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754340+0000", 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2945000000000003e-05 2023-09-23T15:27:40.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:21.754381+0000", 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0768e-05 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179356 7.e 7:7f6e6587:::benchmark_data_smithi017_422824_object85486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:27:40.667 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 418.879717551, 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.33625833300000002, 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179356 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688752+0000", 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688750+0000", 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.669 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688756+0000", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2060000000000004e-06 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688768+0000", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2061e-05 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688776+0000", 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0299999999999994e-06 2023-09-23T15:27:40.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688855+0000", 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8975999999999997e-05 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.688972+0000", 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000116957 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.689084+0000", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000111329 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.692868+0000", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003784379 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.695532+0000", 2023-09-23T15:27:40.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026644920000000001 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024919+0000", 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32938610699999998 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.024969+0000", 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0522000000000002e-05 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025010+0000", 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0778000000000002e-05 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45479.0:179365 7.e 7:78918128:::benchmark_data_smithi017_422824_object85495:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e276)", 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 418.86750376700002, 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32416663400000001, 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.674 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45479", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/106965224", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 179365 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700965+0000", 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700964+0000", 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700970+0000", 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6659999999999996e-06 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700981+0000", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1184e-05 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.700987+0000", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1789999999999996e-06 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701026+0000", 2023-09-23T15:27:40.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8707000000000001e-05 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701105+0000", 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9220000000000004e-05 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.701175+0000", 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9696000000000005e-05 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704027+0000", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0028522439999999999 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:41.704091+0000", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3796999999999999e-05 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025048+0000", 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32095755500000001 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025094+0000", 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5488999999999998e-05 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:20:42.025132+0000", 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8133999999999998e-05 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.45809.0:168782 8.d e330/287 8:b92c966b:::benchmark_data_smithi017_485823_object78645:head v 330'10397, mlcod=330'10395)", 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 162.43536265, 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078966947999999995, 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:27:40.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133106+0000", 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133112+0000", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7420000000000003e-06 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133119+0000", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7970000000000001e-06 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133124+0000", 2023-09-23T15:27:40.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6e-06 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133166+0000", 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1900000000000002e-05 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.133188+0000", 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2391999999999998e-05 2023-09-23T15:27:40.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212024+0000", 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078835815000000004 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:24:58.212073+0000", 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9292000000000002e-05 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:31383 9.2 e345/343 9:41cb1c0b:::benchmark_data_smithi017_543597_object31382:head v 345'1938, mlcod=345'1936)", 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:25:59.626624+0000", 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 100.941845116, 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.066800018000000003, 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626624+0000", 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626624+0000", 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626623+0000", 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:40.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626654+0000", 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1210000000000001e-05 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626671+0000", 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6934000000000001e-05 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626680+0000", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626731+0000", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0457999999999997e-05 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626753+0000", 2023-09-23T15:27:40.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1851000000000001e-05 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.693361+0000", 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.066608604000000002 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.693424+0000", 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2619999999999993e-05 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.690 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:31387 9.0 e345/339 9:0c1bcd92:::benchmark_data_smithi017_543597_object31386:head v 345'2048, mlcod=345'2046)", 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:25:59.626698+0000", 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 100.941770744, 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.067127636000000004, 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626698+0000", 2023-09-23T15:27:40.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626698+0000", 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626698+0000", 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626724+0000", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.565e-05 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626731+0000", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.661e-06 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626736+0000", 2023-09-23T15:27:40.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0760000000000002e-06 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626805+0000", 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8499999999999998e-05 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.626826+0000", 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0767e-05 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.693780+0000", 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.066954628000000002 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.693826+0000", 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5720999999999998e-05 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:31401 9.2 e345/343 9:43c8104a:::benchmark_data_smithi017_543597_object31400:head v 345'1939, mlcod=345'1936)", 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:25:59.629152+0000", 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 100.939317301, 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.065939172000000004, 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.629152+0000", 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.629152+0000", 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.629151+0000", 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:27:40.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.629177+0000", 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6352000000000001e-05 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.689261+0000", 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.060083803999999998 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.689277+0000", 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6123000000000001e-05 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.690413+0000", 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011357909999999999 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.690433+0000", 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0361999999999999e-05 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.695046+0000", 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046124199999999999 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:25:59.695091+0000", 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5172000000000001e-05 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:27:40.701 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:27:40.701 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:40.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: pgmap v1099: 119 pgs: 1 activating+undersized+degraded+remapped, 17 active+clean+remapped, 21 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 67 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 102 B/s rd, 102 B/s wr, 90 op/s; 41677/248691 objects degraded (16.759%); 7453/248691 objects misplaced (2.997%); 8.8 MiB/s, 141 objects/s recovering 2023-09-23T15:27:40.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: failed to encode map e373 with expected crc 2023-09-23T15:27:40.701 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T15:27:40.702 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T15:27:40.702 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: from='osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.702 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi027", "root=default"]}]: dispatch 2023-09-23T15:27:40.702 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:40 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:27:40.745 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:40.748 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:40.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:40.878 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:41.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272] boot 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[222878]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272] boot 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:41 smithi017 ceph-mon[215646]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: osd.7 [v2:172.21.15.27:6824/1440800272,v1:172.21.15.27:6825/1440800272] boot 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1349818573' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: osdmap e374: 8 total, 8 up, 8 in 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: failed to encode map e374 with expected crc 2023-09-23T15:27:41.709 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:41 smithi027 ceph-mon[195233]: osdmap e375: 8 total, 8 up, 8 in 2023-09-23T15:27:41.937 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:27:41.937 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:27:41.937 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:27:41.937 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:27:42.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:27:42.599 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:42.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e359 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e374 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: pgmap v1102: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 112 KiB/s wr, 451 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: 1.19 scrub starts 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 40103/240651 objects degraded (16.664%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[215646]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e359 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e374 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: pgmap v1102: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 112 KiB/s wr, 451 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: 1.19 scrub starts 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 40103/240651 objects degraded (16.664%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:42.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:42 smithi017 ceph-mon[222878]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e359 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e374 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: pgmap v1102: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 127 B/s rd, 112 KiB/s wr, 451 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: failed to encode map e375 with expected crc 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: 1.19 scrub starts 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 40103/240651 objects degraded (16.664%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:42.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:42 smithi027 ceph-mon[195233]: osdmap e376: 8 total, 8 up, 8 in 2023-09-23T15:27:42.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:42.763 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:27:42.799 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:42.911 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:42.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:42.915 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:43.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:43.056 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:43.101 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:43.126 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:43.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:43.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:43.141 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:43.162 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:43.186 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:43.211 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:43.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:43.234 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:43.258 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:43.258 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:43.287 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:43.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:43.319 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:27:43.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.349 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.350 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:27:43.393 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:43.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:43.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:43.506 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:43.585 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:43.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/445248865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[215646]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/445248865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:43.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:43 smithi017 ceph-mon[222878]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T15:27:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: failed to encode map e376 with expected crc 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/445248865' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:43.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:43 smithi027 ceph-mon[195233]: osdmap e377: 8 total, 8 up, 8 in 2023-09-23T15:27:44.616 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:27:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: pgmap v1105: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 224 KiB/s wr, 676 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:27:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[215646]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: pgmap v1105: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 224 KiB/s wr, 676 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:44 smithi017 ceph-mon[222878]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: pgmap v1105: 119 pgs: 1 peering, 17 active+clean+remapped, 1 active+recovering+undersized+remapped, 21 active+undersized, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 67 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 224 KiB/s wr, 676 op/s; 40103/240651 objects degraded (16.664%); 7598/240651 objects misplaced (3.157%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T15:27:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: failed to encode map e377 with expected crc 2023-09-23T15:27:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:44 smithi027 ceph-mon[195233]: failed to encode map e377 with expected crc 2023-09-23T15:27:45.947 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:45.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:45.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:46.054 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:46.099 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:46.177 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:46.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:46.187 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:46.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:46.376 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:46.601 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:46.608 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:46.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:46 smithi017 ceph-mon[222878]: pgmap v1107: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 4 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 103 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 21176/237756 objects degraded (8.907%); 2543/237756 objects misplaced (1.070%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T15:27:46.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:46 smithi017 ceph-mon[215646]: pgmap v1107: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 4 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 103 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 21176/237756 objects degraded (8.907%); 2543/237756 objects misplaced (1.070%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T15:27:46.709 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:46.788 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:46.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:46 smithi027 ceph-mon[195233]: pgmap v1107: 119 pgs: 1 peering, 2 active+recovering+undersized+remapped, 4 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 5 active+undersized+degraded, 103 active+clean; 4.8 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 286 op/s; 21176/237756 objects degraded (8.907%); 2543/237756 objects misplaced (1.070%); 3.7 MiB/s, 57 objects/s recovering 2023-09-23T15:27:46.856 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:46.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:46.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:46.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:46.988 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:47.095 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:47.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:47.205 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:47.279 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:47.478 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:47.511 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:47.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:47.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:47.602 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:27:47.602 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:27:47.621 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:47.692 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:47.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:47 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:27:47.804 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:47 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 21176/237756 objects degraded (8.907%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:47.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:47 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:27:47.807 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:47 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 21176/237756 objects degraded (8.907%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:47.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:47 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:27:47.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:47 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 21176/237756 objects degraded (8.907%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:48.031 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:27:48.096 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:48] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.43.0" 2023-09-23T15:27:48.235 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:48.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:48.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:48.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:27:48.374 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:48.420 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:27:48.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:48.428 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:27:48.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:48.583 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:48.665 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:48.671 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:48 smithi027 ceph-mon[195233]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 2429/234282 objects degraded (1.037%); 2121/234282 objects misplaced (0.905%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:27:48.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:48.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:48.773 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:48.844 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:48 smithi017 ceph-mon[215646]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 2429/234282 objects degraded (1.037%); 2121/234282 objects misplaced (0.905%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:27:48.844 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:48 smithi017 ceph-mon[222878]: pgmap v1108: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 353 op/s; 2429/234282 objects degraded (1.037%); 2121/234282 objects misplaced (0.905%); 6.7 MiB/s, 106 objects/s recovering 2023-09-23T15:27:48.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:49.043 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:49.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:49.069 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:27:49.169 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:49.251 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:49.521 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:49.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:49.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:49.626 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:49.726 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:49.810 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:49.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:49.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:49.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:49.983 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:50.082 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:50.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:50.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:50.191 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:50.277 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:50.524 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:27:50.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[222878]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 2429/232065 objects degraded (1.047%); 2121/232065 objects misplaced (0.914%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T15:27:50.547 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2892120736' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:50.548 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:50.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[215646]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 2429/232065 objects degraded (1.047%); 2121/232065 objects misplaced (0.914%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T15:27:50.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2892120736' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:50.549 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:50 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:50.555 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:50.561 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:50.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:50.762 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:50.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:50 smithi027 ceph-mon[195233]: pgmap v1109: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 383 op/s; 2429/232065 objects degraded (1.047%); 2121/232065 objects misplaced (0.914%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T15:27:50.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2892120736' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:50.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:50 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:27:51.317 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:51.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:51.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:51.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:51.490 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:51.531 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:51.534 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.640 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:27:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.641 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:27:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T15:27:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.642 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:27:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:51.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:51.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: osdmap e378: 8 total, 8 up, 8 in 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: failed to encode map e378 with expected crc 2023-09-23T15:27:51.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:27:51.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:51.925 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:52.019 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:52.022 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:52.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:52.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:52.164 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:52.345 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:52.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:52.360 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:52.461 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:52.549 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:52.766 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:52.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:52.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:52 smithi027 ceph-mon[195233]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T15:27:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:52 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/221868 objects degraded (1.095%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/989378414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.880 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[222878]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/221868 objects degraded (1.095%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/989378414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[215646]: pgmap v1111: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/221868 objects degraded (1.095%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/989378414' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:27:52.952 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:53.133 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:53.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:53.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:53.244 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:53.275 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:27:53.322 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:53.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:53 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:53.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:53 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:53.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:53 smithi027 ceph-mon[195233]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T15:27:53.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[222878]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T15:27:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:27:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:27:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:53 smithi017 ceph-mon[215646]: osdmap e379: 8 total, 8 up, 8 in 2023-09-23T15:27:54.178 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:54.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:54.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:54.288 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:27:54.316 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:54.752 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:27:54.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:54.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:27:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:54 smithi027 ceph-mon[195233]: pgmap v1112: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:27:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:54 smithi027 ceph-mon[195233]: 1.7 scrub starts 2023-09-23T15:27:54.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:54 smithi027 ceph-mon[195233]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:54 smithi027 ceph-mon[195233]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:54 smithi027 ceph-mon[195233]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:27:54.917 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[215646]: pgmap v1112: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[215646]: 1.7 scrub starts 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[215646]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[215646]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[215646]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[222878]: pgmap v1112: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 2429/221868 objects degraded (1.095%); 1860/221868 objects misplaced (0.838%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T15:27:54.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[222878]: 1.7 scrub starts 2023-09-23T15:27:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[222878]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[222878]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.950 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:54 smithi017 ceph-mon[222878]: failed to encode map e379 with expected crc 2023-09-23T15:27:54.981 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:27:54.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:54.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:27:55.088 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:27:55.160 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:55.271 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:27:55.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:55.278 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:27:55.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:55.430 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:55.464 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:55.545 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:55.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:55.643 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:27:55.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:55.689 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:27:55.731 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:27:55.745 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:55.794 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:27:55.831 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:27:55.878 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:27:55.932 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:27:55.948 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:55.949 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:27:55.949 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:27:55.950 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:27:55.950 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:27:55.950 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:27:55.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:55.965 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:55.988 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:27:56.022 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.022 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.022 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.023 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:27:56.067 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:56.109 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:56.188 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:56.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:56.300 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:56.334 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:56.410 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:56.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:56.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:56.517 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:56.580 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:56.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[222878]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2429/215985 objects degraded (1.125%); 1597/215985 objects misplaced (0.739%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:56.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1988635132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:56.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:56.599 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:56.663 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:56.671 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:56 smithi027 ceph-mon[195233]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2429/215985 objects degraded (1.125%); 1597/215985 objects misplaced (0.739%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:56.671 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1988635132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:56.671 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:56 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:56.671 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:56 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:56.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:56.773 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:27:56.818 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:56.920 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:27:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:56.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:27:56.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[215646]: pgmap v1114: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 762 op/s; 2429/215985 objects degraded (1.125%); 1597/215985 objects misplaced (0.739%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:56.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1988635132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:27:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:56 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:57.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:27:57.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:27:57.077 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:57.141 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:27:57.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:57.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:27:57.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:27:57.280 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:57.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:57 smithi027 ceph-mon[195233]: pgmap v1115: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 2429/210105 objects degraded (1.156%); 1597/210105 objects misplaced (0.760%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:57.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:57 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/215985 objects degraded (1.125%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:57 smithi017 ceph-mon[222878]: pgmap v1115: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 2429/210105 objects degraded (1.156%); 1597/210105 objects misplaced (0.760%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:57 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/215985 objects degraded (1.125%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:57 smithi017 ceph-mon[215646]: pgmap v1115: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 2429/210105 objects degraded (1.156%); 1597/210105 objects misplaced (0.760%); 4.1 MiB/s, 65 objects/s recovering 2023-09-23T15:27:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:57 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/215985 objects degraded (1.125%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:27:58.010 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:27:58.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:58.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:27:58.120 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:27:58.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:58.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:27:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:27:58] "GET /metrics HTTP/1.1" 200 36242 "" "Prometheus/2.43.0" 2023-09-23T15:27:58.706 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:27:58.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:58.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:27:58.810 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:58.850 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:58.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:27:58.913 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:58.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:58.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:59.021 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:27:59.060 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:59.102 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:27:59.125 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:27:59.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:59.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:27:59.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:27:59.244 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:59.244 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:27:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:59.245 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:59.246 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:27:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:59.267 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:59.335 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:27:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:59.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:27:59.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:27:59.523 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:27:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:27:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:27:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:27:59.843 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:27:59.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:27:59.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:27:59.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:00.024 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:00.234 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:28:00.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:00 smithi017 ceph-mon[215646]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2429/207987 objects degraded (1.168%); 1597/207987 objects misplaced (0.768%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T15:28:00.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:28:00.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:00 smithi017 ceph-mon[222878]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2429/207987 objects degraded (1.168%); 1597/207987 objects misplaced (0.768%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T15:28:00.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:28:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:00 smithi027 ceph-mon[195233]: pgmap v1116: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 684 op/s; 2429/207987 objects degraded (1.168%); 1597/207987 objects misplaced (0.768%); 2.0 MiB/s, 31 objects/s recovering 2023-09-23T15:28:00.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:28:00.930 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:28:01.148 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:01.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:01.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:01.325 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:01.414 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:01.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:01.429 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:01.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[222878]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[222878]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:28:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[222878]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T15:28:01.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[222878]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[215646]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[215646]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:28:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[215646]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T15:28:01.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:01 smithi017 ceph-mon[215646]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.528 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:01.530 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:01.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:01 smithi027 ceph-mon[195233]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:01 smithi027 ceph-mon[195233]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:01 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1756958550' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:28:01.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:01 smithi027 ceph-mon[195233]: osdmap e380: 8 total, 8 up, 8 in 2023-09-23T15:28:01.571 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:01 smithi027 ceph-mon[195233]: failed to encode map e380 with expected crc 2023-09-23T15:28:01.574 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:01.646 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:01.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:01.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:01.751 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:01.804 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:01.862 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:01.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:01.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:01.880 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:28:01.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:02.143 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:02.240 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:02 smithi027 ceph-mon[195233]: pgmap v1118: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:28:02.420 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:02.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:02 smithi017 ceph-mon[222878]: pgmap v1118: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:28:02.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:02 smithi017 ceph-mon[215646]: pgmap v1118: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 959 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T15:28:02.574 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:02.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:02.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:02.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:02.761 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:03.329 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:28:03.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/198816 objects degraded (1.222%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:03.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1793877596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.355 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/198816 objects degraded (1.222%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:03.355 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1793877596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.356 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:03 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/198816 objects degraded (1.222%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:03.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1793877596' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:03.607 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:03.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:03.719 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:03.756 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:03.819 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:03.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:03.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:03.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:04.038 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: failed to encode map e381 with expected crc 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T15:28:04.243 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[215646]: failed to encode map e381 with expected crc 2023-09-23T15:28:04.244 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:28:04.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: pgmap v1119: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2429/198816 objects degraded (1.222%); 1313/198816 objects misplaced (0.660%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T15:28:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: failed to encode map e381 with expected crc 2023-09-23T15:28:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T15:28:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:04 smithi027 ceph-mon[195233]: failed to encode map e381 with expected crc 2023-09-23T15:28:04.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:04.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: failed to encode map e381 with expected crc 2023-09-23T15:28:04.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:04.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: osdmap e381: 8 total, 8 up, 8 in 2023-09-23T15:28:04.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:04 smithi017 ceph-mon[222878]: failed to encode map e381 with expected crc 2023-09-23T15:28:05.567 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:05.575 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:05 smithi017 ceph-mon[222878]: failed to encode map e381 with expected crc 2023-09-23T15:28:05.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:05.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:05.576 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:05 smithi017 ceph-mon[215646]: failed to encode map e381 with expected crc 2023-09-23T15:28:05.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:05 smithi027 ceph-mon[195233]: failed to encode map e381 with expected crc 2023-09-23T15:28:05.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:05.772 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:05.931 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:05.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:05.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:06.057 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:06.079 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:28:06.079 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:28:06.079 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:28:06.079 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:28:06.079 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50576 8.b 8:d9711445:::benchmark_data_smithi017_485823_object50575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 369.87815980099998, 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1259012579999998, 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50576 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.080 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192256+0000", 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.081 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192254+0000", 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192281+0000", 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7236000000000001e-05 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192304+0000", 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3314000000000001e-05 2023-09-23T15:28:06.082 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192314+0000", 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.6269999999999995e-06 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192529+0000", 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000214637 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.083 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192538+0000", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119735+0000", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92719635 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119749+0000", 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4219999999999999e-05 2023-09-23T15:28:06.084 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307742+0000", 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.187992758 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307887+0000", 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014497199999999999 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.085 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.308028+0000", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014144799999999999 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314256+0000", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0062275179999999996 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314333+0000", 2023-09-23T15:28:06.086 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6796000000000001e-05 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315870+0000", 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001537524 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315924+0000", 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3733999999999997e-05 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.087 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.318157+0000", 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022332469999999998 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50579 8.b 8:dd05f5d9:::benchmark_data_smithi017_485823_object50578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 369.87804601200003, 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.129065862, 2023-09-23T15:28:06.088 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50579 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.089 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192370+0000", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192369+0000", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192402+0000", 2023-09-23T15:28:06.090 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2693e-05 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192410+0000", 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8590000000000005e-06 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192415+0000", 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8710000000000002e-06 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.091 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192564+0000", 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014868300000000001 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.192571+0000", 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1160000000000001e-06 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119777+0000", 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92720541499999998 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.092 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.119787+0000", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9520000000000006e-06 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312151+0000", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1923646489999999 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312321+0000", 2023-09-23T15:28:06.093 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017003 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.312518+0000", 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196746 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.314638+0000", 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021200590000000001 2023-09-23T15:28:06.094 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315028+0000", 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00038974000000000002 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319363+0000", 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043355499999999996 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.095 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319441+0000", 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7822000000000001e-05 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.321435+0000", 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019941450000000001 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83104 9.c 9:3b5b5686:::benchmark_data_smithi017_543597_object83103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.096 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760899516999999, 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.450679148000001, 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83104 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.097 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309514+0000", 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.098 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309542+0000", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7407e-05 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309556+0000", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4404e-05 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309570+0000", 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3918e-05 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.099 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313812+0000", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042423749999999996 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313824+0000", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.166e-05 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.754362+0000", 2023-09-23T15:28:06.100 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.440538513 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.760195+0000", 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058326949999999997 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83557 9.c 9:3ca96c17:::benchmark_data_smithi017_543597_object83556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.101 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760832675000003, 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.541859289, 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83557 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.102 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309582+0000", 2023-09-23T15:28:06.103 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309604+0000", 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.207e-05 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309614+0000", 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.679e-06 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.104 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309621+0000", 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9110000000000004e-06 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313846+0000", 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042255110000000004 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.105 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313853+0000", 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9179999999999997e-06 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.774722+0000", 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.460868532999999 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.781443+0000", 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067207079999999997 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.106 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851313+0000", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.069870627000000005 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851442+0000", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128901 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83560 9.c 9:3d5597f0:::benchmark_data_smithi017_543597_object83559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:28:06.107 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760787231999998, 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.543202913, 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83560 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.108 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309648+0000", 2023-09-23T15:28:06.109 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0092999999999999e-05 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309656+0000", 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8730000000000007e-06 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309662+0000", 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.110 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313873+0000", 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042114159999999999 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313880+0000", 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9659999999999999e-06 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.782319+0000", 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.468439010999999 2023-09-23T15:28:06.111 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.789923+0000", 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076032809999999999 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.801626+0000", 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011703523 2023-09-23T15:28:06.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.836893+0000", 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035267251999999999 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851529+0000", 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014635815 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851566+0000", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7289000000000002e-05 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851601+0000", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5088e-05 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.852831+0000", 2023-09-23T15:28:06.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001229759 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83591 9.c 9:3687f2ca:::benchmark_data_smithi017_543597_object83590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760748360000001, 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.545316396, 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.115 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83591 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309686+0000", 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9743999999999999e-05 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.117 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309694+0000", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.729e-06 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309700+0000", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9669999999999998e-06 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313901+0000", 2023-09-23T15:28:06.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042010720000000001 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313908+0000", 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8669999999999996e-06 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.809499+0000", 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.495590667 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.816644+0000", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071449620000000004 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.828154+0000", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011510423000000001 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.840204+0000", 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012049908 2023-09-23T15:28:06.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.852892+0000", 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012687617 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.853952+0000", 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010600189999999999 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.853989+0000", 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7109e-05 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.854984+0000", 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00099478700000000001 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83628 9.c 9:3ed93163:::benchmark_data_smithi017_543597_object83627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.122 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760709454000001, 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.55767582, 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83628 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.123 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309730+0000", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4318e-05 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309739+0000", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309746+0000", 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1019999999999999e-06 2023-09-23T15:28:06.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313930+0000", 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041835470000000001 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313937+0000", 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8249999999999999e-06 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855030+0000", 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.541093047 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855155+0000", 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124931 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855264+0000", 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010964299999999999 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864529+0000", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0092652810000000002 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864584+0000", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5031000000000002e-05 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864715+0000", 2023-09-23T15:28:06.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130536 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864771+0000", 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5627000000000001e-05 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867382+0000", 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002611262 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83637 9.c 9:3ca6bb70:::benchmark_data_smithi017_543597_object83636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760663909000002, 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.564530451, 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83637 2023-09-23T15:28:06.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.131 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309751+0000", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309779+0000", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7466000000000001e-05 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309787+0000", 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9789999999999993e-06 2023-09-23T15:28:06.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309793+0000", 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2539999999999998e-06 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313957+0000", 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041637480000000001 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313963+0000", 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.781e-06 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857515+0000", 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.543551269 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857634+0000", 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011945 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857744+0000", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010961199999999999 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864649+0000", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069050509999999997 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864661+0000", 2023-09-23T15:28:06.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183e-05 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872189+0000", 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007528814 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872246+0000", 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6660999999999998e-05 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874282+0000", 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020360729999999998 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83643 9.c 9:37b8a565:::benchmark_data_smithi017_543597_object83642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760617510000003, 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.574886889, 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.137 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83643 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309817+0000", 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9907000000000001e-05 2023-09-23T15:28:06.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309825+0000", 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9619999999999993e-06 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309831+0000", 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6400000000000002e-06 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313986+0000", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041548380000000001 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313993+0000", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7379999999999997e-06 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.859721+0000", 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.545728274 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.859860+0000", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013950799999999999 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.860015+0000", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015417 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864666+0000", 2023-09-23T15:28:06.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046517329999999999 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869847+0000", 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051809680000000002 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882639+0000", 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012791497000000001 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882710+0000", 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1630000000000004e-05 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884685+0000", 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001974515 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83670 9.c 9:3f0b0e67:::benchmark_data_smithi017_543597_object83669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760579493000002, 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.576675178, 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83670 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:28:06.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309887+0000", 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1570999999999998e-05 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309896+0000", 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5480000000000002e-06 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.147 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309902+0000", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2110000000000003e-06 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314013+0000", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041110369999999997 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314020+0000", 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5479999999999998e-06 2023-09-23T15:28:06.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862483+0000", 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.548463865 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862614+0000", 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013095899999999999 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862718+0000", 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103745 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869859+0000", 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007141077 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869905+0000", 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5868000000000002e-05 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884831+0000", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014925905 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884902+0000", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1285000000000006e-05 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886511+0000", 2023-09-23T15:28:06.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001608979 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83675 9.c 9:34036259:::benchmark_data_smithi017_543597_object83674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760507957000002, 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.57811843, 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.152 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83675 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:28:06.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309907+0000", 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309953+0000", 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5658000000000002e-05 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309961+0000", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9570000000000002e-06 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309968+0000", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9099999999999999e-06 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314039+0000", 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040712120000000003 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314046+0000", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5000000000000002e-06 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867440+0000", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.553393381999999 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867560+0000", 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120005 2023-09-23T15:28:06.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867672+0000", 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011218 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872157+0000", 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044847680000000001 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874297+0000", 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021403350000000002 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886622+0000", 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012324701 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886696+0000", 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4108000000000003e-05 2023-09-23T15:28:06.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888026+0000", 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013302500000000001 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83705 9.c 9:344ed245:::benchmark_data_smithi017_543597_object83704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760441792999998, 2023-09-23T15:28:06.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.579326626, 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83705 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309973+0000", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310021+0000", 2023-09-23T15:28:06.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7636000000000003e-05 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310030+0000", 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0229999999999998e-06 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310036+0000", 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1680000000000001e-06 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314066+0000", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040297309999999999 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314073+0000", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7900000000000002e-06 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869943+0000", 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.555870074 2023-09-23T15:28:06.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.870056+0000", 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113826 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.870161+0000", 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104693 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877402+0000", 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072409420000000002 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877418+0000", 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5412000000000001e-05 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888089+0000", 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010670999 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888159+0000", 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0282000000000002e-05 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889300+0000", 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011415609999999999 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83714 9.c 9:3ecd0e01:::benchmark_data_smithi017_543597_object83713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760374507999998, 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.583786938999999, 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83714 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.167 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:28:06.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310082+0000", 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0963000000000001e-05 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310090+0000", 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2069999999999995e-06 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310095+0000", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.755e-06 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314094+0000", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003998896 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314101+0000", 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7120000000000003e-06 2023-09-23T15:28:06.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874330+0000", 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.560229039999999 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874474+0000", 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014405 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874604+0000", 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012965900000000001 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882580+0000", 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079760430000000004 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884707+0000", 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021275420000000001 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.892134+0000", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074264400000000003 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.892199+0000", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4642999999999998e-05 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893828+0000", 2023-09-23T15:28:06.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00162948 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83736 9.c 9:31e1d931:::benchmark_data_smithi017_543597_object83735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760314694999998, 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.585146602, 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.174 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83736 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:28:06.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310100+0000", 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310145+0000", 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4842999999999997e-05 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310154+0000", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3890000000000005e-06 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310159+0000", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8950000000000003e-06 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314121+0000", 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039618409999999998 2023-09-23T15:28:06.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314129+0000", 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9820000000000008e-06 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877437+0000", 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.563308244 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.178 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877563+0000", 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012571699999999999 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877684+0000", 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012044699999999999 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884719+0000", 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070358269999999997 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884772+0000", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2868000000000003e-05 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893901+0000", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091287179999999992 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893958+0000", 2023-09-23T15:28:06.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7123999999999997e-05 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895247+0000", 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012892579999999999 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83741 9.c 9:327e7b25:::benchmark_data_smithi017_543597_object83740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:28:06.181 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760251249, 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.586280486, 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83741 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:28:06.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310208+0000", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3733999999999998e-05 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310216+0000", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.303e-06 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310222+0000", 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0059999999999998e-06 2023-09-23T15:28:06.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314151+0000", 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039289360000000001 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314157+0000", 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6100000000000002e-06 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.879806+0000", 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.565648801 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.880006+0000", 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019943000000000001 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.880426+0000", 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000419978 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886532+0000", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061066209999999996 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886567+0000", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5091e-05 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895321+0000", 2023-09-23T15:28:06.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087539499999999999 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895407+0000", 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5644999999999997e-05 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896445+0000", 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001037865 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.188 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83768 9.c 9:351643c6:::benchmark_data_smithi017_543597_object83767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760188739999997, 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.587392286, 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83768 2023-09-23T15:28:06.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310226+0000", 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310270+0000", 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3220000000000003e-05 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310278+0000", 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4339999999999992e-06 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310284+0000", 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2099999999999998e-06 2023-09-23T15:28:06.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314177+0000", 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003892543 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314183+0000", 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4470000000000003e-06 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.194 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889341+0000", 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.575158296 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889546+0000", 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000204611 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889767+0000", 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000221153 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896466+0000", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066984519999999997 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896498+0000", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2681000000000001e-05 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896558+0000", 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9395999999999998e-05 2023-09-23T15:28:06.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896623+0000", 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5241999999999996e-05 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897619+0000", 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00099607899999999997 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83784 9.c 9:3b4fc178:::benchmark_data_smithi017_543597_object83783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760126067000002, 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.600940644, 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83784 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.198 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310337+0000", 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.795e-05 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310346+0000", 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6170000000000007e-06 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310352+0000", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3069999999999999e-06 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314206+0000", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038536709999999999 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314214+0000", 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4409999999999993e-06 2023-09-23T15:28:06.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897667+0000", 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.583452796 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897794+0000", 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012709700000000001 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.202 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897922+0000", 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012811800000000001 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909243+0000", 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01132096 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909259+0000", 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6317e-05 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909284+0000", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5222e-05 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909338+0000", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4014000000000002e-05 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911230+0000", 2023-09-23T15:28:06.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018916580000000001 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83788 9.c 9:3675609c:::benchmark_data_smithi017_543597_object83787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 41.760058035999997, 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.602168643000001, 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.205 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83788 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:28:06.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310357+0000", 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310410+0000", 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3470000000000001e-05 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310420+0000", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1230000000000006e-06 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310426+0000", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.55e-06 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314234+0000", 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038076410000000001 2023-09-23T15:28:06.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314241+0000", 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8789999999999997e-06 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900244+0000", 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.586003567000001 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900448+0000", 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020388399999999999 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900672+0000", 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224396 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911301+0000", 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010628687 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.210 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911316+0000", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5136e-05 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911348+0000", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1900000000000003e-05 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911404+0000", 2023-09-23T15:28:06.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5878000000000003e-05 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.912526+0000", 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001122086 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88021 9.c 9:32f7ad96:::benchmark_data_smithi017_543597_object4230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:28:06.212 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.721868753999999, 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0658238880000002, 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88021 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348546+0000", 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348551+0000", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3070000000000002e-06 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348561+0000", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0054000000000001e-05 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348569+0000", 2023-09-23T15:28:06.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.352466+0000", 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038970519999999998 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.352516+0000", 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0056000000000003e-05 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381306+0000", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028790685 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381435+0000", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012852800000000001 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381595+0000", 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015973300000000001 2023-09-23T15:28:06.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.388797+0000", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007202623 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.392930+0000", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041327489999999998 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415879+0000", 2023-09-23T15:28:06.218 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022948867000000001 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415891+0000", 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2160999999999999e-05 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418056+0000", 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0021654120000001 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418069+0000", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2099999999999999e-05 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.396257+0000", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97818863700000003 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.396349+0000", 2023-09-23T15:28:06.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1415999999999993e-05 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414241+0000", 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017892049 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414371+0000", 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013000300000000001 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.221 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88027 9.c 9:3e9c13fa:::benchmark_data_smithi017_543597_object4236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 25.709200300999999, 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0608875040000001, 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:06.222 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88027 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361214+0000", 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361221+0000", 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7739999999999999e-06 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:06.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361233+0000", 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2184000000000001e-05 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361243+0000", 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0249e-05 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.363856+0000", 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026123880000000002 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.364327+0000", 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047180900000000001 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415920+0000", 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051592551 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415929+0000", 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3179999999999996e-06 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418103+0000", 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.002173486 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418112+0000", 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1540000000000008e-06 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397094+0000", 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97898225000000005 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397186+0000", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.234e-05 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397276+0000", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9622000000000004e-05 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.410961+0000", 2023-09-23T15:28:06.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013685075999999999 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414425+0000", 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034636319999999999 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.420393+0000", 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059684600000000001 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.420440+0000", 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7029e-05 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.422103+0000", 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016625019999999999 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:06.231 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:28:06.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:06.232 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:06.232 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:06.232 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:06.333 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:06.372 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:06 smithi027 ceph-mon[195233]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2429/193278 objects degraded (1.257%); 981/193278 objects misplaced (0.508%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:06.376 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:06.471 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:06.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:06.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:06.508 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:06.526 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:06.568 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:06.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:06.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:06 smithi017 ceph-mon[215646]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2429/193278 objects degraded (1.257%); 981/193278 objects misplaced (0.508%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:06.621 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:06 smithi017 ceph-mon[222878]: pgmap v1121: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 2429/193278 objects degraded (1.257%); 981/193278 objects misplaced (0.508%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:06.623 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:06.827 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:06.877 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:06.916 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:06.957 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:06.991 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:07.013 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:28:07.166 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:07.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:07.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:07.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:07.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/153585920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:07.313 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:07 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/193278 objects degraded (1.257%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:07.313 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/153585920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:07.313 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:07 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/193278 objects degraded (1.257%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:07.330 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:07.355 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:28:07.364 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/153585920' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:07.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:07 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/193278 objects degraded (1.257%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:07.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:28:07.804 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:08.168 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:08] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T15:28:08.282 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:08.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:08.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:08.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:08.443 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:08.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:08 smithi017 ceph-mon[215646]: pgmap v1122: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2429/187305 objects degraded (1.297%); 981/187305 objects misplaced (0.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:08.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:08 smithi017 ceph-mon[222878]: pgmap v1122: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2429/187305 objects degraded (1.297%); 981/187305 objects misplaced (0.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:08.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:08 smithi027 ceph-mon[195233]: pgmap v1122: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 861 op/s; 2429/187305 objects degraded (1.297%); 981/187305 objects misplaced (0.524%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T15:28:08.836 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:08.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:08.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:08.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:09.004 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:09.306 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:09.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:09.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:09.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:09.510 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:09.645 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:09.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:09.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:09.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:09.830 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:09.944 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:09.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:09.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:10.053 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:10.105 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:10.115 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:10.172 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:10.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:10.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:10.214 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:28:10.279 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:10.326 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:10.399 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:10.403 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:10 smithi027 ceph-mon[195233]: pgmap v1123: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 2429/185190 objects degraded (1.312%); 981/185190 objects misplaced (0.530%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T15:28:10.403 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.403 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:10 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:10.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:10.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[222878]: pgmap v1123: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 2429/185190 objects degraded (1.312%); 981/185190 objects misplaced (0.530%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T15:28:10.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.442 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[215646]: pgmap v1123: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 632 op/s; 2429/185190 objects degraded (1.312%); 981/185190 objects misplaced (0.530%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T15:28:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:10 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:10.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:10.541 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:10.601 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:10.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:10.704 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:10.756 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:10.827 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:10.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:10.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:10.933 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:10.971 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:11.043 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:11.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:11.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:11.149 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:11.242 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:11.503 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:11.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:28:11.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:28:11.505 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:11.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:28:11.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:28:11.506 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:11.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:11.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:11.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:11.631 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:28:11.631 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:28:11.631 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:11.634 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:11.936 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:11.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:11.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:12.045 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:12.124 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:12.133 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:12.219 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:12.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:12.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:12.331 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:12.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[215646]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:28:12.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1843879897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.354 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/175698 objects degraded (1.382%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:12.377 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:12.422 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:28:12.469 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:12.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:12.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:12.483 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:12 smithi027 ceph-mon[195233]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:28:12.483 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1843879897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.483 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.484 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:12 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/175698 objects degraded (1.382%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:12.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:12.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[222878]: pgmap v1124: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T15:28:12.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1843879897' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:28:12.614 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:12 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/175698 objects degraded (1.382%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:12.651 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:12.746 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:12.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:12.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:12.772 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:28:12.772 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:28:12.853 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:12.932 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:28:13.615 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:13.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:13.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:13.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2761380101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.623 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2761380101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.672 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: osdmap e382: 8 total, 8 up, 8 in 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: failed to encode map e382 with expected crc 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2761380101' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.673 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:13.703 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:28:13.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:13.810 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:14.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:28:14.242 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: pgmap v1126: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.470 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[215646]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: pgmap v1126: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.474 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:14 smithi017 ceph-mon[222878]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.494 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:14.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:14.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:14.603 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: pgmap v1126: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 805 op/s; 2429/175698 objects degraded (1.382%); 676/175698 objects misplaced (0.385%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: osdmap e383: 8 total, 8 up, 8 in 2023-09-23T15:28:14.657 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.658 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.658 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:14 smithi027 ceph-mon[195233]: failed to encode map e383 with expected crc 2023-09-23T15:28:14.661 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:15.322 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:15.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:15.427 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:15.487 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:15.505 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:15.576 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:15.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:15.686 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:15.748 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:15.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:15.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:15.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:15.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:16.020 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:16.045 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:16.076 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:16.114 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:16.154 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:16.188 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:16.232 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:16.265 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:16.304 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.312 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:28:16.316 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:16.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:16.327 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:16.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:16.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:16 smithi027 ceph-mon[195233]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 2429/170010 objects degraded (1.429%); 315/170010 objects misplaced (0.185%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:16.429 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2517699946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:16.437 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:28:16.437 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:28:16.437 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:28:16.438 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:28:16.438 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:28:16.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:28:16.522 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:16.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:16 smithi017 ceph-mon[215646]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 2429/170010 objects degraded (1.429%); 315/170010 objects misplaced (0.185%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:16.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2517699946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:16.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:16 smithi017 ceph-mon[222878]: pgmap v1128: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 2429/170010 objects degraded (1.429%); 315/170010 objects misplaced (0.185%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:16.698 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2517699946' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:16.719 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:16.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:16.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:16.824 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:16.850 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:17.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:28:17.675 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:17.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:17 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/170010 objects degraded (1.429%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:17.689 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:17 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/170010 objects degraded (1.429%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:17.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:17.791 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:17.842 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:17.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:17 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/170010 objects degraded (1.429%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:17.984 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:17.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:17.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:18.015 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:18.098 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:18.120 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:18.151 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:18.243 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:18.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:18.249 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:18.251 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:18] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-09-23T15:28:18.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:18.454 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:18.500 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:18 smithi017 ceph-mon[215646]: pgmap v1129: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 2429/164055 objects degraded (1.481%); 315/164055 objects misplaced (0.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:18.500 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:18 smithi017 ceph-mon[222878]: pgmap v1129: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 2429/164055 objects degraded (1.481%); 315/164055 objects misplaced (0.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:18.585 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:18.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:18.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:18.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:18 smithi027 ceph-mon[195233]: pgmap v1129: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 880 op/s; 2429/164055 objects degraded (1.481%); 315/164055 objects misplaced (0.192%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T15:28:18.690 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:18.736 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:18.831 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:18.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:18.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:18.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:18.988 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:19.101 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:19.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:19.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:19.207 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:19.236 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:19.485 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:19.487 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:28:19.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:19.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:19 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:28:19.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2165386517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.503 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:19.604 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:19.673 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:19.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:28:19.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2165386517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.827 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:28:19.827 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2165386517' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.828 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:28:19.937 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:28:20.128 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:20.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:20.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:20.134 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50596 8.2 8.a11b4e02 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 383.92037026700001, 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91699152100000003, 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:28:20.135 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50596 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211509+0000", 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211508+0000", 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211609+0000", 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101167 2023-09-23T15:28:20.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211623+0000", 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3767e-05 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211630+0000", 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3939999999999998e-06 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211691+0000", 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1008000000000003e-05 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211700+0000", 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3070000000000008e-06 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128474+0000", 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91677326699999995 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128501+0000", 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7019999999999999e-05 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50597 8.2 8.538bc622 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 383.92023727999998, 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91703620299999999, 2023-09-23T15:28:20.140 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50597 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211642+0000", 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211687+0000", 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5200000000000001e-05 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211692+0000", 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.541e-06 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211696+0000", 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.489e-06 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211726+0000", 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9879999999999999e-05 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.211731+0000", 2023-09-23T15:28:20.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9139999999999999e-06 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128668+0000", 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91693728100000005 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.128678+0000", 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0438999999999999e-05 2023-09-23T15:28:20.145 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87479 8.2 8:4a52ba5c:::benchmark_data_smithi017_485823_object87478:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.59666942199999, 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99550063799999999, 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.146 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87479 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535210+0000", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535209+0000", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535289+0000", 2023-09-23T15:28:20.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0311999999999999e-05 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535304+0000", 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4995e-05 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535312+0000", 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3559999999999993e-06 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535462+0000", 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015008400000000001 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.535476+0000", 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3451e-05 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.512985+0000", 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97750865600000003 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.513085+0000", 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010079800000000001 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.513208+0000", 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012306200000000001 2023-09-23T15:28:20.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520104+0000", 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068957640000000004 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520176+0000", 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1971999999999995e-05 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530643+0000", 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010466824 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530681+0000", 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8059000000000003e-05 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530711+0000", 2023-09-23T15:28:20.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9646999999999999e-05 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87489 8.2 8:48f7c14f:::benchmark_data_smithi017_485823_object87488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.57334526, 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97232053399999996, 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.154 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87489 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558534+0000", 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558532+0000", 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558643+0000", 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011113200000000001 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558657+0000", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3875e-05 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558665+0000", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4529999999999996e-06 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558745+0000", 2023-09-23T15:28:20.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9914999999999996e-05 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.558758+0000", 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3682e-05 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515668+0000", 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.956909644 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515798+0000", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000130142 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.515906+0000", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108136 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520204+0000", 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042975110000000004 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520266+0000", 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2539000000000006e-05 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530807+0000", 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010540546 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530834+0000", 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7095000000000001e-05 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530855+0000", 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0899000000000001e-05 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87525 8.2 8:406bbda9:::benchmark_data_smithi017_485823_object87524:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.161 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.50066427399997, 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89979054000000003, 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87525 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631215+0000", 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631213+0000", 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631296+0000", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3195999999999994e-05 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631314+0000", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7529000000000001e-05 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631322+0000", 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.011e-06 2023-09-23T15:28:20.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631390+0000", 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7599000000000001e-05 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.631404+0000", 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4299e-05 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518240+0000", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88683647899999996 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518345+0000", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010480399999999999 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.518447+0000", 2023-09-23T15:28:20.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101828 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525282+0000", 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0068350889999999999 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530609+0000", 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0053267449999999999 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530947+0000", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00033794000000000001 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530978+0000", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1445999999999997e-05 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531006+0000", 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7543999999999999e-05 2023-09-23T15:28:20.168 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87536 8.2 8:4ea85f1f:::benchmark_data_smithi017_485823_object87535:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.49038670700003, 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.889614613, 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87536 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641493+0000", 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641491+0000", 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641551+0000", 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0201000000000002e-05 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641570+0000", 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8638e-05 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641577+0000", 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.114e-06 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641632+0000", 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4963999999999998e-05 2023-09-23T15:28:20.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.641642+0000", 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0412e-05 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520296+0000", 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87865352100000005 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520391+0000", 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4883999999999995e-05 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.520486+0000", 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5741000000000004e-05 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.527976+0000", 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074899900000000002 2023-09-23T15:28:20.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530771+0000", 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002794487 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531039+0000", 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000268165 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531071+0000", 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1652999999999997e-05 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531107+0000", 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6788000000000001e-05 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87554 8.2 8:45ba82f7:::benchmark_data_smithi017_485823_object87553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.176 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.44793579999998, 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.84732607400000004, 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87554 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683944+0000", 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683930+0000", 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999866 2023-09-23T15:28:20.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.683994+0000", 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3552000000000004e-05 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684021+0000", 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6920999999999999e-05 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684030+0000", 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.431e-06 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684096+0000", 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5705999999999997e-05 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.684105+0000", 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2650000000000002e-06 2023-09-23T15:28:20.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522506+0000", 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83840049699999997 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522650+0000", 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014449800000000001 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.522832+0000", 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018217 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530617+0000", 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077846809999999999 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530890+0000", 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027289300000000002 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531180+0000", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00028948599999999999 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531231+0000", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1178999999999999e-05 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531270+0000", 2023-09-23T15:28:20.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9011000000000003e-05 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87579 8.2 8:4fb1f62c:::benchmark_data_smithi017_485823_object87578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.40946626700003, 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81508181099999999, 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.184 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87579 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722413+0000", 2023-09-23T15:28:20.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722411+0000", 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722470+0000", 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8721999999999998e-05 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722488+0000", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8365000000000001e-05 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722497+0000", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2730000000000002e-06 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722573+0000", 2023-09-23T15:28:20.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6279000000000006e-05 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.722581+0000", 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8970000000000008e-06 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525313+0000", 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80273269199999997 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525443+0000", 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129498 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.525551+0000", 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000108167 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530778+0000", 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052264729999999997 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531338+0000", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00056077399999999999 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537387+0000", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006049034 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537448+0000", 2023-09-23T15:28:20.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0513000000000002e-05 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537495+0000", 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7086000000000003e-05 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87580 8.2 8:43e296c2:::benchmark_data_smithi017_485823_object87579:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.40526394199998, 2023-09-23T15:28:20.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.81100082500000004, 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87580 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726615+0000", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726692+0000", 2023-09-23T15:28:20.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7746999999999999e-05 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726706+0000", 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3783e-05 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726711+0000", 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8690000000000003e-06 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726755+0000", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4307000000000001e-05 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.726765+0000", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4509999999999998e-06 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528023+0000", 2023-09-23T15:28:20.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80125852099999995 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528145+0000", 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121804 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.528294+0000", 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149209 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.530913+0000", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026183740000000001 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531392+0000", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000479421 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537542+0000", 2023-09-23T15:28:20.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061496830000000004 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537587+0000", 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5498999999999999e-05 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537616+0000", 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9082e-05 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.198 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87589 8.2 8:42faca38:::benchmark_data_smithi017_485823_object87588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.39225042999999, 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80387859699999997, 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.199 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87589 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739629+0000", 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739627+0000", 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739727+0000", 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9987000000000003e-05 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739743+0000", 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6314999999999999e-05 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739750+0000", 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4060000000000002e-06 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739825+0000", 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4869000000000004e-05 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.739839+0000", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4426999999999999e-05 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531435+0000", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79159639699999995 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531561+0000", 2023-09-23T15:28:20.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012580999999999999 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.531709+0000", 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014819400000000001 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537509+0000", 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.005799762 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537642+0000", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013332700000000001 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543446+0000", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058038229999999996 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543479+0000", 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2761e-05 2023-09-23T15:28:20.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543508+0000", 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8581999999999998e-05 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87592 8.2 8:47cf9f2f:::benchmark_data_smithi017_485823_object87591:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.38824624300003, 2023-09-23T15:28:20.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.82188067499999995, 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87592 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743633+0000", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743627+0000", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999933 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743699+0000", 2023-09-23T15:28:20.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2266000000000002e-05 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743718+0000", 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9366000000000001e-05 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743725+0000", 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.17e-06 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743784+0000", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8798000000000002e-05 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.743796+0000", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.131e-05 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.534749+0000", 2023-09-23T15:28:20.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79095367599999999 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.534881+0000", 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131784 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.535016+0000", 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134943 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537647+0000", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026313199999999999 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543417+0000", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057694920000000002 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565443+0000", 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022026104000000001 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565484+0000", 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0667999999999998e-05 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565514+0000", 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0358e-05 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87613 8.2 8:43730902:::benchmark_data_smithi017_485823_object87612:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.34228675899999, 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77606009899999995, 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87613 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.214 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789593+0000", 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789591+0000", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789673+0000", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2378000000000003e-05 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789693+0000", 2023-09-23T15:28:20.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9440999999999999e-05 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789701+0000", 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8909999999999995e-06 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789760+0000", 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9577999999999998e-05 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789775+0000", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4209999999999999e-05 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537670+0000", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74789507200000005 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537762+0000", 2023-09-23T15:28:20.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2693000000000002e-05 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.537896+0000", 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013394599999999999 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543526+0000", 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056293139999999998 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552772+0000", 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092463460000000008 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565600+0000", 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012828482 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565631+0000", 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0514999999999998e-05 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565653+0000", 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1815e-05 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87614 8.2 8:4007368f:::benchmark_data_smithi017_485823_object87613:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.221 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.34216401399999, 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77602658099999999, 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87614 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789715+0000", 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:20.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789774+0000", 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9268999999999998e-05 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789783+0000", 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6319999999999997e-06 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789787+0000", 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2769999999999999e-06 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789842+0000", 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5387999999999999e-05 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.789856+0000", 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3838e-05 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540554+0000", 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.75069737299999995 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540682+0000", 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128467 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.540827+0000", 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014544800000000001 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552719+0000", 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011891626000000001 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552805+0000", 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5820000000000002e-05 2023-09-23T15:28:20.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565690+0000", 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012885122000000001 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565719+0000", 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9071e-05 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565742+0000", 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2929e-05 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87629 8.2 8:4962ff39:::benchmark_data_smithi017_485823_object87628:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.30718405800002, 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74115534500000002, 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.229 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87629 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824695+0000", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824694+0000", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824745+0000", 2023-09-23T15:28:20.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1140999999999997e-05 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824760+0000", 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5500999999999999e-05 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824767+0000", 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1519999999999997e-06 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824832+0000", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5376999999999994e-05 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.824845+0000", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2704999999999999e-05 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543560+0000", 2023-09-23T15:28:20.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.71871572399999994 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543668+0000", 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107358 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.543760+0000", 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1947999999999994e-05 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552725+0000", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089653790000000008 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565394+0000", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012669249 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565785+0000", 2023-09-23T15:28:20.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000390541 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565813+0000", 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7770999999999999e-05 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565851+0000", 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8061e-05 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87637 8.2 8:403d4373:::benchmark_data_smithi017_485823_object87636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.29060902200001, 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.724673546, 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87637 2023-09-23T15:28:20.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841270+0000", 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841268+0000", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841309+0000", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0831000000000003e-05 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841327+0000", 2023-09-23T15:28:20.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7875000000000001e-05 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841334+0000", 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4340000000000003e-06 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841387+0000", 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2632e-05 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.841397+0000", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8209999999999997e-06 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546335+0000", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70493799099999999 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546500+0000", 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000165198 2023-09-23T15:28:20.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.546680+0000", 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017972499999999999 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565404+0000", 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018724034000000001 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565569+0000", 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016548299999999999 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565892+0000", 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032258200000000002 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565923+0000", 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0905999999999999e-05 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565944+0000", 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1389000000000001e-05 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87654 8.2 8:4971efb7:::benchmark_data_smithi017_485823_object87653:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.23869345899999, 2023-09-23T15:28:20.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.68569409699999995, 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87654 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893186+0000", 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893184+0000", 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893221+0000", 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7132000000000003e-05 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893243+0000", 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1987000000000001e-05 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893250+0000", 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5619999999999998e-06 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893307+0000", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7126999999999997e-05 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.893318+0000", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0847e-05 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549540+0000", 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.65622212700000004 2023-09-23T15:28:20.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549659+0000", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011890199999999999 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.549804+0000", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014447 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565410+0000", 2023-09-23T15:28:20.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.015606594 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.569942+0000", 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045315420000000004 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578781+0000", 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0088392560000000002 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578836+0000", 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4809000000000003e-05 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578880+0000", 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4035000000000001e-05 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87661 8.2 8:4515bd6f:::benchmark_data_smithi017_485823_object87660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.251 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 333.21437827900002, 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.66148114999999996, 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87661 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917501+0000", 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917500+0000", 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917552+0000", 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1702000000000001e-05 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917568+0000", 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6622000000000001e-05 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917574+0000", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9739999999999999e-06 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917641+0000", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6390000000000006e-05 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:46.917652+0000", 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1897e-05 2023-09-23T15:28:20.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552825+0000", 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.63517245499999997 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.552984+0000", 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015915600000000001 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.553150+0000", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000166009 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565417+0000", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012266716 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.569974+0000", 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045568479999999996 2023-09-23T15:28:20.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578915+0000", 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089418580000000004 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578952+0000", 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6470000000000001e-05 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.578982+0000", 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0361999999999998e-05 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:87704 8.2 8:40e179a2:::benchmark_data_smithi017_485823_object87703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e310)", 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 332.97158757400001, 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.41878299099999999, 2023-09-23T15:28:20.259 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 87704 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160292+0000", 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160289+0000", 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160399+0000", 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011008100000000001 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160438+0000", 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160452+0000", 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3516000000000001e-05 2023-09-23T15:28:20.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160572+0000", 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012081700000000001 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.160590+0000", 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8016000000000002e-05 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556242+0000", 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39565125400000001 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556355+0000", 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011383399999999999 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.556489+0000", 2023-09-23T15:28:20.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013384700000000001 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.565754+0000", 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092646399999999993 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.570003+0000", 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042494630000000002 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579011+0000", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090077760000000003 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579045+0000", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4317999999999999e-05 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:47.579075+0000", 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9383e-05 2023-09-23T15:28:20.266 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88043 9.e 9:7d9f7ec3:::benchmark_data_smithi017_543597_object4252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.736013122000003, 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0278136939999998, 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88043 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395865+0000", 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395870+0000", 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1939999999999997e-06 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395882+0000", 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2089999999999999e-05 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395890+0000", 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0800000000000006e-06 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400157+0000", 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042668879999999999 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400169+0000", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1718e-05 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397541+0000", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99737234299999999 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397554+0000", 2023-09-23T15:28:20.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.274e-05 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.409726+0000", 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.012172013 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.409887+0000", 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016131399999999999 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.410042+0000", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015405900000000001 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.415246+0000", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052046540000000004 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418242+0000", 2023-09-23T15:28:20.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029955680000000001 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423508+0000", 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052663409999999999 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423543+0000", 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.464e-05 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423680+0000", 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137267 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88049 9.e 9:733405ce:::benchmark_data_smithi017_543597_object4258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.710397309000001, 2023-09-23T15:28:20.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0045682330000001, 2023-09-23T15:28:20.276 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:20.276 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88049 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:28:20.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421481+0000", 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421487+0000", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5189999999999998e-06 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421501+0000", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3557000000000001e-05 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421507+0000", 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9089999999999997e-06 2023-09-23T15:28:20.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421556+0000", 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9153000000000002e-05 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421570+0000", 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3984e-05 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397577+0000", 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97600752700000004 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397585+0000", 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5680000000000002e-06 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.412942+0000", 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.015356929 2023-09-23T15:28:20.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.413039+0000", 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6915000000000003e-05 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.413128+0000", 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8647999999999996e-05 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418274+0000", 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051468340000000003 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418285+0000", 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0795000000000001e-05 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.425895+0000", 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076100789999999996 2023-09-23T15:28:20.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.425933+0000", 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7435000000000003e-05 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.426050+0000", 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011771499999999999 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.284 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:20.285 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:20.285 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:28:20.285 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:20.328 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:20.465 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:20.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:20.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: pgmap v1130: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 2429/161871 objects degraded (1.501%); 315/161871 objects misplaced (0.195%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T15:28:20.470 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.471 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:20 smithi027 ceph-mon[195233]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.570 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:20.612 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:20.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:28:20.681 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:20.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:20.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:20.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: pgmap v1130: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 2429/161871 objects degraded (1.501%); 315/161871 objects misplaced (0.195%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:28:20.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:28:20.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.773 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[215646]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: pgmap v1130: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 575 op/s; 2429/161871 objects degraded (1.501%); 315/161871 objects misplaced (0.195%); 2.8 MiB/s, 45 objects/s recovering 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:28:20.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: osdmap e384: 8 total, 8 up, 8 in 2023-09-23T15:28:20.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.775 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:20 smithi017 ceph-mon[222878]: failed to encode map e384 with expected crc 2023-09-23T15:28:20.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:20.828 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:21.030 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:28:21.040 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:21.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:21.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:21.096 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:21.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:21.207 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:21.316 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:21.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:21.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:21.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:21.507 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:21.612 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:21.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:21.684 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:21.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:21.731 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:21.756 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:21.832 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:21.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:21.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:21.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:21.966 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:22.015 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:22.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:22.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:22.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:22.148 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:22.281 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:28:22.300 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:22.304 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:22.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:22.441 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: pgmap v1132: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 2429/152208 objects degraded (1.596%); 5.3 MiB/s, 105 objects/s recovering 2023-09-23T15:28:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: osdmap e385: 8 total, 8 up, 8 in 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/320839998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 2429/152208 objects degraded (1.596%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: osdmap e386: 8 total, 8 up, 8 in 2023-09-23T15:28:22.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:22 smithi027 ceph-mon[195233]: failed to encode map e386 with expected crc 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: pgmap v1132: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 2429/152208 objects degraded (1.596%); 5.3 MiB/s, 105 objects/s recovering 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: osdmap e385: 8 total, 8 up, 8 in 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/320839998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 2429/152208 objects degraded (1.596%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: osdmap e386: 8 total, 8 up, 8 in 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[222878]: failed to encode map e386 with expected crc 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: pgmap v1132: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 978 op/s; 2429/152208 objects degraded (1.596%); 5.3 MiB/s, 105 objects/s recovering 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: osdmap e385: 8 total, 8 up, 8 in 2023-09-23T15:28:22.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: failed to encode map e385 with expected crc 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/320839998' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 2429/152208 objects degraded (1.596%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: osdmap e386: 8 total, 8 up, 8 in 2023-09-23T15:28:22.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:22 smithi017 ceph-mon[215646]: failed to encode map e386 with expected crc 2023-09-23T15:28:22.975 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:22.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:22.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:23.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:23.106 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:23.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:23.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:23.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:23.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:23.292 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:23.466 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:23.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118736 9.a 9:5637ca98:::benchmark_data_smithi017_543597_object34945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.545078+0000", 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0041051450000000001, 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041344770000000001, 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:23.553 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118736 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545078+0000", 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545078+0000", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545076+0000", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545084+0000", 2023-09-23T15:28:23.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8140000000000001e-06 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545097+0000", 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2799999999999999e-05 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545104+0000", 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8120000000000002e-06 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545167+0000", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3309e-05 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545280+0000", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000113056 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.545392+0000", 2023-09-23T15:28:23.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011199199999999999 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118741 9.a 9:57530ff6:::benchmark_data_smithi017_543597_object34950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.548696+0000", 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.000487357, 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037397179999999999, 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:28:23.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118741 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548696+0000", 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:23.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548696+0000", 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548694+0000", 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548702+0000", 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1780000000000003e-06 2023-09-23T15:28:23.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548722+0000", 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.95e-05 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548736+0000", 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4149e-05 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:23.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548906+0000", 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000169634 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548995+0000", 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9483999999999999e-05 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549070+0000", 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4714000000000003e-05 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.562 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:118736 9.a e387/386)", 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.548811+0000", 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.00037201200000000002, 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0036922769999999999, 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548811+0000", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548811+0000", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548810+0000", 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:23.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548816+0000", 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8259999999999998e-06 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548827+0000", 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0550999999999999e-05 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:23.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.548833+0000", 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8189999999999997e-06 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46157.0:118740 9.b e387/339 9:d8e5ae46:::benchmark_data_smithi017_543597_object34949:head v 387'7530, mlcod=387'7528)", 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.549102+0000", 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 8.0755999999999997e-05, 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034330179999999999, 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:23.566 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549102+0000", 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549102+0000", 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.567 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549102+0000", 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549108+0000", 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3330000000000002e-06 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:23.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549137+0000", 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8552e-05 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549143+0000", 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1070000000000001e-06 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549217+0000", 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3917000000000004e-05 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.549236+0000", 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8921000000000001e-05 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552506+0000", 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003270565 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46157.0:118736 9.a e387/386)", 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.552249+0000", 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9969344, 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00032672700000000002, 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552249+0000", 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552249+0000", 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552248+0000", 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:28:23.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552253+0000", 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4949999999999999e-06 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552266+0000", 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3125000000000001e-05 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.552272+0000", 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9120000000000003e-06 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.573 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:28:23.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.574 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:28:23.574 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:28:23.574 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 5 2023-09-23T15:28:23.574 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:28:23.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: osdmap e387: 8 total, 8 up, 8 in 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.582 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:23 smithi027 ceph-mon[195233]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.614 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: osdmap e387: 8 total, 8 up, 8 in 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[215646]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: failed to encode map e386 with expected crc 2023-09-23T15:28:23.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: osdmap e387: 8 total, 8 up, 8 in 2023-09-23T15:28:23.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:23 smithi017 ceph-mon[222878]: failed to encode map e387 with expected crc 2023-09-23T15:28:23.774 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:23.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:23.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:23.878 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:23.908 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:24.010 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:24.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:24.013 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:24.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:24.143 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:24.239 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:24.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:24.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:24.343 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:24.382 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:24.412 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:28:24.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[215646]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2429/152208 objects degraded (1.596%); 3.3 MiB/s, 80 objects/s recovering 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[215646]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[215646]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[215646]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[215646]: osdmap e388: 8 total, 8 up, 8 in 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[222878]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2429/152208 objects degraded (1.596%); 3.3 MiB/s, 80 objects/s recovering 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[222878]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[222878]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[222878]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:24 smithi017 ceph-mon[222878]: osdmap e388: 8 total, 8 up, 8 in 2023-09-23T15:28:24.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:24.739 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:24.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:24.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:24.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:24.740 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:24.761 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:24.786 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:24.787 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.788 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:28:24.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:24 smithi027 ceph-mon[195233]: pgmap v1135: 119 pgs: 1 active+recovering+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 657 op/s; 2429/152208 objects degraded (1.596%); 3.3 MiB/s, 80 objects/s recovering 2023-09-23T15:28:24.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:24 smithi027 ceph-mon[195233]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:24 smithi027 ceph-mon[195233]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:24 smithi027 ceph-mon[195233]: failed to encode map e387 with expected crc 2023-09-23T15:28:24.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:24 smithi027 ceph-mon[195233]: osdmap e388: 8 total, 8 up, 8 in 2023-09-23T15:28:24.898 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:25.101 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:25.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:25.104 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:25.205 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:25.236 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:25.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:28:25.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1145875142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[222878]: osdmap e389: 8 total, 8 up, 8 in 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:28:25.650 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1145875142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:25.651 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:25 smithi017 ceph-mon[215646]: osdmap e389: 8 total, 8 up, 8 in 2023-09-23T15:28:25.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: failed to encode map e388 with expected crc 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1145875142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:25.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:25 smithi027 ceph-mon[195233]: osdmap e389: 8 total, 8 up, 8 in 2023-09-23T15:28:26.037 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:26.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:26.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:26.141 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:26.162 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:26.173 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:26.285 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:28:26.461 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:26.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:26.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:26.567 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:26.609 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:26.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: pgmap v1138: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 1529/148194 objects degraded (1.032%); 1.7 MiB/s, 219 objects/s recovering 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3997790087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: osdmap e390: 8 total, 8 up, 8 in 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[215646]: failed to encode map e390 with expected crc 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: pgmap v1138: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 1529/148194 objects degraded (1.032%); 1.7 MiB/s, 219 objects/s recovering 2023-09-23T15:28:26.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3997790087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: osdmap e390: 8 total, 8 up, 8 in 2023-09-23T15:28:26.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:26 smithi017 ceph-mon[222878]: failed to encode map e390 with expected crc 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: pgmap v1138: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 334 op/s; 1529/148194 objects degraded (1.032%); 1.7 MiB/s, 219 objects/s recovering 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: failed to encode map e389 with expected crc 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3997790087' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: osdmap e390: 8 total, 8 up, 8 in 2023-09-23T15:28:26.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:26 smithi027 ceph-mon[195233]: failed to encode map e390 with expected crc 2023-09-23T15:28:26.730 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:26.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:26.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:26.836 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:26.868 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:26.928 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:26.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:26.931 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:26.965 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:28:26.965 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:28:27.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:27.063 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:27.262 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:27.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:27.365 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:27.393 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:27.448 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:27.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:27.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:27.552 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:27.592 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:27.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:28:27.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 1529/148194 objects degraded (1.032%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:27.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: osdmap e391: 8 total, 8 up, 8 in 2023-09-23T15:28:27.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[215646]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 1529/148194 objects degraded (1.032%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:27.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: osdmap e391: 8 total, 8 up, 8 in 2023-09-23T15:28:27.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:27 smithi017 ceph-mon[222878]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.748 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: failed to encode map e390 with expected crc 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 1529/148194 objects degraded (1.032%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: osdmap e391: 8 total, 8 up, 8 in 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:27 smithi027 ceph-mon[195233]: failed to encode map e391 with expected crc 2023-09-23T15:28:27.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:28:27.769 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:27.771 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:28:27.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:27.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:27.882 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:27.945 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:28.044 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:28.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:28.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:28.152 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:28.207 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:28.283 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:28.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:28.392 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:28.424 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:28.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:28] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-09-23T15:28:28.453 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:28.555 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:28.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:28.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:28.659 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:28.695 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:28.712 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: pgmap v1141: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1238/145698 objects degraded (0.850%); 2.4 MiB/s, 292 objects/s recovering 2023-09-23T15:28:28.712 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.712 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: 1.7 scrub starts 2023-09-23T15:28:28.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:28.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:28.759 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:28.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:28.780 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:28.800 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:28.864 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:28.899 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: pgmap v1141: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1238/145698 objects degraded (0.850%); 2.4 MiB/s, 292 objects/s recovering 2023-09-23T15:28:28.899 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: 1.7 scrub starts 2023-09-23T15:28:28.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:28.900 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:28.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: pgmap v1141: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait, 4 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 542 op/s; 1238/145698 objects degraded (0.850%); 2.4 MiB/s, 292 objects/s recovering 2023-09-23T15:28:28.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: 1.7 scrub starts 2023-09-23T15:28:28.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: failed to encode map e391 with expected crc 2023-09-23T15:28:28.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:28.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:28.903 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:29.265 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:29.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:29.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:29.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:29.398 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:29.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[215646]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[215646]: osdmap e392: 8 total, 8 up, 8 in 2023-09-23T15:28:29.525 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[215646]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:28:29.532 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:28:29.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:29 smithi027 ceph-mon[195233]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:29 smithi027 ceph-mon[195233]: osdmap e392: 8 total, 8 up, 8 in 2023-09-23T15:28:29.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:29 smithi027 ceph-mon[195233]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:28:29.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[222878]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[222878]: osdmap e392: 8 total, 8 up, 8 in 2023-09-23T15:28:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[222878]: failed to encode map e392 with expected crc 2023-09-23T15:28:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:28:30.258 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:28:30.258 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:28:30.272 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:30.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:30.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:30.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:30.410 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:30.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: pgmap v1144: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 902/144573 objects degraded (0.624%); 1.3 MiB/s, 156 objects/s recovering 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: osdmap e393: 8 total, 8 up, 8 in 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:30 smithi027 ceph-mon[195233]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: pgmap v1144: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 902/144573 objects degraded (0.624%); 1.3 MiB/s, 156 objects/s recovering 2023-09-23T15:28:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:28:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: osdmap e393: 8 total, 8 up, 8 in 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[215646]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: pgmap v1144: 119 pgs: 1 active+recovering+undersized+remapped, 2 peering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 902/144573 objects degraded (0.624%); 1.3 MiB/s, 156 objects/s recovering 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:30.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1554852927' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:28:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: osdmap e393: 8 total, 8 up, 8 in 2023-09-23T15:28:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:30 smithi017 ceph-mon[222878]: failed to encode map e393 with expected crc 2023-09-23T15:28:30.993 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:28:31.012 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:31.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:31.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:31.127 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:31.251 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:31.505 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:31.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:31.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:31.612 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:31.660 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:31.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[215646]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[215646]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[215646]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[215646]: osdmap e394: 8 total, 8 up, 8 in 2023-09-23T15:28:31.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[215646]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[222878]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[222878]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[222878]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[222878]: osdmap e394: 8 total, 8 up, 8 in 2023-09-23T15:28:31.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:31 smithi017 ceph-mon[222878]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.727 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:31.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:31.732 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:31.833 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:31 smithi027 ceph-mon[195233]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:31 smithi027 ceph-mon[195233]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:31 smithi027 ceph-mon[195233]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:31 smithi027 ceph-mon[195233]: osdmap e394: 8 total, 8 up, 8 in 2023-09-23T15:28:31.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:31 smithi027 ceph-mon[195233]: failed to encode map e394 with expected crc 2023-09-23T15:28:31.861 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:32.211 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:32.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:32.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:32.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:32.346 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:32.376 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:32.397 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:32.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:32.480 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:32.500 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:32.502 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:32.521 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:32.536 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:32.544 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:32.568 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:32.595 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:32.605 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:32.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:32.628 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:32.670 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:32.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.693 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.694 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:28:32.714 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:32.771 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: pgmap v1147: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 608/140382 objects degraded (0.433%); 1.1 MiB/s, 159 objects/s recovering 2023-09-23T15:28:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: osdmap e395: 8 total, 8 up, 8 in 2023-09-23T15:28:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: Health check update: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:32.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2899707142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:32.857 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: pgmap v1147: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 608/140382 objects degraded (0.433%); 1.1 MiB/s, 159 objects/s recovering 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: osdmap e395: 8 total, 8 up, 8 in 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: Health check update: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2899707142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: pgmap v1147: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 442 op/s; 608/140382 objects degraded (0.433%); 1.1 MiB/s, 159 objects/s recovering 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: osdmap e395: 8 total, 8 up, 8 in 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: failed to encode map e395 with expected crc 2023-09-23T15:28:32.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: Health check update: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T15:28:32.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2899707142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:32.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:32.869 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:32.970 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:33.018 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:33.105 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:33.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:33.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:33.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:33.261 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:33.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: osdmap e396: 8 total, 8 up, 8 in 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/129740594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:33.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:33.625 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:28:33.789 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:33.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:33.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: osdmap e396: 8 total, 8 up, 8 in 2023-09-23T15:28:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: failed to encode map e396 with expected crc 2023-09-23T15:28:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/129740594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:33.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:33 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:33.894 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:33.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: failed to encode map e396 with expected crc 2023-09-23T15:28:34.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: osdmap e396: 8 total, 8 up, 8 in 2023-09-23T15:28:34.006 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: failed to encode map e396 with expected crc 2023-09-23T15:28:34.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: failed to encode map e396 with expected crc 2023-09-23T15:28:34.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: failed to encode map e396 with expected crc 2023-09-23T15:28:34.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/129740594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:34.007 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:33 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:28:34.007 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:34.423 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:34.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:34.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:34.529 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:34.562 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:34.606 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:34.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:34.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:34.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: pgmap v1150: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 608/140382 objects degraded (0.433%); 576 KiB/s, 75 objects/s recovering 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: osdmap e397: 8 total, 8 up, 8 in 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[215646]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: pgmap v1150: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 608/140382 objects degraded (0.433%); 576 KiB/s, 75 objects/s recovering 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: osdmap e397: 8 total, 8 up, 8 in 2023-09-23T15:28:34.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:34 smithi017 ceph-mon[222878]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.712 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:34.742 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:34.840 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: pgmap v1150: 119 pgs: 1 peering, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 348 op/s; 608/140382 objects degraded (0.433%); 576 KiB/s, 75 objects/s recovering 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: osdmap e397: 8 total, 8 up, 8 in 2023-09-23T15:28:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:34 smithi027 ceph-mon[195233]: failed to encode map e397 with expected crc 2023-09-23T15:28:34.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:34.859 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:34.960 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:35.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:35.104 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:35.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:35.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:35.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:28:35.217 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:35.260 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:35.339 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:35.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:35.347 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:35.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:35.458 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:35.490 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:35.575 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:35.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:35.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:35.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:35.749 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T15:28:35.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T15:28:35.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T15:28:35.973 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:35.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:35.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:36.009 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:124758 9.4 9:2b85ed10:::benchmark_data_smithi017_543597_object40967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:35.985102+0000", 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.033299241, 2023-09-23T15:28:36.019 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.033338843, 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 124758 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985102+0000", 2023-09-23T15:28:36.020 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985102+0000", 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985101+0000", 2023-09-23T15:28:36.021 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985105+0000", 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6059999999999996e-06 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985115+0000", 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2599999999999994e-06 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.022 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:35.985121+0000", 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9769999999999997e-06 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.001422+0000", 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.016301751999999999 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.023 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.001574+0000", 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015205000000000001 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.001699+0000", 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012481100000000001 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.024 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.004992+0000", 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0032925179999999999 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005257+0000", 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000265438 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.025 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:124778 9.4 9:264870e7:::benchmark_data_smithi017_543597_object40987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.005723+0000", 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.012677969000000001, 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012840129, 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 124778 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.026 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005723+0000", 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005723+0000", 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.027 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005722+0000", 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005729+0000", 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4309999999999999e-06 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005744+0000", 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.505e-05 2023-09-23T15:28:36.028 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005750+0000", 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.64e-06 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005787+0000", 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6227999999999999e-05 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.029 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005892+0000", 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000105052 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.005961+0000", 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9510000000000004e-05 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010902+0000", 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049404169999999999 2023-09-23T15:28:36.030 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.017227+0000", 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0063255179999999996 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:124779 9.4 9:2d392a90:::benchmark_data_smithi017_543597_object40988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e398)", 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.010719+0000", 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0076828640000000002, 2023-09-23T15:28:36.031 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079497869999999998, 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 124779 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010719+0000", 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.032 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010719+0000", 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010718+0000", 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.033 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010725+0000", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3849999999999996e-06 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010743+0000", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8028000000000001e-05 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.010752+0000", 2023-09-23T15:28:36.034 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4630000000000007e-06 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013468+0000", 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0027159189999999998 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013588+0000", 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120031 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.035 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013694+0000", 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010526000000000001 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.017443+0000", 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037493660000000001 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.036 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:124781 9.d e398/387 9:b399b1cd:::benchmark_data_smithi017_543597_object40990:head v 398'7828, mlcod=398'7826)", 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.013149+0000", 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0052528799999999997, 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055978690000000001, 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013149+0000", 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.037 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013149+0000", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013147+0000", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013159+0000", 2023-09-23T15:28:36.038 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1137e-05 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013177+0000", 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8791e-05 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013186+0000", 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3210000000000005e-06 2023-09-23T15:28:36.039 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013258+0000", 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2595999999999994e-05 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.013284+0000", 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6224000000000002e-05 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.040 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:124782 9.d e398/387 9:b2223ef2:::benchmark_data_smithi017_543597_object40991:head v 398'7829, mlcod=398'7826)", 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.016165+0000", 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.0022369349999999998, 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026203839999999999, 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.041 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016165+0000", 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016165+0000", 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016164+0000", 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:36.042 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016170+0000", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2469999999999997e-06 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016180+0000", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9329999999999996e-06 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016185+0000", 2023-09-23T15:28:36.043 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.378e-06 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016225+0000", 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.9657000000000003e-05 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016248+0000", 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3258999999999999e-05 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.044 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:124783 9.2 e398/363 9:48c6998c:::benchmark_data_smithi017_543597_object40992:head v 398'7779, mlcod=398'7777)", 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.016957+0000", 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 0.001444858, 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018664039999999999, 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "started", 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.045 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016957+0000", 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016957+0000", 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016956+0000", 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:36.046 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016962+0000", 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4679999999999997e-06 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016979+0000", 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7159e-05 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.047 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.016987+0000", 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8520000000000004e-06 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.017077+0000", 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0092000000000004e-05 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.017100+0000", 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2184999999999999e-05 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.048 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop_reply(client.46157.0:124779 9.4 e398/376)", 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:28:36.018808+0000", 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 4294967295.9995937, 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8485000000000005e-05, 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.049 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018808+0000", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018808+0000", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018806+0000", 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:36.050 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018817+0000", 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0740000000000001e-05 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018829+0000", 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1764e-05 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.051 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018837+0000", 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.841e-06 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:28:36.018881+0000", 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4353e-05 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: ], 2023-09-23T15:28:36.052 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 7 2023-09-23T15:28:36.053 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:28:36.079 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:36.173 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:36.395 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:36.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:36.504 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:36.566 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:36.668 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:36.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:36.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[222878]: pgmap v1152: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded) 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[222878]: failed to encode map e398 with expected crc 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[222878]: osdmap e398: 8 total, 8 up, 8 in 2023-09-23T15:28:36.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[215646]: pgmap v1152: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:36.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded) 2023-09-23T15:28:36.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[215646]: failed to encode map e398 with expected crc 2023-09-23T15:28:36.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T15:28:36.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:36 smithi017 ceph-mon[215646]: osdmap e398: 8 total, 8 up, 8 in 2023-09-23T15:28:36.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:36.812 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:36.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:36 smithi027 ceph-mon[195233]: pgmap v1152: 119 pgs: 119 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 451 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:36.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:36 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 608/140382 objects degraded (0.433%), 2 pgs degraded) 2023-09-23T15:28:36.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:36 smithi027 ceph-mon[195233]: failed to encode map e398 with expected crc 2023-09-23T15:28:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:36 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T15:28:36.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:36 smithi027 ceph-mon[195233]: osdmap e398: 8 total, 8 up, 8 in 2023-09-23T15:28:37.560 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:37.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:37.669 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:37.680 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:37.698 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:37.730 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:28:37.754 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:37.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:37.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: osdmap e399: 8 total, 8 up, 8 in 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.823 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[215646]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: osdmap e399: 8 total, 8 up, 8 in 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.834 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:37 smithi017 ceph-mon[222878]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.861 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: osdmap e399: 8 total, 8 up, 8 in 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.904 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:37 smithi027 ceph-mon[195233]: failed to encode map e399 with expected crc 2023-09-23T15:28:37.909 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:37.914 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:37.972 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:37.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:37.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:38.081 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:38.109 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:38] "GET /metrics HTTP/1.1" 200 36387 "" "Prometheus/2.43.0" 2023-09-23T15:28:38.169 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:38.417 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:38.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:38.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:38.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:38.597 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: pgmap v1155: 119 pgs: 1 unknown, 118 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: osdmap e400: 8 total, 8 up, 8 in 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[215646]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: pgmap v1155: 119 pgs: 1 unknown, 118 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: osdmap e400: 8 total, 8 up, 8 in 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:38.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:38 smithi017 ceph-mon[222878]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.032 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:39.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: pgmap v1155: 119 pgs: 1 unknown, 118 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 652 op/s; 1.3 MiB/s, 140 objects/s recovering 2023-09-23T15:28:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: osdmap e400: 8 total, 8 up, 8 in 2023-09-23T15:28:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:38 smithi027 ceph-mon[195233]: failed to encode map e400 with expected crc 2023-09-23T15:28:39.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:39.164 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:39.237 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:39.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:39.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:39.345 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:39.399 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:39.635 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: osdmap e401: 8 total, 8 up, 8 in 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2788203447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:39.636 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:39.665 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:28:39.880 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:39.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: osdmap e401: 8 total, 8 up, 8 in 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: failed to encode map e401 with expected crc 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2788203447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:39.895 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:39.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:39.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: failed to encode map e401 with expected crc 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: failed to encode map e401 with expected crc 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: osdmap e401: 8 total, 8 up, 8 in 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: failed to encode map e401 with expected crc 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: failed to encode map e401 with expected crc 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: failed to encode map e401 with expected crc 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2788203447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:40.023 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:40.026 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:40.151 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:40.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:40.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:40.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:40.341 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:40.617 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:40.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:40.625 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:40.725 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:40.753 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: pgmap v1158: 119 pgs: 1 unknown, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-09-23T15:28:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: osdmap e402: 8 total, 8 up, 8 in 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[222878]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: pgmap v1158: 119 pgs: 1 unknown, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: osdmap e402: 8 total, 8 up, 8 in 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: failed to encode map e402 with expected crc 2023-09-23T15:28:40.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:40 smithi017 ceph-mon[215646]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.087 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:41.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:41.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: pgmap v1158: 119 pgs: 1 unknown, 118 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 452 op/s 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: osdmap e402: 8 total, 8 up, 8 in 2023-09-23T15:28:41.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:40 smithi027 ceph-mon[195233]: failed to encode map e402 with expected crc 2023-09-23T15:28:41.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:41.237 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:41.261 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:41.318 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:41.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:41.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:41.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:28:41.476 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:41.553 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:28:41.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:41.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:28:41.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:41.754 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:41.828 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:41 smithi027 ceph-mon[195233]: pgmap v1160: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 8.2 KiB/s rd, 22 KiB/s wr, 934 op/s; 76 KiB/s, 0 objects/s recovering 2023-09-23T15:28:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:41 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:28:41.872 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:41.873 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:41.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:41.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:41.912 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:41.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:41 smithi017 ceph-mon[215646]: pgmap v1160: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 8.2 KiB/s rd, 22 KiB/s wr, 934 op/s; 76 KiB/s, 0 objects/s recovering 2023-09-23T15:28:41.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:41 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:28:41.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:41 smithi017 ceph-mon[222878]: pgmap v1160: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 8.2 KiB/s rd, 22 KiB/s wr, 934 op/s; 76 KiB/s, 0 objects/s recovering 2023-09-23T15:28:41.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:41 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:28:41.961 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:41.985 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:42.022 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:42.025 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:42.060 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:42.098 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:42.099 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:42.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:42.140 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:42.145 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.145 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.146 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:28:42.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:42.261 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:42.366 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:42.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:42.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:42.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:42.521 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:42.610 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:42.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:42.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:42.703 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3359293747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:42.703 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/811962060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:42.703 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:42.717 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:28:42.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:42.805 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3359293747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/811962060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3359293747' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/811962060' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:43.199 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:28:43.259 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:28:43.635 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:43.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:43.644 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:43.745 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:43.819 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:43.906 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:43.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:43.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: osdmap e403: 8 total, 8 up, 8 in 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: pgmap v1162: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6.7 KiB/s rd, 18 KiB/s wr, 758 op/s; 61 KiB/s, 0 objects/s recovering 2023-09-23T15:28:43.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: osdmap e403: 8 total, 8 up, 8 in 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: failed to encode map e403 with expected crc 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: pgmap v1162: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6.7 KiB/s rd, 18 KiB/s wr, 758 op/s; 61 KiB/s, 0 objects/s recovering 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:43.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:44.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:44.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: failed to encode map e403 with expected crc 2023-09-23T15:28:44.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:28:44.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: osdmap e403: 8 total, 8 up, 8 in 2023-09-23T15:28:44.058 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: failed to encode map e403 with expected crc 2023-09-23T15:28:44.059 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: failed to encode map e403 with expected crc 2023-09-23T15:28:44.059 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: pgmap v1162: 119 pgs: 2 peering, 117 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 6.7 KiB/s rd, 18 KiB/s wr, 758 op/s; 61 KiB/s, 0 objects/s recovering 2023-09-23T15:28:44.059 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:44.059 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:44.070 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:44.166 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:44.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:44.269 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:44.314 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:44.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:28:44.390 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:44.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:44.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:44.498 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:44.592 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:44.671 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:44.744 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:44.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:44.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:44.776 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:28:44.776 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:28:44.850 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:44.904 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:44.988 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:44.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:45.099 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:45.165 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:45.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:44 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T15:28:45.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:44 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T15:28:45.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:44 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs peering) 2023-09-23T15:28:45.654 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:28:45.784 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:45.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:45.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:45.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:45.936 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:45.990 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:45 smithi017 ceph-mon[215646]: pgmap v1163: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 1.06k op/s; 52 KiB/s, 0 objects/s recovering 2023-09-23T15:28:46.006 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:46.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:46.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:46.118 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:46.164 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:46.254 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:46.258 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:45 smithi027 ceph-mon[195233]: pgmap v1163: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 1.06k op/s; 52 KiB/s, 0 objects/s recovering 2023-09-23T15:28:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:46.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:46.359 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:46.408 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:46.421 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:45 smithi017 ceph-mon[222878]: pgmap v1163: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 5.6 KiB/s rd, 15 KiB/s wr, 1.06k op/s; 52 KiB/s, 0 objects/s recovering 2023-09-23T15:28:46.452 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:28:46.474 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:46.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:46.588 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:46.667 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:46.813 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:46.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:46.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:46.884 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:46.895 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:46.934 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:47.030 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:47.193 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:47.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:47.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:47.308 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:47.404 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:47.566 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:47.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:47.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:47.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:47.790 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:47.934 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:47.986 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:47.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:47.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:48.095 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:48.164 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:48.270 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:48.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:48.296 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:28:48.301 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:48] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-09-23T15:28:48.382 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:48.410 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:48.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:48 smithi027 ceph-mon[195233]: pgmap v1164: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 4.4 KiB/s rd, 11 KiB/s wr, 940 op/s; 40 KiB/s, 0 objects/s recovering 2023-09-23T15:28:48.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1307900520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:48.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[215646]: pgmap v1164: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 4.4 KiB/s rd, 11 KiB/s wr, 940 op/s; 40 KiB/s, 0 objects/s recovering 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1307900520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[222878]: pgmap v1164: 119 pgs: 119 active+clean; 2.2 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 4.4 KiB/s rd, 11 KiB/s wr, 940 op/s; 40 KiB/s, 0 objects/s recovering 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1307900520' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:48.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:28:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:49.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: osdmap e404: 8 total, 8 up, 8 in 2023-09-23T15:28:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: 1.6 scrub starts 2023-09-23T15:28:49.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:49 smithi027 ceph-mon[195233]: 1.6 scrub ok 2023-09-23T15:28:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: osdmap e404: 8 total, 8 up, 8 in 2023-09-23T15:28:49.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: 1.6 scrub starts 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[215646]: 1.6 scrub ok 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: osdmap e404: 8 total, 8 up, 8 in 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.696 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: failed to encode map e404 with expected crc 2023-09-23T15:28:49.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: 1.6 scrub starts 2023-09-23T15:28:49.697 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:49 smithi017 ceph-mon[222878]: 1.6 scrub ok 2023-09-23T15:28:49.932 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:49.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:49.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:50.040 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:50.146 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:50.290 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:50.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:50.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:50.371 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[215646]: 1.c scrub starts 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[215646]: pgmap v1166: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[215646]: 1.c scrub ok 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[215646]: 1.7 scrub starts 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[215646]: 1.7 scrub ok 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[222878]: 1.c scrub starts 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[222878]: pgmap v1166: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[222878]: 1.c scrub ok 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[222878]: 1.7 scrub starts 2023-09-23T15:28:50.372 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:50 smithi017 ceph-mon[222878]: 1.7 scrub ok 2023-09-23T15:28:50.399 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:50.476 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:50.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:50 smithi027 ceph-mon[195233]: 1.c scrub starts 2023-09-23T15:28:50.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:50 smithi027 ceph-mon[195233]: pgmap v1166: 119 pgs: 119 active+clean; 2.1 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2023-09-23T15:28:50.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:50 smithi027 ceph-mon[195233]: 1.c scrub ok 2023-09-23T15:28:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:50 smithi027 ceph-mon[195233]: 1.7 scrub starts 2023-09-23T15:28:50.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:50 smithi027 ceph-mon[195233]: 1.7 scrub ok 2023-09-23T15:28:50.787 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:50.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:50.795 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:50.896 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:28:50.946 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:51.032 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:28:51.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:51.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:28:51.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:51.190 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:51.329 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:51 smithi027 ceph-mon[195233]: 1.e scrub starts 2023-09-23T15:28:51.329 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:51 smithi027 ceph-mon[195233]: 1.e scrub ok 2023-09-23T15:28:51.329 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:51 smithi027 ceph-mon[195233]: 1.b scrub starts 2023-09-23T15:28:51.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:51 smithi027 ceph-mon[195233]: 1.b scrub ok 2023-09-23T15:28:51.330 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:28:51.359 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:28:51.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:51.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:51.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:51.606 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:51.637 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:51.637 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:51.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:51.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[222878]: 1.e scrub starts 2023-09-23T15:28:51.642 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[222878]: 1.e scrub ok 2023-09-23T15:28:51.643 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[222878]: 1.b scrub starts 2023-09-23T15:28:51.643 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[222878]: 1.b scrub ok 2023-09-23T15:28:51.643 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:28:51.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[215646]: 1.e scrub starts 2023-09-23T15:28:51.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[215646]: 1.e scrub ok 2023-09-23T15:28:51.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[215646]: 1.b scrub starts 2023-09-23T15:28:51.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[215646]: 1.b scrub ok 2023-09-23T15:28:51.644 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:28:51.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:51.718 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:51.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:51.751 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:51.770 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.770 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.771 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:28:51.805 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:52.073 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:28:52.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: 1.19 scrub starts 2023-09-23T15:28:52.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: 1.19 scrub ok 2023-09-23T15:28:52.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: pgmap v1167: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T15:28:52.346 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: osdmap e405: 8 total, 8 up, 8 in 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1650783326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: 1.1e scrub starts 2023-09-23T15:28:52.347 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[215646]: 1.1e scrub ok 2023-09-23T15:28:52.371 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:52.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:52.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:52.476 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:52.512 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: 1.19 scrub starts 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: 1.19 scrub ok 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: pgmap v1167: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: osdmap e405: 8 total, 8 up, 8 in 2023-09-23T15:28:52.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1650783326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:52.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: 1.1e scrub starts 2023-09-23T15:28:52.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:52 smithi027 ceph-mon[195233]: 1.1e scrub ok 2023-09-23T15:28:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: 1.19 scrub starts 2023-09-23T15:28:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: 1.19 scrub ok 2023-09-23T15:28:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: pgmap v1167: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2023-09-23T15:28:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: failed to encode map e405 with expected crc 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3743488705' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: osdmap e405: 8 total, 8 up, 8 in 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1650783326' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: 1.1e scrub starts 2023-09-23T15:28:52.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:52 smithi017 ceph-mon[222878]: 1.1e scrub ok 2023-09-23T15:28:53.065 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:28:53.065 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:28:53.247 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:53.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:53.361 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:53.396 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:53.405 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:53.565 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83106 9.e 9:7e487db8:::benchmark_data_smithi017_543597_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.636189069, 2023-09-23T15:28:53.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1221425209999998, 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83106 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:28:53.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916522+0000", 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916551+0000", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9121000000000001e-05 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916561+0000", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4059999999999994e-06 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916570+0000", 2023-09-23T15:28:53.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5149999999999995e-06 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918608+0000", 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020381800000000001 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918619+0000", 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1222e-05 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913525+0000", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490589500000004 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913537+0000", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1749000000000001e-05 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.028480+0000", 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1149425470000001 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.029082+0000", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060205899999999995 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038593+0000", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095110260000000005 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038666+0000", 2023-09-23T15:28:53.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3088000000000002e-05 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83114 9.d 9:bfb40ed3:::benchmark_data_smithi017_543597_object83113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.636136474, 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024477069, 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.579 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83114 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:28:53.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916575+0000", 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916600+0000", 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4606000000000001e-05 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916607+0000", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916613+0000", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2219999999999999e-06 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916703+0000", 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9252000000000005e-05 2023-09-23T15:28:53.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916711+0000", 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9039999999999998e-06 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.931237+0000", 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0145257830000001 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933189+0000", 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001951745 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933360+0000", 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017091799999999999 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940451+0000", 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070909079999999999 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940940+0000", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048871099999999996 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940977+0000", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7722999999999998e-05 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.941053+0000", 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5729000000000006e-05 2023-09-23T15:28:53.585 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83118 9.e 9:7615419c:::benchmark_data_smithi017_543597_object83117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.636086487, 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1299481519999999, 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.586 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83118 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916625+0000", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916651+0000", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5911e-05 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916659+0000", 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2023-09-23T15:28:53.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916665+0000", 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9460000000000003e-06 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918662+0000", 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019976350000000002 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918669+0000", 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2910000000000002e-06 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913570+0000", 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490066300000002 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913580+0000", 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.030629+0000", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170488599999999 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032411+0000", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001782063 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032533+0000", 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012223299999999999 2023-09-23T15:28:53.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038742+0000", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006208613 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038809+0000", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6854999999999996e-05 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045301+0000", 2023-09-23T15:28:53.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064921479999999997 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045369+0000", 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8693000000000001e-05 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046574+0000", 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012047169999999999 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.593 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83121 9.e 9:71f6c9a6:::benchmark_data_smithi017_543597_object83120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.63595673899999, 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1305874610000002, 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.594 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83121 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916755+0000", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916784+0000", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9522e-05 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916793+0000", 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1000000000000004e-06 2023-09-23T15:28:53.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916799+0000", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7530000000000004e-06 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918708+0000", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001908734 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918717+0000", 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6629999999999999e-06 2023-09-23T15:28:53.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914427+0000", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99571021100000001 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914437+0000", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0592e-05 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034594+0000", 2023-09-23T15:28:53.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12015687 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034760+0000", 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001661 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034983+0000", 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022256399999999999 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038750+0000", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037670210000000002 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038859+0000", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010865 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046631+0000", 2023-09-23T15:28:53.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077723879999999999 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046672+0000", 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0759000000000001e-05 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047343+0000", 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067126700000000002 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83124 9.3 9:c55b3f52:::benchmark_data_smithi017_543597_object83123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 117.633711214, 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0418616510000001, 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83124 2023-09-23T15:28:53.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919000+0000", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919028+0000", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8895e-05 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919041+0000", 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2996000000000001e-05 2023-09-23T15:28:53.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919051+0000", 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5729999999999997e-06 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919101+0000", 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9864999999999998e-05 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919108+0000", 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2760000000000003e-06 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.944052+0000", 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024943395 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946310+0000", 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022579900000000001 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946472+0000", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016254900000000001 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.958448+0000", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975497999999999 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960747+0000", 2023-09-23T15:28:53.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002299592 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960794+0000", 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6409000000000002e-05 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960863+0000", 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9227999999999995e-05 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83137 9.e 9:7d999fd1:::benchmark_data_smithi017_543597_object83136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 116.56111504499999, 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056604109, 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83137 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.609 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991596+0000", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991621+0000", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4907000000000001e-05 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991634+0000", 2023-09-23T15:28:53.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2556e-05 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991642+0000", 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1340000000000004e-06 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991689+0000", 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7781000000000002e-05 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991696+0000", 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5239999999999997e-06 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038894+0000", 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.047198039 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039062+0000", 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016794100000000001 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039246+0000", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018399900000000001 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045231+0000", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059852339999999999 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046596+0000", 2023-09-23T15:28:53.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001365125 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047421+0000", 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082434100000000003 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047483+0000", 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2686000000000003e-05 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048201+0000", 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071811100000000001 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83228 9.e 9:7f83ce07:::benchmark_data_smithi017_543597_object83227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 116.39656583, 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89863112300000003, 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.616 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83228 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156145+0000", 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156194+0000", 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8671000000000002e-05 2023-09-23T15:28:53.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156210+0000", 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6716999999999999e-05 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156219+0000", 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156278+0000", 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8897999999999998e-05 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156292+0000", 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042118+0000", 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88582658800000003 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042277+0000", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015909299999999999 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042470+0000", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193022 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048224+0000", 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057530630000000001 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048291+0000", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7751999999999998e-05 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.053961+0000", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056697969999999999 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054020+0000", 2023-09-23T15:28:53.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9419999999999997e-05 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054778+0000", 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075720099999999999 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83232 9.e 9:72608dfb:::benchmark_data_smithi017_543597_object83231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:28:53.623 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 116.384704192, 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88775156300000002, 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83232 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:28:53.624 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168006+0000", 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.625 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168035+0000", 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8824e-05 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168053+0000", 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8185e-05 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168065+0000", 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1391e-05 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.626 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168131+0000", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168140+0000", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7129999999999994e-06 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048345+0000", 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88020536000000005 2023-09-23T15:28:53.627 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048556+0000", 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021095900000000001 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048793+0000", 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023668699999999999 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.628 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054856+0000", 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060630129999999999 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054871+0000", 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851000000000001e-05 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054915+0000", 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4753999999999999e-05 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.629 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054975+0000", 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9562999999999999e-05 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078479899999999998 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.630 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83259 9.4 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 115.497007235, 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.144342432, 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83259 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.631 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055704+0000", 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.632 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055749+0000", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5618000000000003e-05 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012000000000001e-05 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055769+0000", 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.633 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055829+0000", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0362000000000002e-05 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056020+0000", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190998 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056238+0000", 2023-09-23T15:28:53.634 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021735600000000001 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061804+0000", 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055659680000000001 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065167+0000", 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033630359999999998 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.635 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197822+0000", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132655089 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197887+0000", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200048+0000", 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002161035 2023-09-23T15:28:53.636 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83260 9.4 9:295d7846:::benchmark_data_smithi017_543597_object83259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 115.496927704, 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.146144756, 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83260 2023-09-23T15:28:53.637 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.638 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055784+0000", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055805+0000", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055812+0000", 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4960000000000001e-06 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.639 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055819+0000", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.058865+0000", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030458220000000001 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059023+0000", 2023-09-23T15:28:53.640 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015745099999999999 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059189+0000", 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166126 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065176+0000", 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005987132 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.641 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068875+0000", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003699698 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200125+0000", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131249117999999 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200200+0000", 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4951999999999997e-05 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.642 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.201929+0000", 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001729897 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83261 9.4 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 115.496102394, 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.147188163999999, 2023-09-23T15:28:53.643 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83261 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.644 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056609+0000", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056631+0000", 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1569999999999998e-05 2023-09-23T15:28:53.645 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056646+0000", 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4924e-05 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056654+0000", 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0160000000000008e-06 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.646 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061842+0000", 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051878740000000003 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062037+0000", 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019499900000000001 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062246+0000", 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020959400000000001 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.647 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065285+0000", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030387909999999999 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069749+0000", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044636470000000003 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202002+0000", 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132253035 2023-09-23T15:28:53.648 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202066+0000", 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4215999999999996e-05 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203798+0000", 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017322940000000001 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.649 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83265 9.4 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 115.487159121, 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.139692201999999, 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83265 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.650 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.651 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065572+0000", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8663000000000001e-05 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065576+0000", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7530000000000001e-06 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065580+0000", 2023-09-23T15:28:53.652 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.326e-06 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065625+0000", 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5077999999999998e-05 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065774+0000", 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014929099999999999 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:28:53.653 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065946+0000", 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017193699999999999 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068958+0000", 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030115939999999998 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069898+0000", 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093998699999999998 2023-09-23T15:28:53.654 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203861+0000", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.133963208000001 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203918+0000", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6950999999999997e-05 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.205245+0000", 2023-09-23T15:28:53.655 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013277479999999999 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:85712 9.4 9:245c7638:::benchmark_data_smithi017_543597_object1921:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:37.251490+0000", 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 76.301222269999997, 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.32249433599999999, 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.656 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 85712 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251490+0000", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251490+0000", 2023-09-23T15:28:53.657 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251488+0000", 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251494+0000", 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2830000000000001e-06 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.658 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251505+0000", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1059e-05 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251512+0000", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2629999999999997e-06 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251564+0000", 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1804000000000001e-05 2023-09-23T15:28:53.659 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251722+0000", 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015791000000000001 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.251847+0000", 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012543299999999999 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.660 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.258509+0000", 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066617029999999997 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.262640+0000", 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041312379999999998 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.573354+0000", 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31071388 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.661 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.573413+0000", 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9357000000000001e-05 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.573984+0000", 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00057103500000000001 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.662 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:85716 9.4 9:2adf907e:::benchmark_data_smithi017_543597_object1925:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e372)", 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:37.254587+0000", 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 76.298125451999994, 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31975354700000003, 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 85716 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.663 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254587+0000", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254587+0000", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254585+0000", 2023-09-23T15:28:53.664 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254592+0000", 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1600000000000001e-06 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254610+0000", 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7331000000000001e-05 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.665 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254619+0000", 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.2089999999999994e-06 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.254904+0000", 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00028475000000000001 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.255049+0000", 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014547 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.666 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.255158+0000", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010892599999999999 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.258519+0000", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003361242 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.262675+0000", 2023-09-23T15:28:53.667 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041556350000000004 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.574029+0000", 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.31135371000000001 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.574072+0000", 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3577e-05 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.668 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:37.574340+0000", 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026838699999999998 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88039 9.d 9:b338c46d:::benchmark_data_smithi017_543597_object4248:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141932444000005, 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90200742499999997, 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:28:53.669 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88039 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:28:53.670 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410779+0000", 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410784+0000", 2023-09-23T15:28:53.671 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5480000000000001e-06 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410796+0000", 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1653e-05 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410807+0000", 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1181e-05 2023-09-23T15:28:53.672 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410861+0000", 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3981999999999999e-05 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410880+0000", 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8553999999999999e-05 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.673 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420827+0000", 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0099472469999999993 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420840+0000", 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2969999999999999e-05 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.302886+0000", 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88204606500000005 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.674 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303061+0000", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175076 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312708+0000", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096470840000000002 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312787+0000", 2023-09-23T15:28:53.675 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9296999999999997e-05 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88041 9.d 9:b6e8eb25:::benchmark_data_smithi017_543597_object4250:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141891427000004, 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90221110000000004, 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.676 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88041 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.677 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410820+0000", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410824+0000", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3720000000000001e-06 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410831+0000", 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:28:53.678 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410837+0000", 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.19e-06 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410931+0000", 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3634000000000005e-05 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:28:53.679 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410938+0000", 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422325+0000", 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011386744000000001 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422336+0000", 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1028e-05 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.680 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303400+0000", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88106453799999995 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303535+0000", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013442700000000001 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303696+0000", 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016155799999999999 2023-09-23T15:28:53.681 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309737+0000", 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060407810000000003 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309784+0000", 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7237000000000002e-05 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.682 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312835+0000", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030510619999999998 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312900+0000", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4393999999999994e-05 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313032+0000", 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013220300000000001 2023-09-23T15:28:53.683 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88044 9.d 9:bfc8f2ba:::benchmark_data_smithi017_543597_object4253:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141871007999995, 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90561986000000005, 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.684 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88044 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.685 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410843+0000", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2079999999999999e-06 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410849+0000", 2023-09-23T15:28:53.686 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6489999999999996e-06 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410854+0000", 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9810000000000003e-06 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410963+0000", 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109166 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.687 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410970+0000", 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5649999999999998e-06 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422375+0000", 2023-09-23T15:28:53.688 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01140536 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422395+0000", 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9913000000000001e-05 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306676+0000", 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88428063000000001 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.689 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306827+0000", 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015183300000000001 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306965+0000", 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137427 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.690 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313079+0000", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061140869999999998 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313118+0000", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316281+0000", 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031639160000000001 2023-09-23T15:28:53.691 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316328+0000", 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6897000000000002e-05 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316461+0000", 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013290900000000001 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.692 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88045 9.d 9:b617d99b:::benchmark_data_smithi017_543597_object4254:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141854105999997, 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91186458299999995, 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88045 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.693 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.694 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410860+0000", 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5600000000000001e-06 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410866+0000", 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6860000000000003e-06 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.695 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410871+0000", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9980000000000003e-06 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410993+0000", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122359 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411000+0000", 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0950000000000001e-06 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.696 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422421+0000", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011421093 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422430+0000", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309805+0000", 2023-09-23T15:28:53.697 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88737535099999998 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309933+0000", 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012809900000000001 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310023+0000", 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9247999999999997e-05 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.698 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313160+0000", 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031370759999999999 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316471+0000", 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003311278 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322477+0000", 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060065279999999997 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.699 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322536+0000", 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8792000000000002e-05 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322723+0000", 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018669199999999999 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.700 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88046 9.3 9:c0e328f8:::benchmark_data_smithi017_543597_object4255:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141837326000001, 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89982923699999995, 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88046 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.701 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.702 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410877+0000", 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4669999999999998e-06 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410885+0000", 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.841e-06 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.703 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410890+0000", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411148+0000", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025797 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411157+0000", 2023-09-23T15:28:53.704 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5380000000000002e-06 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413770+0000", 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026130089999999999 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413782+0000", 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.289e-05 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.705 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303621+0000", 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88983906499999998 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303734+0000", 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011293099999999999 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:28:53.706 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303859+0000", 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012465600000000001 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310319+0000", 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006459752 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310340+0000", 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1526999999999999e-05 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.707 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310470+0000", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012976200000000001 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310519+0000", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8906999999999998e-05 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310704+0000", 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018528199999999999 2023-09-23T15:28:53.708 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88047 9.3 9:c39718c3:::benchmark_data_smithi017_543597_object4256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 72.141809825999999, 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90486523799999996, 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:28:53.709 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88047 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:28:53.710 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410902+0000", 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410905+0000", 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6520000000000002e-06 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.711 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410911+0000", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410917+0000", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5759999999999996e-06 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411181+0000", 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026399799999999999 2023-09-23T15:28:53.712 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411187+0000", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6200000000000001e-06 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413808+0000", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026202399999999998 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413816+0000", 2023-09-23T15:28:53.713 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2530000000000004e-06 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307084+0000", 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89326843099999997 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307210+0000", 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125623 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.714 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307340+0000", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013041499999999999 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.311327+0000", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039864119999999999 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313609+0000", 2023-09-23T15:28:53.715 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002282315 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315515+0000", 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019060640000000001 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315579+0000", 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3367999999999994e-05 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:28:53.716 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315768+0000", 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018929 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:28:53.717 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:28:53.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:53.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:53.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[215646]: 1.1a scrub starts 2023-09-23T15:28:53.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[215646]: 1.1a scrub ok 2023-09-23T15:28:53.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[215646]: 1.9 scrub starts 2023-09-23T15:28:53.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[215646]: 1.9 scrub ok 2023-09-23T15:28:53.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[222878]: 1.1a scrub starts 2023-09-23T15:28:53.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[222878]: 1.1a scrub ok 2023-09-23T15:28:53.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[222878]: 1.9 scrub starts 2023-09-23T15:28:53.719 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:53 smithi017 ceph-mon[222878]: 1.9 scrub ok 2023-09-23T15:28:53.818 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:53.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:53 smithi027 ceph-mon[195233]: 1.1a scrub starts 2023-09-23T15:28:53.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:53 smithi027 ceph-mon[195233]: 1.1a scrub ok 2023-09-23T15:28:53.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:53 smithi027 ceph-mon[195233]: 1.9 scrub starts 2023-09-23T15:28:53.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:53 smithi027 ceph-mon[195233]: 1.9 scrub ok 2023-09-23T15:28:53.891 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:54.681 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:54 smithi017 ceph-mon[215646]: pgmap v1169: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T15:28:54.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:54 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:54.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:54 smithi017 ceph-mon[222878]: pgmap v1169: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T15:28:54.683 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:54 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:54.722 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:54.724 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:28:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:54.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:54.759 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:54 smithi027 ceph-mon[195233]: pgmap v1169: 119 pgs: 1 active+clean+scrubbing, 118 active+clean; 1.9 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 714 op/s 2023-09-23T15:28:54.759 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:54 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:54.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:54.885 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:55.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:28:55.189 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:55.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:55.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:55.263 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:55.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:55.337 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:55.480 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:55.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:55.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:55.587 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:55.653 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:55.786 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:55.790 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:55.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:55.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:55.907 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:28:55.968 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:56.011 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:28:56.133 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:28:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:56.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:28:56.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:56.267 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:56.313 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:28:56.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:56.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:28:56.417 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:28:56.518 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:56.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:56 smithi027 ceph-mon[195233]: pgmap v1170: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T15:28:56.610 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:56 smithi017 ceph-mon[215646]: pgmap v1170: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T15:28:56.611 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:56 smithi017 ceph-mon[222878]: pgmap v1170: 119 pgs: 119 active+clean; 1.8 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 815 op/s 2023-09-23T15:28:56.679 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:28:56.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:56.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:28:56.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:28:56.864 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:57.272 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:28:57.272 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:28:57.272 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:28:57.272 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:28:57.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:28:57.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: 1.1b scrub ok 2023-09-23T15:28:57.387 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: 1.d scrub starts 2023-09-23T15:28:57.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: 1.d scrub ok 2023-09-23T15:28:57.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3493662897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:57.388 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:57.411 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: 1.1b scrub ok 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: 1.d scrub starts 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: 1.d scrub ok 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3493662897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:57.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:57.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:28:57.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: 1.1b scrub ok 2023-09-23T15:28:57.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: 1.d scrub starts 2023-09-23T15:28:57.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: 1.d scrub ok 2023-09-23T15:28:57.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3493662897' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:57.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:28:58.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:28:58.294 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:28:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:28:58] "GET /metrics HTTP/1.1" 200 36384 "" "Prometheus/2.43.0" 2023-09-23T15:28:58.311 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:28:58.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:58.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:28:58.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:58.482 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:58.603 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:28:58.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: pgmap v1171: 119 pgs: 119 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T15:28:58.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:58.697 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: osdmap e406: 8 total, 8 up, 8 in 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: 1.1d scrub starts 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: 1.1d scrub ok 2023-09-23T15:28:58.698 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: pgmap v1171: 119 pgs: 119 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.699 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: osdmap e406: 8 total, 8 up, 8 in 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: 1.1d scrub starts 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: 1.1d scrub ok 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:58.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:58.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: pgmap v1171: 119 pgs: 119 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2023-09-23T15:28:58.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:28:58.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: osdmap e406: 8 total, 8 up, 8 in 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: failed to encode map e406 with expected crc 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: 1.1d scrub starts 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: 1.1d scrub ok 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:28:58.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:28:58.865 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:58.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:58.874 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:58.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:28:59.039 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:59.195 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:28:59.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:59.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:28:59.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:28:59.375 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:59.515 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:28:59.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:59 smithi017 ceph-mon[215646]: 1.1c deep-scrub starts 2023-09-23T15:28:59.517 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:28:59 smithi017 ceph-mon[215646]: 1.1c deep-scrub ok 2023-09-23T15:28:59.517 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:59 smithi017 ceph-mon[222878]: 1.1c deep-scrub starts 2023-09-23T15:28:59.517 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:28:59 smithi017 ceph-mon[222878]: 1.1c deep-scrub ok 2023-09-23T15:28:59.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:28:59.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:28:59.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:28:59.546 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:28:59.570 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:28:59.589 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:28:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:59 smithi027 ceph-mon[195233]: 1.1c deep-scrub starts 2023-09-23T15:28:59.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:28:59 smithi027 ceph-mon[195233]: 1.1c deep-scrub ok 2023-09-23T15:28:59.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:28:59.631 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:28:59.664 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:28:59.681 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:28:59.722 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:28:59.764 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.785 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:28:59.948 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:28:59.980 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:28:59.989 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:00.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:00.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:00.166 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:00.251 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:00.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:00.255 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:00.356 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:00.410 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:00.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:00 smithi027 ceph-mon[195233]: pgmap v1173: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:29:00.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1837611368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:00.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:00 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:29:00.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:00 smithi027 ceph-mon[195233]: 1.17 scrub ok 2023-09-23T15:29:00.643 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:00.643 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[222878]: pgmap v1173: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1837611368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[222878]: 1.17 scrub ok 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[215646]: pgmap v1173: 119 pgs: 119 active+clean; 1.7 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 579 op/s 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1837611368' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:29:00.671 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:00 smithi017 ceph-mon[215646]: 1.17 scrub ok 2023-09-23T15:29:00.812 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:00.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:00.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:00.925 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:00.954 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:01.000 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:01.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:01.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:01.103 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:01.108 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:29:01.131 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:01.494 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:01.501 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1173,"stamp":"2023-09-23T15:28:58.877111+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1773989670,"num_objects":27023,"num_object_clones":0,"num_object_copies":81069,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27023,"num_whiteouts":0,"num_read":6015,"num_read_kb":13248,"num_write":141501,"num_write_kb":5375250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13130,"num_bytes_recovered":705834613,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108093,"ondisk_log_size":108093,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9474524,"kb_used_data":5800412,"kb_used_omap":57,"kb_used_meta":3673990,"kb_avail":740486692,"statfs":{"total":767960285184,"available":758258372608,"internally_reserved":0,"allocated":5939621888,"data_stored":5926816034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":3762165785},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-303890432,"num_objects":-4637,"num_object_clones":0,"num_object_copies":-13911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-4637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4637,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002547"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":753,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294518+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:28:48.294518+0000","last_peered":"2023-09-23T15:28:48.294518+0000","last_clean":"2023-09-23T15:28:48.294518+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:28:48.294518+0000","last_undegraded":"2023-09-23T15:28:48.294518+0000","last_fullsized":"2023-09-23T15:28:48.294518+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294813+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:28:48.294813+0000","last_peered":"2023-09-23T15:28:48.294813+0000","last_clean":"2023-09-23T15:28:48.294813+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:28:48.294813+0000","last_undegraded":"2023-09-23T15:28:48.294813+0000","last_fullsized":"2023-09-23T15:28:48.294813+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":359,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.300679+0000","last_change":"2023-09-23T15:27:23.313341+0000","last_active":"2023-09-23T15:28:48.300679+0000","last_peered":"2023-09-23T15:28:48.300679+0000","last_clean":"2023-09-23T15:28:48.300679+0000","last_became_active":"2023-09-23T15:27:23.313012+0000","last_became_peered":"2023-09-23T15:27:23.313012+0000","last_unstale":"2023-09-23T15:28:48.300679+0000","last_undegraded":"2023-09-23T15:28:48.300679+0000","last_fullsized":"2023-09-23T15:28:48.300679+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:17:06.833473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":477,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294178+0000","last_change":"2023-09-23T15:28:39.667064+0000","last_active":"2023-09-23T15:28:48.294178+0000","last_peered":"2023-09-23T15:28:48.294178+0000","last_clean":"2023-09-23T15:28:48.294178+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:28:48.294178+0000","last_undegraded":"2023-09-23T15:28:48.294178+0000","last_fullsized":"2023-09-23T15:28:48.294178+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:50.492624+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:50.492624+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":8,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":329,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.311548+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:28:48.311548+0000","last_peered":"2023-09-23T15:28:48.311548+0000","last_clean":"2023-09-23T15:28:48.311548+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:28:48.311548+0000","last_undegraded":"2023-09-23T15:28:48.311548+0000","last_fullsized":"2023-09-23T15:28:48.311548+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.436971+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:28:51.436971+0000","last_peered":"2023-09-23T15:28:51.436971+0000","last_clean":"2023-09-23T15:28:51.436971+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:28:51.436971+0000","last_undegraded":"2023-09-23T15:28:51.436971+0000","last_fullsized":"2023-09-23T15:28:51.436971+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":540,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.947595+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:28:51.947595+0000","last_peered":"2023-09-23T15:28:51.947595+0000","last_clean":"2023-09-23T15:28:51.947595+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:28:51.947595+0000","last_undegraded":"2023-09-23T15:28:51.947595+0000","last_fullsized":"2023-09-23T15:28:51.947595+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":396,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298012+0000","last_change":"2023-09-23T15:27:12.402468+0000","last_active":"2023-09-23T15:28:48.298012+0000","last_peered":"2023-09-23T15:28:48.298012+0000","last_clean":"2023-09-23T15:28:48.298012+0000","last_became_active":"2023-09-23T15:27:12.402003+0000","last_became_peered":"2023-09-23T15:27:12.402003+0000","last_unstale":"2023-09-23T15:28:48.298012+0000","last_undegraded":"2023-09-23T15:28:48.298012+0000","last_fullsized":"2023-09-23T15:28:48.298012+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:45.443108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":355,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295191+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:28:48.295191+0000","last_peered":"2023-09-23T15:28:48.295191+0000","last_clean":"2023-09-23T15:28:48.295191+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:28:48.295191+0000","last_undegraded":"2023-09-23T15:28:48.295191+0000","last_fullsized":"2023-09-23T15:28:48.295191+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":673,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303265+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:28:48.303265+0000","last_peered":"2023-09-23T15:28:48.303265+0000","last_clean":"2023-09-23T15:28:48.303265+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:28:48.303265+0000","last_undegraded":"2023-09-23T15:28:48.303265+0000","last_fullsized":"2023-09-23T15:28:48.303265+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373650+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:28:51.373650+0000","last_peered":"2023-09-23T15:28:51.373650+0000","last_clean":"2023-09-23T15:28:51.373650+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:28:51.373650+0000","last_undegraded":"2023-09-23T15:28:51.373650+0000","last_fullsized":"2023-09-23T15:28:51.373650+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":393,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298454+0000","last_change":"2023-09-23T15:27:12.402484+0000","last_active":"2023-09-23T15:28:48.298454+0000","last_peered":"2023-09-23T15:28:48.298454+0000","last_clean":"2023-09-23T15:28:48.298454+0000","last_became_active":"2023-09-23T15:27:12.402209+0000","last_became_peered":"2023-09-23T15:27:12.402209+0000","last_unstale":"2023-09-23T15:28:48.298454+0000","last_undegraded":"2023-09-23T15:28:48.298454+0000","last_fullsized":"2023-09-23T15:28:48.298454+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:59:07.497544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":815,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303645+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:28:48.303645+0000","last_peered":"2023-09-23T15:28:48.303645+0000","last_clean":"2023-09-23T15:28:48.303645+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:28:48.303645+0000","last_undegraded":"2023-09-23T15:28:48.303645+0000","last_fullsized":"2023-09-23T15:28:48.303645+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":422,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296676+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:28:48.296676+0000","last_peered":"2023-09-23T15:28:48.296676+0000","last_clean":"2023-09-23T15:28:48.296676+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:28:48.296676+0000","last_undegraded":"2023-09-23T15:28:48.296676+0000","last_fullsized":"2023-09-23T15:28:48.296676+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":353,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298801+0000","last_change":"2023-09-23T15:27:41.424467+0000","last_active":"2023-09-23T15:28:48.298801+0000","last_peered":"2023-09-23T15:28:48.298801+0000","last_clean":"2023-09-23T15:28:48.298801+0000","last_became_active":"2023-09-23T15:27:41.424098+0000","last_became_peered":"2023-09-23T15:27:41.424098+0000","last_unstale":"2023-09-23T15:28:48.298801+0000","last_undegraded":"2023-09-23T15:28:48.298801+0000","last_fullsized":"2023-09-23T15:28:48.298801+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:46.478911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":498,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294943+0000","last_change":"2023-09-23T15:28:39.669069+0000","last_active":"2023-09-23T15:28:48.294943+0000","last_peered":"2023-09-23T15:28:48.294943+0000","last_clean":"2023-09-23T15:28:48.294943+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:28:48.294943+0000","last_undegraded":"2023-09-23T15:28:48.294943+0000","last_fullsized":"2023-09-23T15:28:48.294943+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:21:22.489617+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:21:22.489617+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0042263439999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":150,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303854+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:28:48.303854+0000","last_peered":"2023-09-23T15:28:48.303854+0000","last_clean":"2023-09-23T15:28:48.303854+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:28:48.303854+0000","last_undegraded":"2023-09-23T15:28:48.303854+0000","last_fullsized":"2023-09-23T15:28:48.303854+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":297,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.314063+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:28:48.314063+0000","last_peered":"2023-09-23T15:28:48.314063+0000","last_clean":"2023-09-23T15:28:48.314063+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:28:48.314063+0000","last_undegraded":"2023-09-23T15:28:48.314063+0000","last_fullsized":"2023-09-23T15:28:48.314063+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":543,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.315721+0000","last_change":"2023-09-23T15:28:39.670469+0000","last_active":"2023-09-23T15:28:48.315721+0000","last_peered":"2023-09-23T15:28:48.315721+0000","last_clean":"2023-09-23T15:28:48.315721+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:28:48.315721+0000","last_undegraded":"2023-09-23T15:28:48.315721+0000","last_fullsized":"2023-09-23T15:28:48.315721+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:41.310801+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:41.310801+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":71,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298083+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:28:48.298083+0000","last_peered":"2023-09-23T15:28:48.298083+0000","last_clean":"2023-09-23T15:28:48.298083+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:28:48.298083+0000","last_undegraded":"2023-09-23T15:28:48.298083+0000","last_fullsized":"2023-09-23T15:28:48.298083+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.297433+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:28:48.297433+0000","last_peered":"2023-09-23T15:28:48.297433+0000","last_clean":"2023-09-23T15:28:48.297433+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:28:48.297433+0000","last_undegraded":"2023-09-23T15:28:48.297433+0000","last_fullsized":"2023-09-23T15:28:48.297433+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":717,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437222+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:28:51.437222+0000","last_peered":"2023-09-23T15:28:51.437222+0000","last_clean":"2023-09-23T15:28:51.437222+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:28:51.437222+0000","last_undegraded":"2023-09-23T15:28:51.437222+0000","last_fullsized":"2023-09-23T15:28:51.437222+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":154,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.302618+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:28:48.302618+0000","last_peered":"2023-09-23T15:28:48.302618+0000","last_clean":"2023-09-23T15:28:48.302618+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:28:48.302618+0000","last_undegraded":"2023-09-23T15:28:48.302618+0000","last_fullsized":"2023-09-23T15:28:48.302618+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":403,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298867+0000","last_change":"2023-09-23T15:27:41.426159+0000","last_active":"2023-09-23T15:28:48.298867+0000","last_peered":"2023-09-23T15:28:48.298867+0000","last_clean":"2023-09-23T15:28:48.298867+0000","last_became_active":"2023-09-23T15:27:41.425781+0000","last_became_peered":"2023-09-23T15:27:41.425781+0000","last_unstale":"2023-09-23T15:28:48.298867+0000","last_undegraded":"2023-09-23T15:28:48.298867+0000","last_fullsized":"2023-09-23T15:28:48.298867+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:16.375308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":780,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294772+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:28:48.294772+0000","last_peered":"2023-09-23T15:28:48.294772+0000","last_clean":"2023-09-23T15:28:48.294772+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:28:48.294772+0000","last_undegraded":"2023-09-23T15:28:48.294772+0000","last_fullsized":"2023-09-23T15:28:48.294772+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":400,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372651+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:28:51.372651+0000","last_peered":"2023-09-23T15:28:51.372651+0000","last_clean":"2023-09-23T15:28:51.372651+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:28:51.372651+0000","last_undegraded":"2023-09-23T15:28:51.372651+0000","last_fullsized":"2023-09-23T15:28:51.372651+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":407,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295715+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:28:48.295715+0000","last_peered":"2023-09-23T15:28:48.295715+0000","last_clean":"2023-09-23T15:28:48.295715+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:28:48.295715+0000","last_undegraded":"2023-09-23T15:28:48.295715+0000","last_fullsized":"2023-09-23T15:28:48.295715+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":731,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.311855+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:28:48.311855+0000","last_peered":"2023-09-23T15:28:48.311855+0000","last_clean":"2023-09-23T15:28:48.311855+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:28:48.311855+0000","last_undegraded":"2023-09-23T15:28:48.311855+0000","last_fullsized":"2023-09-23T15:28:48.311855+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":148,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.302310+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:28:48.302310+0000","last_peered":"2023-09-23T15:28:48.302310+0000","last_clean":"2023-09-23T15:28:48.302310+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:28:48.302310+0000","last_undegraded":"2023-09-23T15:28:48.302310+0000","last_fullsized":"2023-09-23T15:28:48.302310+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":389,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437117+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:28:51.437117+0000","last_peered":"2023-09-23T15:28:51.437117+0000","last_clean":"2023-09-23T15:28:51.437117+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:28:51.437117+0000","last_undegraded":"2023-09-23T15:28:51.437117+0000","last_fullsized":"2023-09-23T15:28:51.437117+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":721,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.299122+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:28:48.299122+0000","last_peered":"2023-09-23T15:28:48.299122+0000","last_clean":"2023-09-23T15:28:48.299122+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:28:48.299122+0000","last_undegraded":"2023-09-23T15:28:48.299122+0000","last_fullsized":"2023-09-23T15:28:48.299122+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":341,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295320+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:28:48.295320+0000","last_peered":"2023-09-23T15:28:48.295320+0000","last_clean":"2023-09-23T15:28:48.295320+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:28:48.295320+0000","last_undegraded":"2023-09-23T15:28:48.295320+0000","last_fullsized":"2023-09-23T15:28:48.295320+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":404,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.308020+0000","last_change":"2023-09-23T15:27:41.442107+0000","last_active":"2023-09-23T15:28:48.308020+0000","last_peered":"2023-09-23T15:28:48.308020+0000","last_clean":"2023-09-23T15:28:48.308020+0000","last_became_active":"2023-09-23T15:27:41.441806+0000","last_became_peered":"2023-09-23T15:27:41.441806+0000","last_unstale":"2023-09-23T15:28:48.308020+0000","last_undegraded":"2023-09-23T15:28:48.308020+0000","last_fullsized":"2023-09-23T15:28:48.308020+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:31.229332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"406'8984","reported_seq":9709,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:58.778493+0000","last_change":"2023-09-23T15:28:32.579291+0000","last_active":"2023-09-23T15:28:58.778493+0000","last_peered":"2023-09-23T15:28:58.778493+0000","last_clean":"2023-09-23T15:28:58.778493+0000","last_became_active":"2023-09-23T15:28:32.565882+0000","last_became_peered":"2023-09-23T15:28:32.565882+0000","last_unstale":"2023-09-23T15:28:58.778493+0000","last_undegraded":"2023-09-23T15:28:58.778493+0000","last_fullsized":"2023-09-23T15:28:58.778493+0000","mapping_epoch":395,"log_start":"345'2100","ondisk_log_start":"345'2100","created":339,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6884,"log_dups_size":0,"ondisk_log_size":6884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:41.421234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101974016,"num_objects":1556,"num_object_clones":0,"num_object_copies":4668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1556,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8984,"num_write_kb":337280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":3342336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":691,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303736+0000","last_change":"2023-09-23T15:27:42.381596+0000","last_active":"2023-09-23T15:28:48.303736+0000","last_peered":"2023-09-23T15:28:48.303736+0000","last_clean":"2023-09-23T15:28:48.303736+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:28:48.303736+0000","last_undegraded":"2023-09-23T15:28:48.303736+0000","last_fullsized":"2023-09-23T15:28:48.303736+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":144,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303127+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:28:48.303127+0000","last_peered":"2023-09-23T15:28:48.303127+0000","last_clean":"2023-09-23T15:28:48.303127+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:28:48.303127+0000","last_undegraded":"2023-09-23T15:28:48.303127+0000","last_fullsized":"2023-09-23T15:28:48.303127+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":414,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294641+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:28:48.294641+0000","last_peered":"2023-09-23T15:28:48.294641+0000","last_clean":"2023-09-23T15:28:48.294641+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:28:48.294641+0000","last_undegraded":"2023-09-23T15:28:48.294641+0000","last_fullsized":"2023-09-23T15:28:48.294641+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"405'8883","reported_seq":14523,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:55.649654+0000","last_change":"2023-09-23T15:27:43.281637+0000","last_active":"2023-09-23T15:28:55.649654+0000","last_peered":"2023-09-23T15:28:55.649654+0000","last_clean":"2023-09-23T15:28:55.649654+0000","last_became_active":"2023-09-23T15:27:43.271323+0000","last_became_peered":"2023-09-23T15:27:43.271323+0000","last_unstale":"2023-09-23T15:28:55.649654+0000","last_undegraded":"2023-09-23T15:28:55.649654+0000","last_fullsized":"2023-09-23T15:28:55.649654+0000","mapping_epoch":376,"log_start":"346'2756","ondisk_log_start":"346'2756","created":339,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6127,"log_dups_size":0,"ondisk_log_size":6127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:59.795930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109903872,"num_objects":1677,"num_object_clones":0,"num_object_copies":5031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8883,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2762,"num_bytes_recovered":181010432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":725,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.882800+0000","last_change":"2023-09-23T15:28:48.882800+0000","last_active":"2023-09-23T15:28:48.882800+0000","last_peered":"2023-09-23T15:28:48.882800+0000","last_clean":"2023-09-23T15:28:48.882800+0000","last_became_active":"2023-09-23T15:27:12.393499+0000","last_became_peered":"2023-09-23T15:27:12.393499+0000","last_unstale":"2023-09-23T15:28:48.882800+0000","last_undegraded":"2023-09-23T15:28:48.882800+0000","last_fullsized":"2023-09-23T15:28:48.882800+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.882635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.882635+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:38.002743+0000","scrub_duration":0.08250528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":144,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303483+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:28:48.303483+0000","last_peered":"2023-09-23T15:28:48.303483+0000","last_clean":"2023-09-23T15:28:48.303483+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:28:48.303483+0000","last_undegraded":"2023-09-23T15:28:48.303483+0000","last_fullsized":"2023-09-23T15:28:48.303483+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294328+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:28:48.294328+0000","last_peered":"2023-09-23T15:28:48.294328+0000","last_clean":"2023-09-23T15:28:48.294328+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:28:48.294328+0000","last_undegraded":"2023-09-23T15:28:48.294328+0000","last_fullsized":"2023-09-23T15:28:48.294328+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"405'8613","reported_seq":9316,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.946921+0000","last_change":"2023-09-23T15:28:27.475997+0000","last_active":"2023-09-23T15:28:54.946921+0000","last_peered":"2023-09-23T15:28:54.946921+0000","last_clean":"2023-09-23T15:28:54.946921+0000","last_became_active":"2023-09-23T15:28:27.405233+0000","last_became_peered":"2023-09-23T15:28:27.405233+0000","last_unstale":"2023-09-23T15:28:54.946921+0000","last_undegraded":"2023-09-23T15:28:54.946921+0000","last_fullsized":"2023-09-23T15:28:54.946921+0000","mapping_epoch":390,"log_start":"345'1597","ondisk_log_start":"345'1597","created":339,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7016,"log_dups_size":0,"ondisk_log_size":7016,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:17.717875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":107806720,"num_objects":1645,"num_object_clones":0,"num_object_copies":4935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8613,"num_write_kb":328256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":652,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295377+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:28:48.295377+0000","last_peered":"2023-09-23T15:28:48.295377+0000","last_clean":"2023-09-23T15:28:48.295377+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:28:48.295377+0000","last_undegraded":"2023-09-23T15:28:48.295377+0000","last_fullsized":"2023-09-23T15:28:48.295377+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":132,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303369+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:28:48.303369+0000","last_peered":"2023-09-23T15:28:48.303369+0000","last_clean":"2023-09-23T15:28:48.303369+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:28:48.303369+0000","last_undegraded":"2023-09-23T15:28:48.303369+0000","last_fullsized":"2023-09-23T15:28:48.303369+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":385,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296195+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:28:48.296195+0000","last_peered":"2023-09-23T15:28:48.296195+0000","last_clean":"2023-09-23T15:28:48.296195+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:28:48.296195+0000","last_undegraded":"2023-09-23T15:28:48.296195+0000","last_fullsized":"2023-09-23T15:28:48.296195+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"405'8706","reported_seq":8780,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.991635+0000","last_change":"2023-09-23T15:25:35.723494+0000","last_active":"2023-09-23T15:28:54.991635+0000","last_peered":"2023-09-23T15:28:54.991635+0000","last_clean":"2023-09-23T15:28:54.991635+0000","last_became_active":"2023-09-23T15:25:35.723097+0000","last_became_peered":"2023-09-23T15:25:35.723097+0000","last_unstale":"2023-09-23T15:28:54.991635+0000","last_undegraded":"2023-09-23T15:28:54.991635+0000","last_fullsized":"2023-09-23T15:28:54.991635+0000","mapping_epoch":339,"log_start":"345'1728","ondisk_log_start":"345'1728","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:58.863529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108396544,"num_objects":1654,"num_object_clones":0,"num_object_copies":4962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":331520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":622,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:49.949075+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:28:49.949075+0000","last_peered":"2023-09-23T15:28:49.949075+0000","last_clean":"2023-09-23T15:28:49.949075+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:28:49.949075+0000","last_undegraded":"2023-09-23T15:28:49.949075+0000","last_fullsized":"2023-09-23T15:28:49.949075+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":381,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296454+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:28:48.296454+0000","last_peered":"2023-09-23T15:28:48.296454+0000","last_clean":"2023-09-23T15:28:48.296454+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:28:48.296454+0000","last_undegraded":"2023-09-23T15:28:48.296454+0000","last_fullsized":"2023-09-23T15:28:48.296454+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372798+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:28:51.372798+0000","last_peered":"2023-09-23T15:28:51.372798+0000","last_clean":"2023-09-23T15:28:51.372798+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:28:51.372798+0000","last_undegraded":"2023-09-23T15:28:51.372798+0000","last_fullsized":"2023-09-23T15:28:51.372798+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"405'8816","reported_seq":9554,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.782520+0000","last_change":"2023-09-23T15:28:25.337060+0000","last_active":"2023-09-23T15:28:54.782520+0000","last_peered":"2023-09-23T15:28:54.782520+0000","last_clean":"2023-09-23T15:28:54.782520+0000","last_became_active":"2023-09-23T15:28:25.321577+0000","last_became_peered":"2023-09-23T15:28:25.321577+0000","last_unstale":"2023-09-23T15:28:54.782520+0000","last_undegraded":"2023-09-23T15:28:54.782520+0000","last_fullsized":"2023-09-23T15:28:54.782520+0000","mapping_epoch":388,"log_start":"346'2858","ondisk_log_start":"346'2858","created":339,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5958,"log_dups_size":0,"ondisk_log_size":5958,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:42.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8816,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":313,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":657,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:52.028805+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:28:52.028805+0000","last_peered":"2023-09-23T15:28:52.028805+0000","last_clean":"2023-09-23T15:28:52.028805+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:28:52.028805+0000","last_undegraded":"2023-09-23T15:28:52.028805+0000","last_fullsized":"2023-09-23T15:28:52.028805+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":400,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298201+0000","last_change":"2023-09-23T15:27:23.323912+0000","last_active":"2023-09-23T15:28:48.298201+0000","last_peered":"2023-09-23T15:28:48.298201+0000","last_clean":"2023-09-23T15:28:48.298201+0000","last_became_active":"2023-09-23T15:27:23.316529+0000","last_became_peered":"2023-09-23T15:27:23.316529+0000","last_unstale":"2023-09-23T15:28:48.298201+0000","last_undegraded":"2023-09-23T15:28:48.298201+0000","last_fullsized":"2023-09-23T15:28:48.298201+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:55.689850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298029+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:28:48.298029+0000","last_peered":"2023-09-23T15:28:48.298029+0000","last_clean":"2023-09-23T15:28:48.298029+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:28:48.298029+0000","last_undegraded":"2023-09-23T15:28:48.298029+0000","last_fullsized":"2023-09-23T15:28:48.298029+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"405'8941","reported_seq":9017,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:55.604818+0000","last_change":"2023-09-23T15:25:35.722919+0000","last_active":"2023-09-23T15:28:55.604818+0000","last_peered":"2023-09-23T15:28:55.604818+0000","last_clean":"2023-09-23T15:28:55.604818+0000","last_became_active":"2023-09-23T15:25:35.722667+0000","last_became_peered":"2023-09-23T15:25:35.722667+0000","last_unstale":"2023-09-23T15:28:55.604818+0000","last_undegraded":"2023-09-23T15:28:55.604818+0000","last_fullsized":"2023-09-23T15:28:55.604818+0000","mapping_epoch":339,"log_start":"346'2719","ondisk_log_start":"346'2719","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6222,"log_dups_size":0,"ondisk_log_size":6222,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:51.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111869952,"num_objects":1707,"num_object_clones":0,"num_object_copies":5121,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1707,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8941,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":729,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294852+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:28:48.294852+0000","last_peered":"2023-09-23T15:28:48.294852+0000","last_clean":"2023-09-23T15:28:48.294852+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:28:48.294852+0000","last_undegraded":"2023-09-23T15:28:48.294852+0000","last_fullsized":"2023-09-23T15:28:48.294852+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":372,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296269+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:28:48.296269+0000","last_peered":"2023-09-23T15:28:48.296269+0000","last_clean":"2023-09-23T15:28:48.296269+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:28:48.296269+0000","last_undegraded":"2023-09-23T15:28:48.296269+0000","last_fullsized":"2023-09-23T15:28:48.296269+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298421+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:28:48.298421+0000","last_peered":"2023-09-23T15:28:48.298421+0000","last_clean":"2023-09-23T15:28:48.298421+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:28:48.298421+0000","last_undegraded":"2023-09-23T15:28:48.298421+0000","last_fullsized":"2023-09-23T15:28:48.298421+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"405'8656","reported_seq":9098,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.400604+0000","last_change":"2023-09-23T15:28:30.734365+0000","last_active":"2023-09-23T15:28:54.400604+0000","last_peered":"2023-09-23T15:28:54.400604+0000","last_clean":"2023-09-23T15:28:54.400604+0000","last_became_active":"2023-09-23T15:28:30.700318+0000","last_became_peered":"2023-09-23T15:28:30.700318+0000","last_unstale":"2023-09-23T15:28:54.400604+0000","last_undegraded":"2023-09-23T15:28:54.400604+0000","last_fullsized":"2023-09-23T15:28:54.400604+0000","mapping_epoch":393,"log_start":"346'2856","ondisk_log_start":"346'2856","created":339,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5800,"log_dups_size":0,"ondisk_log_size":5800,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:11.270292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114950144,"num_objects":1754,"num_object_clones":0,"num_object_copies":5262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8656,"num_write_kb":333120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":304,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":684,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:50.442371+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:28:50.442371+0000","last_peered":"2023-09-23T15:28:50.442371+0000","last_clean":"2023-09-23T15:28:50.442371+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:28:50.442371+0000","last_undegraded":"2023-09-23T15:28:50.442371+0000","last_fullsized":"2023-09-23T15:28:50.442371+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":61,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294398+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:28:48.294398+0000","last_peered":"2023-09-23T15:28:48.294398+0000","last_clean":"2023-09-23T15:28:48.294398+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:28:48.294398+0000","last_undegraded":"2023-09-23T15:28:48.294398+0000","last_fullsized":"2023-09-23T15:28:48.294398+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298903+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:28:48.298903+0000","last_peered":"2023-09-23T15:28:48.298903+0000","last_clean":"2023-09-23T15:28:48.298903+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:28:48.298903+0000","last_undegraded":"2023-09-23T15:28:48.298903+0000","last_fullsized":"2023-09-23T15:28:48.298903+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"405'8736","reported_seq":8818,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.768536+0000","last_change":"2023-09-23T15:27:12.390259+0000","last_active":"2023-09-23T15:28:54.768536+0000","last_peered":"2023-09-23T15:28:54.768536+0000","last_clean":"2023-09-23T15:28:54.768536+0000","last_became_active":"2023-09-23T15:27:12.384140+0000","last_became_peered":"2023-09-23T15:27:12.384140+0000","last_unstale":"2023-09-23T15:28:54.768536+0000","last_undegraded":"2023-09-23T15:28:54.768536+0000","last_fullsized":"2023-09-23T15:28:54.768536+0000","mapping_epoch":363,"log_start":"346'2784","ondisk_log_start":"346'2784","created":339,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5952,"log_dups_size":0,"ondisk_log_size":5952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:31.985761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117047296,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8736,"num_write_kb":336704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":741,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.299036+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:28:48.299036+0000","last_peered":"2023-09-23T15:28:48.299036+0000","last_clean":"2023-09-23T15:28:48.299036+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:28:48.299036+0000","last_undegraded":"2023-09-23T15:28:48.299036+0000","last_fullsized":"2023-09-23T15:28:48.299036+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":414,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298002+0000","last_change":"2023-09-23T15:27:41.418214+0000","last_active":"2023-09-23T15:28:48.298002+0000","last_peered":"2023-09-23T15:28:48.298002+0000","last_clean":"2023-09-23T15:28:48.298002+0000","last_became_active":"2023-09-23T15:27:41.417924+0000","last_became_peered":"2023-09-23T15:27:41.417924+0000","last_unstale":"2023-09-23T15:28:48.298002+0000","last_undegraded":"2023-09-23T15:28:48.298002+0000","last_fullsized":"2023-09-23T15:28:48.298002+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:45.122620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":339,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.315872+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:28:48.315872+0000","last_peered":"2023-09-23T15:28:48.315872+0000","last_clean":"2023-09-23T15:28:48.315872+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:28:48.315872+0000","last_undegraded":"2023-09-23T15:28:48.315872+0000","last_fullsized":"2023-09-23T15:28:48.315872+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"405'8882","reported_seq":8956,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.954067+0000","last_change":"2023-09-23T15:25:35.716554+0000","last_active":"2023-09-23T15:28:54.954067+0000","last_peered":"2023-09-23T15:28:54.954067+0000","last_clean":"2023-09-23T15:28:54.954067+0000","last_became_active":"2023-09-23T15:25:35.716257+0000","last_became_peered":"2023-09-23T15:25:35.716257+0000","last_unstale":"2023-09-23T15:28:54.954067+0000","last_undegraded":"2023-09-23T15:28:54.954067+0000","last_fullsized":"2023-09-23T15:28:54.954067+0000","mapping_epoch":339,"log_start":"345'1882","ondisk_log_start":"345'1882","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:37.096030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113508352,"num_objects":1732,"num_object_clones":0,"num_object_copies":5196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8882,"num_write_kb":339648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":645,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:49.592974+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:28:49.592974+0000","last_peered":"2023-09-23T15:28:49.592974+0000","last_clean":"2023-09-23T15:28:49.592974+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:28:49.592974+0000","last_undegraded":"2023-09-23T15:28:49.592974+0000","last_fullsized":"2023-09-23T15:28:49.592974+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":621,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.314724+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:28:48.314724+0000","last_peered":"2023-09-23T15:28:48.314724+0000","last_clean":"2023-09-23T15:28:48.314724+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:28:48.314724+0000","last_undegraded":"2023-09-23T15:28:48.314724+0000","last_fullsized":"2023-09-23T15:28:48.314724+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":381,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295637+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:28:48.295637+0000","last_peered":"2023-09-23T15:28:48.295637+0000","last_clean":"2023-09-23T15:28:48.295637+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:28:48.295637+0000","last_undegraded":"2023-09-23T15:28:48.295637+0000","last_fullsized":"2023-09-23T15:28:48.295637+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":369,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303755+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:28:48.303755+0000","last_peered":"2023-09-23T15:28:48.303755+0000","last_clean":"2023-09-23T15:28:48.303755+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:28:48.303755+0000","last_undegraded":"2023-09-23T15:28:48.303755+0000","last_fullsized":"2023-09-23T15:28:48.303755+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"406'8905","reported_seq":9678,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:58.725285+0000","last_change":"2023-09-23T15:28:29.680984+0000","last_active":"2023-09-23T15:28:58.725285+0000","last_peered":"2023-09-23T15:28:58.725285+0000","last_clean":"2023-09-23T15:28:58.725285+0000","last_became_active":"2023-09-23T15:28:29.667811+0000","last_became_peered":"2023-09-23T15:28:29.667811+0000","last_unstale":"2023-09-23T15:28:58.725285+0000","last_undegraded":"2023-09-23T15:28:58.725285+0000","last_fullsized":"2023-09-23T15:28:58.725285+0000","mapping_epoch":392,"log_start":"345'2056","ondisk_log_start":"345'2056","created":339,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6849,"log_dups_size":0,"ondisk_log_size":6849,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:29:51.355536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8905,"num_write_kb":335360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":702,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294333+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:28:48.294333+0000","last_peered":"2023-09-23T15:28:48.294333+0000","last_clean":"2023-09-23T15:28:48.294333+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:28:48.294333+0000","last_undegraded":"2023-09-23T15:28:48.294333+0000","last_fullsized":"2023-09-23T15:28:48.294333+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":940,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.299274+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:28:48.299274+0000","last_peered":"2023-09-23T15:28:48.299274+0000","last_clean":"2023-09-23T15:28:48.299274+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:28:48.299274+0000","last_undegraded":"2023-09-23T15:28:48.299274+0000","last_fullsized":"2023-09-23T15:28:48.299274+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.297326+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:28:48.297326+0000","last_peered":"2023-09-23T15:28:48.297326+0000","last_clean":"2023-09-23T15:28:48.297326+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:28:48.297326+0000","last_undegraded":"2023-09-23T15:28:48.297326+0000","last_fullsized":"2023-09-23T15:28:48.297326+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":303,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.314401+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:28:48.314401+0000","last_peered":"2023-09-23T15:28:48.314401+0000","last_clean":"2023-09-23T15:28:48.314401+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:28:48.314401+0000","last_undegraded":"2023-09-23T15:28:48.314401+0000","last_fullsized":"2023-09-23T15:28:48.314401+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"405'8678","reported_seq":9371,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.929393+0000","last_change":"2023-09-23T15:28:23.357956+0000","last_active":"2023-09-23T15:28:54.929393+0000","last_peered":"2023-09-23T15:28:54.929393+0000","last_clean":"2023-09-23T15:28:54.929393+0000","last_became_active":"2023-09-23T15:28:23.286114+0000","last_became_peered":"2023-09-23T15:28:23.286114+0000","last_unstale":"2023-09-23T15:28:54.929393+0000","last_undegraded":"2023-09-23T15:28:54.929393+0000","last_fullsized":"2023-09-23T15:28:54.929393+0000","mapping_epoch":386,"log_start":"345'1667","ondisk_log_start":"345'1667","created":339,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:16.750949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8678,"num_write_kb":331968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":768,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.302515+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:28:48.302515+0000","last_peered":"2023-09-23T15:28:48.302515+0000","last_clean":"2023-09-23T15:28:48.302515+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:28:48.302515+0000","last_undegraded":"2023-09-23T15:28:48.302515+0000","last_fullsized":"2023-09-23T15:28:48.302515+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1024,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303588+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:28:48.303588+0000","last_peered":"2023-09-23T15:28:48.303588+0000","last_clean":"2023-09-23T15:28:48.303588+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:28:48.303588+0000","last_undegraded":"2023-09-23T15:28:48.303588+0000","last_fullsized":"2023-09-23T15:28:48.303588+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":299,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:50.659982+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:28:50.659982+0000","last_peered":"2023-09-23T15:28:50.659982+0000","last_clean":"2023-09-23T15:28:50.659982+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:28:50.659982+0000","last_undegraded":"2023-09-23T15:28:50.659982+0000","last_fullsized":"2023-09-23T15:28:50.659982+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":307,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.315842+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:28:48.315842+0000","last_peered":"2023-09-23T15:28:48.315842+0000","last_clean":"2023-09-23T15:28:48.315842+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:28:48.315842+0000","last_undegraded":"2023-09-23T15:28:48.315842+0000","last_fullsized":"2023-09-23T15:28:48.315842+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"405'8714","reported_seq":8796,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.758545+0000","last_change":"2023-09-23T15:27:12.391982+0000","last_active":"2023-09-23T15:28:54.758545+0000","last_peered":"2023-09-23T15:28:54.758545+0000","last_clean":"2023-09-23T15:28:54.758545+0000","last_became_active":"2023-09-23T15:27:12.386967+0000","last_became_peered":"2023-09-23T15:27:12.386967+0000","last_unstale":"2023-09-23T15:28:54.758545+0000","last_undegraded":"2023-09-23T15:28:54.758545+0000","last_fullsized":"2023-09-23T15:28:54.758545+0000","mapping_epoch":363,"log_start":"346'2796","ondisk_log_start":"346'2796","created":339,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5918,"log_dups_size":0,"ondisk_log_size":5918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:32.675846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8714,"num_write_kb":334912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":751,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296241+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:28:48.296241+0000","last_peered":"2023-09-23T15:28:48.296241+0000","last_clean":"2023-09-23T15:28:48.296241+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:28:48.296241+0000","last_undegraded":"2023-09-23T15:28:48.296241+0000","last_fullsized":"2023-09-23T15:28:48.296241+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":711,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.300733+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:28:48.300733+0000","last_peered":"2023-09-23T15:28:48.300733+0000","last_clean":"2023-09-23T15:28:48.300733+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:28:48.300733+0000","last_undegraded":"2023-09-23T15:28:48.300733+0000","last_fullsized":"2023-09-23T15:28:48.300733+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":328,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.299114+0000","last_change":"2023-09-23T15:27:12.392661+0000","last_active":"2023-09-23T15:28:48.299114+0000","last_peered":"2023-09-23T15:28:48.299114+0000","last_clean":"2023-09-23T15:28:48.299114+0000","last_became_active":"2023-09-23T15:27:12.389493+0000","last_became_peered":"2023-09-23T15:27:12.389493+0000","last_unstale":"2023-09-23T15:28:48.299114+0000","last_undegraded":"2023-09-23T15:28:48.299114+0000","last_fullsized":"2023-09-23T15:28:48.299114+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:49.162226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":397,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296611+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:28:48.296611+0000","last_peered":"2023-09-23T15:28:48.296611+0000","last_clean":"2023-09-23T15:28:48.296611+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:28:48.296611+0000","last_undegraded":"2023-09-23T15:28:48.296611+0000","last_fullsized":"2023-09-23T15:28:48.296611+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"406'9024","reported_seq":9101,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:58.725477+0000","last_change":"2023-09-23T15:25:35.708972+0000","last_active":"2023-09-23T15:28:58.725477+0000","last_peered":"2023-09-23T15:28:58.725477+0000","last_clean":"2023-09-23T15:28:58.725477+0000","last_became_active":"2023-09-23T15:25:35.708745+0000","last_became_peered":"2023-09-23T15:25:35.708745+0000","last_unstale":"2023-09-23T15:28:58.725477+0000","last_undegraded":"2023-09-23T15:28:58.725477+0000","last_fullsized":"2023-09-23T15:28:58.725477+0000","mapping_epoch":339,"log_start":"345'2200","ondisk_log_start":"345'2200","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6824,"log_dups_size":0,"ondisk_log_size":6824,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:56.572163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103677952,"num_objects":1582,"num_object_clones":0,"num_object_copies":4746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9024,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":747,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.311648+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:28:48.311648+0000","last_peered":"2023-09-23T15:28:48.311648+0000","last_clean":"2023-09-23T15:28:48.311648+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:28:48.311648+0000","last_undegraded":"2023-09-23T15:28:48.311648+0000","last_fullsized":"2023-09-23T15:28:48.311648+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":118,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.300603+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:28:48.300603+0000","last_peered":"2023-09-23T15:28:48.300603+0000","last_clean":"2023-09-23T15:28:48.300603+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:28:48.300603+0000","last_undegraded":"2023-09-23T15:28:48.300603+0000","last_fullsized":"2023-09-23T15:28:48.300603+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":393,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437127+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:28:51.437127+0000","last_peered":"2023-09-23T15:28:51.437127+0000","last_clean":"2023-09-23T15:28:51.437127+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:28:51.437127+0000","last_undegraded":"2023-09-23T15:28:51.437127+0000","last_fullsized":"2023-09-23T15:28:51.437127+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":387,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373225+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:28:51.373225+0000","last_peered":"2023-09-23T15:28:51.373225+0000","last_clean":"2023-09-23T15:28:51.373225+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:28:51.373225+0000","last_undegraded":"2023-09-23T15:28:51.373225+0000","last_fullsized":"2023-09-23T15:28:51.373225+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"405'8584","reported_seq":19243,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.417673+0000","last_change":"2023-09-23T15:28:26.307366+0000","last_active":"2023-09-23T15:28:54.417673+0000","last_peered":"2023-09-23T15:28:54.417673+0000","last_clean":"2023-09-23T15:28:54.417673+0000","last_became_active":"2023-09-23T15:28:26.291301+0000","last_became_peered":"2023-09-23T15:28:26.291301+0000","last_unstale":"2023-09-23T15:28:54.417673+0000","last_undegraded":"2023-09-23T15:28:54.417673+0000","last_fullsized":"2023-09-23T15:28:54.417673+0000","mapping_epoch":389,"log_start":"346'2757","ondisk_log_start":"346'2757","created":339,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5827,"log_dups_size":0,"ondisk_log_size":5827,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:13.182062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114819072,"num_objects":1752,"num_object_clones":0,"num_object_copies":5256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8584,"num_write_kb":330752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5269,"num_bytes_recovered":333905920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":731,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.566404+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:28:48.566404+0000","last_peered":"2023-09-23T15:28:48.566404+0000","last_clean":"2023-09-23T15:28:48.566404+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:28:48.566404+0000","last_undegraded":"2023-09-23T15:28:48.566404+0000","last_fullsized":"2023-09-23T15:28:48.566404+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1144,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303471+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:28:48.303471+0000","last_peered":"2023-09-23T15:28:48.303471+0000","last_clean":"2023-09-23T15:28:48.303471+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:28:48.303471+0000","last_undegraded":"2023-09-23T15:28:48.303471+0000","last_fullsized":"2023-09-23T15:28:48.303471+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":283,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.314593+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:28:48.314593+0000","last_peered":"2023-09-23T15:28:48.314593+0000","last_clean":"2023-09-23T15:28:48.314593+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:28:48.314593+0000","last_undegraded":"2023-09-23T15:28:48.314593+0000","last_fullsized":"2023-09-23T15:28:48.314593+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":386,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373010+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:28:51.373010+0000","last_peered":"2023-09-23T15:28:51.373010+0000","last_clean":"2023-09-23T15:28:51.373010+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:28:51.373010+0000","last_undegraded":"2023-09-23T15:28:51.373010+0000","last_fullsized":"2023-09-23T15:28:51.373010+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"405'8748","reported_seq":9401,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.401239+0000","last_change":"2023-09-23T15:28:24.301029+0000","last_active":"2023-09-23T15:28:54.401239+0000","last_peered":"2023-09-23T15:28:54.401239+0000","last_clean":"2023-09-23T15:28:54.401239+0000","last_became_active":"2023-09-23T15:28:24.286192+0000","last_became_peered":"2023-09-23T15:28:24.286192+0000","last_unstale":"2023-09-23T15:28:54.401239+0000","last_undegraded":"2023-09-23T15:28:54.401239+0000","last_fullsized":"2023-09-23T15:28:54.401239+0000","mapping_epoch":387,"log_start":"349'2947","ondisk_log_start":"349'2947","created":339,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5801,"log_dups_size":0,"ondisk_log_size":5801,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:33.615267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112132120,"num_objects":1712,"num_object_clones":0,"num_object_copies":5136,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1712,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8748,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":300,"num_bytes_recovered":3014680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":696,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298546+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:28:48.298546+0000","last_peered":"2023-09-23T15:28:48.298546+0000","last_clean":"2023-09-23T15:28:48.298546+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:28:48.298546+0000","last_undegraded":"2023-09-23T15:28:48.298546+0000","last_fullsized":"2023-09-23T15:28:48.298546+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":177,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.304162+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:28:48.304162+0000","last_peered":"2023-09-23T15:28:48.304162+0000","last_clean":"2023-09-23T15:28:48.304162+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:28:48.304162+0000","last_undegraded":"2023-09-23T15:28:48.304162+0000","last_fullsized":"2023-09-23T15:28:48.304162+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1038,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.316156+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:28:48.316156+0000","last_peered":"2023-09-23T15:28:48.316156+0000","last_clean":"2023-09-23T15:28:48.316156+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:28:48.316156+0000","last_undegraded":"2023-09-23T15:28:48.316156+0000","last_fullsized":"2023-09-23T15:28:48.316156+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":394,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298098+0000","last_change":"2023-09-23T15:27:41.419984+0000","last_active":"2023-09-23T15:28:48.298098+0000","last_peered":"2023-09-23T15:28:48.298098+0000","last_clean":"2023-09-23T15:28:48.298098+0000","last_became_active":"2023-09-23T15:27:41.419701+0000","last_became_peered":"2023-09-23T15:27:41.419701+0000","last_unstale":"2023-09-23T15:28:48.298098+0000","last_undegraded":"2023-09-23T15:28:48.298098+0000","last_fullsized":"2023-09-23T15:28:48.298098+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:55.086471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"405'8714","reported_seq":14403,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:54.784289+0000","last_change":"2023-09-23T15:28:27.558580+0000","last_active":"2023-09-23T15:28:54.784289+0000","last_peered":"2023-09-23T15:28:54.784289+0000","last_clean":"2023-09-23T15:28:54.784289+0000","last_became_active":"2023-09-23T15:28:22.658649+0000","last_became_peered":"2023-09-23T15:28:22.658649+0000","last_unstale":"2023-09-23T15:28:54.784289+0000","last_undegraded":"2023-09-23T15:28:54.784289+0000","last_fullsized":"2023-09-23T15:28:54.784289+0000","mapping_epoch":385,"log_start":"346'2812","ondisk_log_start":"346'2812","created":339,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5902,"log_dups_size":0,"ondisk_log_size":5902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:46.751262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112590848,"num_objects":1718,"num_object_clones":0,"num_object_copies":5154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8714,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2785,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":724,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298640+0000","last_change":"2023-09-23T15:27:41.438036+0000","last_active":"2023-09-23T15:28:48.298640+0000","last_peered":"2023-09-23T15:28:48.298640+0000","last_clean":"2023-09-23T15:28:48.298640+0000","last_became_active":"2023-09-23T15:27:41.437722+0000","last_became_peered":"2023-09-23T15:27:41.437722+0000","last_unstale":"2023-09-23T15:28:48.298640+0000","last_undegraded":"2023-09-23T15:28:48.298640+0000","last_fullsized":"2023-09-23T15:28:48.298640+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:37.744292+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":858,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295095+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:28:48.295095+0000","last_peered":"2023-09-23T15:28:48.295095+0000","last_clean":"2023-09-23T15:28:48.295095+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:28:48.295095+0000","last_undegraded":"2023-09-23T15:28:48.295095+0000","last_fullsized":"2023-09-23T15:28:48.295095+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":446,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373317+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:28:51.373317+0000","last_peered":"2023-09-23T15:28:51.373317+0000","last_clean":"2023-09-23T15:28:51.373317+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:28:51.373317+0000","last_undegraded":"2023-09-23T15:28:51.373317+0000","last_fullsized":"2023-09-23T15:28:51.373317+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294464+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:28:48.294464+0000","last_peered":"2023-09-23T15:28:48.294464+0000","last_clean":"2023-09-23T15:28:48.294464+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:28:48.294464+0000","last_undegraded":"2023-09-23T15:28:48.294464+0000","last_fullsized":"2023-09-23T15:28:48.294464+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":639,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:50.816286+0000","last_change":"2023-09-23T15:28:50.816286+0000","last_active":"2023-09-23T15:28:50.816286+0000","last_peered":"2023-09-23T15:28:50.816286+0000","last_clean":"2023-09-23T15:28:50.816286+0000","last_became_active":"2023-09-23T15:28:39.666756+0000","last_became_peered":"2023-09-23T15:28:39.666756+0000","last_unstale":"2023-09-23T15:28:50.816286+0000","last_undegraded":"2023-09-23T15:28:50.816286+0000","last_fullsized":"2023-09-23T15:28:50.816286+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.816141+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.816141+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:36.466168+0000","scrub_duration":0.017140743999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":425,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372937+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:28:51.372937+0000","last_peered":"2023-09-23T15:28:51.372937+0000","last_clean":"2023-09-23T15:28:51.372937+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:28:51.372937+0000","last_undegraded":"2023-09-23T15:28:51.372937+0000","last_fullsized":"2023-09-23T15:28:51.372937+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":410,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.303784+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:28:48.303784+0000","last_peered":"2023-09-23T15:28:48.303784+0000","last_clean":"2023-09-23T15:28:48.303784+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:28:48.303784+0000","last_undegraded":"2023-09-23T15:28:48.303784+0000","last_fullsized":"2023-09-23T15:28:48.303784+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":608,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.796791+0000","last_change":"2023-09-23T15:28:51.796932+0000","last_active":"2023-09-23T15:28:51.796791+0000","last_peered":"2023-09-23T15:28:51.796791+0000","last_clean":"2023-09-23T15:28:51.796791+0000","last_became_active":"2023-09-23T15:28:39.656548+0000","last_became_peered":"2023-09-23T15:28:39.656548+0000","last_unstale":"2023-09-23T15:28:51.796791+0000","last_undegraded":"2023-09-23T15:28:51.796791+0000","last_fullsized":"2023-09-23T15:28:51.796791+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.796761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.796761+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:04:26.204960+0000","scrub_duration":0.011035615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":446,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.296506+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:28:48.296506+0000","last_peered":"2023-09-23T15:28:48.296506+0000","last_clean":"2023-09-23T15:28:48.296506+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:28:48.296506+0000","last_undegraded":"2023-09-23T15:28:48.296506+0000","last_fullsized":"2023-09-23T15:28:48.296506+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.297407+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:28:48.297407+0000","last_peered":"2023-09-23T15:28:48.297407+0000","last_clean":"2023-09-23T15:28:48.297407+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:28:48.297407+0000","last_undegraded":"2023-09-23T15:28:48.297407+0000","last_fullsized":"2023-09-23T15:28:48.297407+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":426,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.295552+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:28:48.295552+0000","last_peered":"2023-09-23T15:28:48.295552+0000","last_clean":"2023-09-23T15:28:48.295552+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:28:48.295552+0000","last_undegraded":"2023-09-23T15:28:48.295552+0000","last_fullsized":"2023-09-23T15:28:48.295552+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294667+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:28:48.294667+0000","last_peered":"2023-09-23T15:28:48.294667+0000","last_clean":"2023-09-23T15:28:48.294667+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:28:48.294667+0000","last_undegraded":"2023-09-23T15:28:48.294667+0000","last_fullsized":"2023-09-23T15:28:48.294667+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":582,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.302658+0000","last_change":"2023-09-23T15:28:39.659554+0000","last_active":"2023-09-23T15:28:48.302658+0000","last_peered":"2023-09-23T15:28:48.302658+0000","last_clean":"2023-09-23T15:28:48.302658+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:28:48.302658+0000","last_undegraded":"2023-09-23T15:28:48.302658+0000","last_fullsized":"2023-09-23T15:28:48.302658+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:25:48.079935+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:25:48.079935+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0050498640000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":385,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.307769+0000","last_change":"2023-09-23T15:27:12.384840+0000","last_active":"2023-09-23T15:28:48.307769+0000","last_peered":"2023-09-23T15:28:48.307769+0000","last_clean":"2023-09-23T15:28:48.307769+0000","last_became_active":"2023-09-23T15:27:12.384518+0000","last_became_peered":"2023-09-23T15:27:12.384518+0000","last_unstale":"2023-09-23T15:28:48.307769+0000","last_undegraded":"2023-09-23T15:28:48.307769+0000","last_fullsized":"2023-09-23T15:28:48.307769+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:18.377550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":384,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.298724+0000","last_change":"2023-09-23T15:27:12.385884+0000","last_active":"2023-09-23T15:28:48.298724+0000","last_peered":"2023-09-23T15:28:48.298724+0000","last_clean":"2023-09-23T15:28:48.298724+0000","last_became_active":"2023-09-23T15:27:12.385612+0000","last_became_peered":"2023-09-23T15:27:12.385612+0000","last_unstale":"2023-09-23T15:28:48.298724+0000","last_undegraded":"2023-09-23T15:28:48.298724+0000","last_fullsized":"2023-09-23T15:28:48.298724+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:50.641495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":572,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.315573+0000","last_change":"2023-09-23T15:28:39.657108+0000","last_active":"2023-09-23T15:28:48.315573+0000","last_peered":"2023-09-23T15:28:48.315573+0000","last_clean":"2023-09-23T15:28:48.315573+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:28:48.315573+0000","last_undegraded":"2023-09-23T15:28:48.315573+0000","last_fullsized":"2023-09-23T15:28:48.315573+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:20:12.948130+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:20:12.948130+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.001670057,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437035+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:28:51.437035+0000","last_peered":"2023-09-23T15:28:51.437035+0000","last_clean":"2023-09-23T15:28:51.437035+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:28:51.437035+0000","last_undegraded":"2023-09-23T15:28:51.437035+0000","last_fullsized":"2023-09-23T15:28:51.437035+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":47,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T15:28:48.294063+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:28:48.294063+0000","last_peered":"2023-09-23T15:28:48.294063+0000","last_clean":"2023-09-23T15:28:48.294063+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:28:48.294063+0000","last_undegraded":"2023-09-23T15:28:48.294063+0000","last_fullsized":"2023-09-23T15:28:48.294063+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":776571,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":19,"num_read_kb":35,"num_write":10,"num_write_kb":92,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":329775,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1769013272,"num_objects":26994,"num_object_clones":0,"num_object_copies":80982,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26994,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":140584,"num_write_kb":5362433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12970,"num_bytes_recovered":701300760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5851590656,"data_stored":5851578440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102069,"ondisk_log_size":102069,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768720,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1104196,"kb_used_data":1015484,"kb_used_omap":13,"kb_used_meta":88690,"kb_avail":92640956,"statfs":{"total":95995035648,"available":94864338944,"internally_reserved":0,"allocated":1039855616,"data_stored":1038549392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":90818654},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":5.7919999999999998,"5min":5.7919999999999998,"15min":5.7919999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":36.941000000000003,"5min":36.941000000000003,"15min":36.941000000000003},"last":0.27400000000000002},{"interface":"front","average":{"1min":5.9080000000000004,"5min":5.9080000000000004,"15min":5.9080000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":36.904000000000003,"5min":36.904000000000003,"15min":36.904000000000003},"last":1.081}]},{"osd":1,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":21.283000000000001,"5min":21.283000000000001,"15min":21.283000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":157.096,"5min":157.096,"15min":157.096},"last":1.1539999999999999},{"interface":"front","average":{"1min":21.431999999999999,"5min":21.431999999999999,"15min":21.431999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":157.011,"5min":157.011,"15min":157.011},"last":1.169}]},{"osd":2,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.8050000000000002,"5min":2.8050000000000002,"15min":2.8050000000000002},"last":1.228},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.109,"5min":4.109,"15min":4.109},"last":1.1020000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0709999999999997,"5min":4.0709999999999997,"15min":4.0709999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":32.847999999999999,"5min":32.847999999999999,"15min":32.847999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":4.4809999999999999,"5min":4.4809999999999999,"15min":4.4809999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":33.064999999999998,"5min":33.064999999999998,"15min":33.064999999999998},"last":1.1830000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.601,"5min":1.601,"15min":1.601},"last":0.316},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":1.21}]},{"osd":5,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.4279999999999999,"5min":3.4279999999999999,"15min":3.4279999999999999},"last":1.0289999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.2389999999999999,"5min":2.2389999999999999,"15min":2.2389999999999999},"last":0.375}]},{"osd":6,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.5390000000000001,"5min":3.5390000000000001,"15min":3.5390000000000001},"last":1.133},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.6619999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.39900000000000002}]}]},{"osd":6,"up_from":304,"seq":1305670058063,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":462772,"kb_used_data":340972,"kb_used_omap":13,"kb_used_meta":121778,"kb_avail":93282380,"statfs":{"total":95995035648,"available":95521157120,"internally_reserved":0,"allocated":349155328,"data_stored":347864793,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":124701027},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":4.7110000000000003,"15min":2.5569999999999999},"min":{"1min":0.27500000000000002,"5min":0.224,"15min":0.224},"max":{"1min":9.5350000000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":2.8279999999999998},{"interface":"front","average":{"1min":1.583,"5min":3.3079999999999998,"15min":3.0209999999999999},"min":{"1min":0.29299999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.4329999999999998,"5min":86.227999999999994,"15min":86.227999999999994},"last":1.9610000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":3.1779999999999999,"15min":2.887},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":9.2910000000000004,"5min":86.534000000000006,"15min":86.534000000000006},"last":1.2350000000000001},{"interface":"front","average":{"1min":1.998,"5min":2.8420000000000001,"15min":2.855},"min":{"1min":0.28000000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.5419999999999998,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.624}]},{"osd":2,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.99,"15min":1.6799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.2379999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":1.2669999999999999},{"interface":"front","average":{"1min":1.9399999999999999,"5min":1.9670000000000001,"15min":1.8069999999999999},"min":{"1min":0.38100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":9.3650000000000002,"5min":9.3650000000000002,"15min":9.3650000000000002},"last":1.1819999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":10.295999999999999,"5min":3.742,"15min":2.7669999999999999},"min":{"1min":0.35499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":142.67599999999999,"5min":142.67599999999999,"15min":142.67599999999999},"last":1.04},{"interface":"front","average":{"1min":10.250999999999999,"5min":4.1340000000000003,"15min":2.8919999999999999},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":142.541,"5min":142.541,"15min":142.541},"last":1.296}]},{"osd":4,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.075,"15min":0.98399999999999999},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6479999999999997,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.036,"5min":1.391,"15min":1.677},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.0140000000000002,"5min":13.627000000000001,"15min":13.627000000000001},"last":0.47299999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":8.3190000000000008,"5min":3.4620000000000002,"15min":2.298},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":130.91300000000001,"5min":130.91300000000001,"15min":130.91300000000001},"last":2.3570000000000002},{"interface":"front","average":{"1min":8.4179999999999993,"5min":3.0030000000000001,"15min":1.99},"min":{"1min":0.36599999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":130.92599999999999,"5min":130.92599999999999,"15min":130.92599999999999},"last":1.7170000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3209999999999997,"5min":9.3209999999999997,"15min":9.3209999999999997},"last":1.2090000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.5709999999999997,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":1.0089999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015364,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1550564,"kb_used_data":985756,"kb_used_omap":6,"kb_used_meta":564793,"kb_avail":92194588,"statfs":{"total":95995035648,"available":94407258112,"internally_reserved":0,"allocated":1009414144,"data_stored":1007400707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":578348107},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":5.2649999999999997,"5min":2.6059999999999999,"15min":3.1059999999999999},"min":{"1min":0.24199999999999999,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":57.329000000000001,"5min":57.329000000000001,"15min":60.817999999999998},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.905,"5min":2.8500000000000001,"15min":3.6160000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":7.8200000000000003,"5min":17.128,"15min":63.594999999999999},"last":0.83699999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":3.0499999999999998,"15min":3.7959999999999998},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":54.091999999999999,"5min":54.091999999999999,"15min":109.375},"last":3.9119999999999999},{"interface":"front","average":{"1min":6.0119999999999996,"5min":2.5289999999999999,"15min":3.157},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":57.279000000000003,"5min":57.279000000000003,"15min":105.16},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":2.319,"15min":5.4379999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":15.191000000000001,"5min":15.191000000000001,"15min":211.405},"last":1.6459999999999999},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.3199999999999998,"15min":5.6139999999999999},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.8949999999999996,"5min":15.637,"15min":210.988},"last":1.6719999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":8.4640000000000004,"5min":3.508,"15min":3.5640000000000001},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.29799999999999999},"max":{"1min":72.602000000000004,"5min":72.602000000000004,"15min":80.570999999999998},"last":3.371},{"interface":"front","average":{"1min":5.7869999999999999,"5min":2.7210000000000001,"15min":3.246},"min":{"1min":0.318,"5min":0.312,"15min":0.308},"max":{"1min":73.454999999999998,"5min":73.454999999999998,"15min":80.75},"last":0.501}]},{"osd":5,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":5.5880000000000001,"15min":3.669},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":5.6390000000000002,"5min":173.18799999999999,"15min":173.18799999999999},"last":1.4850000000000001},{"interface":"front","average":{"1min":4.4400000000000004,"5min":6.0170000000000003,"15min":3.9039999999999999},"min":{"1min":0.42499999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":54.167000000000002,"5min":172.822,"15min":172.822},"last":3.4020000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:28:37 2023","interfaces":[{"interface":"back","average":{"1min":4.8070000000000004,"5min":3.1000000000000001,"15min":3.0800000000000001},"min":{"1min":0.45500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":54.198,"5min":54.198,"15min":54.198},"last":3.4239999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.9710000000000001,"15min":2.2290000000000001},"min":{"1min":0.45500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":15.605,"15min":15.605},"last":0.80800000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":6.7309999999999999,"5min":6.7309999999999999,"15min":6.7309999999999999},"last":1.3200000000000001},{"interface":"front","average":{"1min":5.0730000000000004,"5min":5.0730000000000004,"15min":5.0730000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":54.223999999999997,"5min":54.223999999999997,"15min":54.223999999999997},"last":0.40899999999999997}]}]},{"osd":0,"up_from":95,"seq":408021893411,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1568980,"kb_used_data":889804,"kb_used_omap":8,"kb_used_meta":679159,"kb_avail":92176172,"statfs":{"total":95995035648,"available":94388400128,"internally_reserved":0,"allocated":911159296,"data_stored":909283876,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":695459412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:28:09 2023","interfaces":[{"interface":"back","average":{"1min":10.531000000000001,"5min":6.6459999999999999,"15min":4.3559999999999999},"min":{"1min":0.22,"5min":0.217,"15min":0.18099999999999999},"max":{"1min":141.52199999999999,"5min":141.52199999999999,"15min":141.52199999999999},"last":0.216},{"interface":"front","average":{"1min":11.266,"5min":6.79,"15min":4.1719999999999997},"min":{"1min":0.23799999999999999,"5min":0.223,"15min":0.20999999999999999},"max":{"1min":141.62799999999999,"5min":141.62799999999999,"15min":141.62799999999999},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.0699999999999998,"15min":3.5720000000000001},"min":{"1min":0.23400000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.968999999999999,"5min":14.994,"15min":93.873999999999995},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.7809999999999999,"15min":3.488},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":10.577,"5min":10.874000000000001,"15min":94.652000000000001},"last":0.27300000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:27:55 2023","interfaces":[{"interface":"back","average":{"1min":9.7850000000000001,"5min":6.444,"15min":6.2450000000000001},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":162.154,"5min":197.78899999999999,"15min":197.78899999999999},"last":80.415999999999997},{"interface":"front","average":{"1min":10.334,"5min":6.4459999999999997,"15min":6.9539999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":162.215,"5min":200.58799999999999,"15min":200.58799999999999},"last":80.463999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.5819999999999999,"5min":2.113,"15min":2.1339999999999999},"min":{"1min":0.40699999999999997,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":11.887,"5min":15.038,"15min":31.341999999999999},"last":0.438},{"interface":"front","average":{"1min":3.1970000000000001,"5min":2.2360000000000002,"15min":2.1480000000000001},"min":{"1min":0.32900000000000001,"5min":0.314,"15min":0.29399999999999998},"max":{"1min":13.656000000000001,"5min":13.656000000000001,"15min":31.059999999999999},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.5529999999999999,"15min":2.3370000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.489000000000001,"5min":44.57,"15min":64.349999999999994},"last":0.48599999999999999},{"interface":"front","average":{"1min":2.637,"5min":2.9380000000000002,"15min":2.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.312},"max":{"1min":9.798,"5min":44.588000000000001,"15min":44.588000000000001},"last":0.51400000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3100000000000001,"5min":2.5670000000000002,"15min":2.7290000000000001},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":11.974,"5min":32.097999999999999,"15min":32.097999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":4.0190000000000001,"5min":2.927,"15min":3.2160000000000002},"min":{"1min":0.42099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":14.563000000000001,"5min":32.188000000000002,"15min":32.188000000000002},"last":0.52100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.2949999999999999,"15min":2.2949999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.537000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":0.45100000000000001},{"interface":"front","average":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":12.651,"5min":12.651,"15min":12.651},"last":0.496}]}]},{"osd":2,"up_from":105,"seq":450971566360,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000184,"kb_used_data":349232,"kb_used_omap":11,"kb_used_meta":650932,"kb_avail":92744968,"statfs":{"total":95995035648,"available":94970847232,"internally_reserved":0,"allocated":357613568,"data_stored":355754603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":666555152},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":6.9180000000000001,"15min":5.8639999999999999},"min":{"1min":0.252,"5min":0.21199999999999999,"15min":0.17199999999999999},"max":{"1min":18.739999999999998,"5min":235.517,"15min":235.517},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.3759999999999999,"5min":6.8529999999999998,"15min":5.5759999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.8030000000000008,"5min":235.33799999999999,"15min":235.33799999999999},"last":0.58799999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:28:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0659999999999998,"15min":3.9300000000000002},"min":{"1min":0.28299999999999997,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":18.510000000000002,"5min":71.819000000000003,"15min":84.641000000000005},"last":1.724},{"interface":"front","average":{"1min":1.248,"5min":2.8490000000000002,"15min":3.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.24399999999999999,"15min":0.20899999999999999},"max":{"1min":5.7469999999999999,"5min":71.805000000000007,"15min":84.751999999999995},"last":0.63100000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:27:55 2023","interfaces":[{"interface":"back","average":{"1min":1.5920000000000001,"5min":3.4279999999999999,"15min":5.5149999999999997},"min":{"1min":0.255,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":18.699999999999999,"5min":77.031999999999996,"15min":87.108000000000004},"last":3.2469999999999999},{"interface":"front","average":{"1min":1.4970000000000001,"5min":3.4489999999999998,"15min":5.556},"min":{"1min":0.313,"5min":0.27200000000000002,"15min":0.24399999999999999},"max":{"1min":9.9909999999999997,"5min":76.742000000000004,"15min":86.158000000000001},"last":0.87}]},{"osd":4,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7290000000000001,"15min":2.097},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28000000000000003},"max":{"1min":40.119999999999997,"5min":40.119999999999997,"15min":84.608000000000004},"last":0.66100000000000003},{"interface":"front","average":{"1min":3.4630000000000001,"5min":1.716,"15min":2.4060000000000001},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":40.198,"5min":40.198,"15min":87.019999999999996},"last":0.5}]},{"osd":5,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":2.5070000000000001,"15min":2.6120000000000001},"min":{"1min":0.34799999999999998,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":9.1899999999999995,"5min":53.594999999999999,"15min":84.498000000000005},"last":0.90500000000000003},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.375,"15min":2.4460000000000002},"min":{"1min":0.36499999999999999,"5min":0.318,"15min":0.30399999999999999},"max":{"1min":8.7579999999999991,"5min":54.009999999999998,"15min":84.664000000000001},"last":1.0940000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:28:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.9039999999999999,"15min":2.157},"min":{"1min":0.41199999999999998,"5min":0.36699999999999999,"15min":0.35399999999999998},"max":{"1min":9.2289999999999992,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":0.93700000000000006},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.373,"15min":1.825},"min":{"1min":0.46600000000000003,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":2.4900000000000002,"15min":2.4900000000000002},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":9.3559999999999999,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":0.51800000000000002},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":10.964,"5min":10.964,"15min":10.964},"last":0.97399999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402832,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1593420,"kb_used_data":888004,"kb_used_omap":3,"kb_used_meta":705404,"kb_avail":92151732,"statfs":{"total":95995035648,"available":94363373568,"internally_reserved":0,"allocated":909316096,"data_stored":907439518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":722333865},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":4.7629999999999999,"15min":3.6619999999999999},"min":{"1min":0.223,"5min":0.20599999999999999,"15min":0.183},"max":{"1min":4.4539999999999997,"5min":108.318,"15min":108.318},"last":4.6440000000000001},{"interface":"front","average":{"1min":1.5089999999999999,"5min":5.2089999999999996,"15min":4.0110000000000001},"min":{"1min":0.23200000000000001,"5min":0.20399999999999999,"15min":0.184},"max":{"1min":17.117999999999999,"5min":108.889,"15min":108.889},"last":14.574999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:28:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9800000000000004,"5min":4.6260000000000003,"15min":4.9790000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":121.26900000000001,"5min":121.26900000000001,"15min":126.875},"last":10.702},{"interface":"front","average":{"1min":8.3140000000000001,"5min":4.899,"15min":5.25},"min":{"1min":0.25800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":121.012,"5min":121.012,"15min":128.249},"last":4.673}]},{"osd":2,"last update":"Sat Sep 23 15:28:00 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.879,"15min":2.8889999999999998},"min":{"1min":0.29599999999999999,"5min":0.26100000000000001,"15min":0.255},"max":{"1min":4.5199999999999996,"5min":14.853,"15min":61.732999999999997},"last":4.6980000000000004},{"interface":"front","average":{"1min":1.802,"5min":2.3100000000000001,"15min":3.7959999999999998},"min":{"1min":0.318,"5min":0.249,"15min":0.249},"max":{"1min":17.091000000000001,"5min":17.091000000000001,"15min":119.449},"last":14.343999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.5820000000000001,"15min":2.2400000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":17.027999999999999,"5min":17.027999999999999,"15min":54.051000000000002},"last":14.4},{"interface":"front","average":{"1min":0.874,"5min":1.8180000000000001,"15min":2.0979999999999999},"min":{"1min":0.34200000000000003,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.0919999999999996,"5min":13.115,"15min":52.228999999999999},"last":3.9060000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:28:00 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":2.9289999999999998,"15min":3.0070000000000001},"min":{"1min":0.36599999999999999,"5min":0.315,"15min":0.28100000000000003},"max":{"1min":16.986999999999998,"5min":82.793000000000006,"15min":148.619},"last":17.195},{"interface":"front","average":{"1min":0.88600000000000001,"5min":2.9540000000000002,"15min":2.8119999999999998},"min":{"1min":0.32200000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":5.1210000000000004,"5min":82.875,"15min":148.56999999999999},"last":3.867}]},{"osd":6,"last update":"Sat Sep 23 15:28:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":3.6749999999999998,"15min":2.7080000000000002},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.8440000000000003,"5min":84.665000000000006,"15min":84.665000000000006},"last":3.9380000000000002},{"interface":"front","average":{"1min":2.6150000000000002,"5min":3.5630000000000002,"15min":3.621},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":17.061,"5min":84.897999999999996,"15min":84.897999999999996},"last":14.432}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":12.247999999999999,"5min":12.247999999999999,"15min":12.247999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":216.56899999999999,"5min":216.56899999999999,"15min":216.56899999999999},"last":4.6070000000000002},{"interface":"front","average":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":216.548,"5min":216.548,"15min":216.548},"last":17.300000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986599,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":737208,"kb_used_data":342448,"kb_used_omap":0,"kb_used_meta":394752,"kb_avail":93007944,"statfs":{"total":95995035648,"available":95240134656,"internally_reserved":0,"allocated":350666752,"data_stored":349372178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":404226048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":4.1310000000000002,"15min":3.9980000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.219},"max":{"1min":29.326000000000001,"5min":159.624,"15min":162.75999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":3.2149999999999999,"5min":4.0750000000000002,"15min":4.0579999999999998},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.221},"max":{"1min":29.721,"5min":159.71000000000001,"15min":162.34800000000001},"last":2.331}]},{"osd":1,"last update":"Sat Sep 23 15:28:11 2023","interfaces":[{"interface":"back","average":{"1min":7.5410000000000004,"5min":5.7249999999999996,"15min":3.8109999999999999},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.245},"max":{"1min":116.33499999999999,"5min":144.76900000000001,"15min":144.76900000000001},"last":3.5089999999999999},{"interface":"front","average":{"1min":8.0540000000000003,"5min":5.8680000000000003,"15min":4.1150000000000002},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":115.91200000000001,"5min":144.58799999999999,"15min":144.58799999999999},"last":3.5950000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.5169999999999999,"15min":2.7829999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0499999999999998,"5min":10.933,"15min":78.751000000000005},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.482,"5min":1.7949999999999999,"15min":3.0419999999999998},"min":{"1min":0.315,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":7.4699999999999998,"5min":7.4699999999999998,"15min":77.930999999999997},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:27:57 2023","interfaces":[{"interface":"back","average":{"1min":1.2529999999999999,"5min":1.833,"15min":5.2850000000000001},"min":{"1min":0.374,"5min":0.34300000000000003,"15min":0.28899999999999998},"max":{"1min":5.5119999999999996,"5min":8.1859999999999999,"15min":156.46000000000001},"last":2.4769999999999999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.5660000000000001,"15min":5.3280000000000003},"min":{"1min":0.32900000000000001,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.6030000000000002,"5min":8.3829999999999991,"15min":156.74299999999999},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.3819999999999999,"15min":1.2450000000000001},"min":{"1min":0.26500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":5.258,"5min":40.685000000000002,"15min":87.912999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.83399999999999996,"5min":1.375,"15min":1.361},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.25,"5min":38.439,"15min":88.602999999999994},"last":0.46100000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.89400000000000002,"15min":1.262},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.3550000000000004,"5min":6.8010000000000002,"15min":7.6470000000000002},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.022,"15min":1.8240000000000001},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4220000000000002,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.401,"15min":1.401},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":5.7249999999999996,"5min":5.7249999999999996,"15min":5.7249999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.8300000000000001,"5min":4.8300000000000001,"15min":4.8300000000000001},"last":0.50700000000000001}]}]},{"osd":5,"up_from":39,"seq":167503724962,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1457200,"kb_used_data":988712,"kb_used_omap":0,"kb_used_meta":468480,"kb_avail":92287952,"statfs":{"total":95995035648,"available":94502862848,"internally_reserved":0,"allocated":1012441088,"data_stored":1011150967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":479723520},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":6.2350000000000003,"15min":4.5250000000000004},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.444,"5min":196.77000000000001,"15min":196.77000000000001},"last":18.937000000000001},{"interface":"front","average":{"1min":1.524,"5min":6.1219999999999999,"15min":4.6130000000000004},"min":{"1min":0.26400000000000001,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":10.554,"5min":196.91399999999999,"15min":196.91399999999999},"last":18.562000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:28:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":6.0720000000000001,"15min":4.0540000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":25.934000000000001,"5min":193.66499999999999,"15min":193.66499999999999},"last":22.463999999999999},{"interface":"front","average":{"1min":2.552,"5min":4.8250000000000002,"15min":3.7440000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":25.699000000000002,"5min":193.52099999999999,"15min":193.52099999999999},"last":21.879000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.7869999999999999,"15min":2.8039999999999998},"min":{"1min":0.308,"5min":0.29699999999999999,"15min":0.25},"max":{"1min":4.5350000000000001,"5min":11.754,"15min":193.00999999999999},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.9610000000000001,"15min":3.089},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":7.3879999999999999,"5min":10.542999999999999,"15min":193.15700000000001},"last":0.33800000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:27:59 2023","interfaces":[{"interface":"back","average":{"1min":0.90700000000000003,"5min":7.7949999999999999,"15min":6.609},"min":{"1min":0.35999999999999999,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":4.4329999999999998,"5min":266.70299999999997,"15min":266.70299999999997},"last":18.257000000000001},{"interface":"front","average":{"1min":0.872,"5min":8.5090000000000003,"15min":6.75},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":3.29,"5min":266.80399999999997,"15min":266.80399999999997},"last":18.271000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.647,"15min":1.579},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.7850000000000001,"5min":10.749000000000001,"15min":138.68600000000001},"last":0.377},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6539999999999999,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27200000000000002},"max":{"1min":3.6349999999999998,"5min":10.516,"15min":138.49600000000001},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.554,"15min":2.1160000000000001},"min":{"1min":0.376,"5min":0.308,"15min":0.308},"max":{"1min":4.7569999999999997,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.6259999999999999,"15min":2.1480000000000001},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.4720000000000004,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.9500000000000002,"5min":3.9500000000000002,"15min":3.9500000000000002},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.0060000000000002,"5min":6.0060000000000002,"15min":6.0060000000000002},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":898695168,"data_stored":898695168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":992612352,"data_stored":992608280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":345964544,"data_stored":345964544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":903151616,"data_stored":903151616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":337903616,"data_stored":337903616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1006571520,"data_stored":1006567448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":343736320,"data_stored":343736320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1022955520,"data_stored":1022951448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:29:01.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:29:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:01.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:01.525 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:29:01.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:01.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[222878]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:01.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[222878]: 1.16 scrub starts 2023-09-23T15:29:01.677 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[222878]: 1.16 scrub ok 2023-09-23T15:29:01.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[215646]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:01.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[215646]: 1.16 scrub starts 2023-09-23T15:29:01.678 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:01 smithi017 ceph-mon[215646]: 1.16 scrub ok 2023-09-23T15:29:01.684 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:01.803 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:01.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:01.850 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:01.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:01 smithi027 ceph-mon[195233]: from='client.36498 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:01.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:01 smithi027 ceph-mon[195233]: 1.16 scrub starts 2023-09-23T15:29:01.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:01 smithi027 ceph-mon[195233]: 1.16 scrub ok 2023-09-23T15:29:01.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:01.978 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:02.107 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:29:02.110 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:02.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:02.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:02.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:02.299 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:02.462 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:02 smithi017 ceph-mon[215646]: pgmap v1174: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-09-23T15:29:02.463 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:02 smithi017 ceph-mon[222878]: pgmap v1174: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-09-23T15:29:02.631 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:02.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:02.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:02.797 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:02 smithi027 ceph-mon[195233]: pgmap v1174: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 795 op/s 2023-09-23T15:29:02.800 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:02.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:29:02.929 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:02.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:03.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:03.065 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:03.110 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:03.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:03.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:03.214 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:03.261 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:03.450 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:03.543 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:03.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:03.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:03.656 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:03.721 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:03.866 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:03.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:03.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:04.025 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:04.115 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:04.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:04.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:04.171 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:04.181 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:04.181 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:04.224 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:04.307 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:04.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:04 smithi027 ceph-mon[195233]: pgmap v1175: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-09-23T15:29:04.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:04 smithi017 ceph-mon[215646]: pgmap v1175: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-09-23T15:29:04.745 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:04 smithi017 ceph-mon[222878]: pgmap v1175: 119 pgs: 119 active+clean; 1.5 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 759 op/s 2023-09-23T15:29:04.955 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:04.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:05.064 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:05.123 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:05.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:29:05.225 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:05.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:05.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:05.335 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:05.388 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:05.509 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:29:05.515 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:05.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:05.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:05.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:05.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:05.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:05.706 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:05.813 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:29:05.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: pgmap v1176: 119 pgs: 119 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: osdmap e407: 8 total, 8 up, 8 in 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[215646]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: pgmap v1176: 119 pgs: 119 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: osdmap e407: 8 total, 8 up, 8 in 2023-09-23T15:29:06.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:06 smithi017 ceph-mon[222878]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: pgmap v1176: 119 pgs: 119 active+clean; 1.3 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 754 op/s 2023-09-23T15:29:06.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2401820699' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: osdmap e407: 8 total, 8 up, 8 in 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.853 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:06 smithi027 ceph-mon[195233]: failed to encode map e407 with expected crc 2023-09-23T15:29:06.976 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:06.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:06.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:07.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:07.162 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:07.272 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:07.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:07.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:07.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:07.432 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:07.442 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:07.459 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:29:07.579 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:07.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:07.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:07.688 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:07.766 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:07.842 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:07.842 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:08.234 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:08.250 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:08.268 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:08.288 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:08.306 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:08.406 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:08.444 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:08.446 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:08] "GET /metrics HTTP/1.1" 200 36371 "" "Prometheus/2.43.0" 2023-09-23T15:29:08.484 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:08.510 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.511 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:29:08.565 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1178,"stamp":"2023-09-23T15:29:06.883714+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1325209547,"num_objects":20177,"num_object_clones":0,"num_object_copies":60531,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20177,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":148336,"num_write_kb":5375158,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13129,"num_bytes_recovered":705767526,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108238,"ondisk_log_size":108238,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8059144,"kb_used_data":4422984,"kb_used_omap":57,"kb_used_meta":3635974,"kb_avail":741902072,"statfs":{"total":767960285184,"available":759707721728,"internally_reserved":0,"allocated":4529135616,"data_stored":4516286255,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":3723237401},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-493475675,"num_objects":-7528,"num_object_clones":0,"num_object_copies":-22584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7528,"num_whiteouts":0,"num_read":-19,"num_read_kb":-35,"num_write":7517,"num_write_kb":-92,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-67087,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.496055"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":754,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333177+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:28:51.333177+0000","last_peered":"2023-09-23T15:28:51.333177+0000","last_clean":"2023-09-23T15:28:51.333177+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:28:51.333177+0000","last_undegraded":"2023-09-23T15:28:51.333177+0000","last_fullsized":"2023-09-23T15:28:51.333177+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335354+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:28:51.335354+0000","last_peered":"2023-09-23T15:28:51.335354+0000","last_clean":"2023-09-23T15:28:51.335354+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:28:51.335354+0000","last_undegraded":"2023-09-23T15:28:51.335354+0000","last_fullsized":"2023-09-23T15:28:51.335354+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368651+0000","last_change":"2023-09-23T15:27:23.313341+0000","last_active":"2023-09-23T15:28:51.368651+0000","last_peered":"2023-09-23T15:28:51.368651+0000","last_clean":"2023-09-23T15:28:51.368651+0000","last_became_active":"2023-09-23T15:27:23.313012+0000","last_became_peered":"2023-09-23T15:27:23.313012+0000","last_unstale":"2023-09-23T15:28:51.368651+0000","last_undegraded":"2023-09-23T15:28:51.368651+0000","last_fullsized":"2023-09-23T15:28:51.368651+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:17:06.833473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":488,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:55.922145+0000","last_change":"2023-09-23T15:28:55.922068+0000","last_active":"2023-09-23T15:28:55.922145+0000","last_peered":"2023-09-23T15:28:55.922145+0000","last_clean":"2023-09-23T15:28:55.922145+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:28:55.922145+0000","last_undegraded":"2023-09-23T15:28:55.922145+0000","last_fullsized":"2023-09-23T15:28:55.922145+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:55.921999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:55.921999+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:38.044680+0000","scrub_duration":0.0074994720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":331,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.406664+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:28:57.406664+0000","last_peered":"2023-09-23T15:28:57.406664+0000","last_clean":"2023-09-23T15:28:57.406664+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:28:57.406664+0000","last_undegraded":"2023-09-23T15:28:57.406664+0000","last_fullsized":"2023-09-23T15:28:57.406664+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":402,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.436971+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:28:51.436971+0000","last_peered":"2023-09-23T15:28:51.436971+0000","last_clean":"2023-09-23T15:28:51.436971+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:28:51.436971+0000","last_undegraded":"2023-09-23T15:28:51.436971+0000","last_fullsized":"2023-09-23T15:28:51.436971+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":541,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.947618+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:28:51.947618+0000","last_peered":"2023-09-23T15:28:51.947618+0000","last_clean":"2023-09-23T15:28:51.947618+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:28:51.947618+0000","last_undegraded":"2023-09-23T15:28:51.947618+0000","last_fullsized":"2023-09-23T15:28:51.947618+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":397,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.369772+0000","last_change":"2023-09-23T15:27:12.402468+0000","last_active":"2023-09-23T15:28:51.369772+0000","last_peered":"2023-09-23T15:28:51.369772+0000","last_clean":"2023-09-23T15:28:51.369772+0000","last_became_active":"2023-09-23T15:27:12.402003+0000","last_became_peered":"2023-09-23T15:27:12.402003+0000","last_unstale":"2023-09-23T15:28:51.369772+0000","last_undegraded":"2023-09-23T15:28:51.369772+0000","last_fullsized":"2023-09-23T15:28:51.369772+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:45.443108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":356,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333637+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:28:51.333637+0000","last_peered":"2023-09-23T15:28:51.333637+0000","last_clean":"2023-09-23T15:28:51.333637+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:28:51.333637+0000","last_undegraded":"2023-09-23T15:28:51.333637+0000","last_fullsized":"2023-09-23T15:28:51.333637+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":674,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368102+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:28:51.368102+0000","last_peered":"2023-09-23T15:28:51.368102+0000","last_clean":"2023-09-23T15:28:51.368102+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:28:51.368102+0000","last_undegraded":"2023-09-23T15:28:51.368102+0000","last_fullsized":"2023-09-23T15:28:51.368102+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":356,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373650+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:28:51.373650+0000","last_peered":"2023-09-23T15:28:51.373650+0000","last_clean":"2023-09-23T15:28:51.373650+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:28:51.373650+0000","last_undegraded":"2023-09-23T15:28:51.373650+0000","last_fullsized":"2023-09-23T15:28:51.373650+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":394,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.369881+0000","last_change":"2023-09-23T15:27:12.402484+0000","last_active":"2023-09-23T15:28:51.369881+0000","last_peered":"2023-09-23T15:28:51.369881+0000","last_clean":"2023-09-23T15:28:51.369881+0000","last_became_active":"2023-09-23T15:27:12.402209+0000","last_became_peered":"2023-09-23T15:27:12.402209+0000","last_unstale":"2023-09-23T15:28:51.369881+0000","last_undegraded":"2023-09-23T15:28:51.369881+0000","last_fullsized":"2023-09-23T15:28:51.369881+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:59:07.497544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":816,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.374932+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:28:51.374932+0000","last_peered":"2023-09-23T15:28:51.374932+0000","last_clean":"2023-09-23T15:28:51.374932+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:28:51.374932+0000","last_undegraded":"2023-09-23T15:28:51.374932+0000","last_fullsized":"2023-09-23T15:28:51.374932+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":423,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335328+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:28:51.335328+0000","last_peered":"2023-09-23T15:28:51.335328+0000","last_clean":"2023-09-23T15:28:51.335328+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:28:51.335328+0000","last_undegraded":"2023-09-23T15:28:51.335328+0000","last_fullsized":"2023-09-23T15:28:51.335328+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":354,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.371030+0000","last_change":"2023-09-23T15:27:41.424467+0000","last_active":"2023-09-23T15:28:51.371030+0000","last_peered":"2023-09-23T15:28:51.371030+0000","last_clean":"2023-09-23T15:28:51.371030+0000","last_became_active":"2023-09-23T15:27:41.424098+0000","last_became_peered":"2023-09-23T15:27:41.424098+0000","last_unstale":"2023-09-23T15:28:51.371030+0000","last_undegraded":"2023-09-23T15:28:51.371030+0000","last_fullsized":"2023-09-23T15:28:51.371030+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:46.478911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":509,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:59.984288+0000","last_change":"2023-09-23T15:28:59.984242+0000","last_active":"2023-09-23T15:28:59.984288+0000","last_peered":"2023-09-23T15:28:59.984288+0000","last_clean":"2023-09-23T15:28:59.984288+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:28:59.984288+0000","last_undegraded":"2023-09-23T15:28:59.984288+0000","last_fullsized":"2023-09-23T15:28:59.984288+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:59.984131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:59.984131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:34.209362+0000","scrub_duration":0.015840911999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":151,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368325+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:28:51.368325+0000","last_peered":"2023-09-23T15:28:51.368325+0000","last_clean":"2023-09-23T15:28:51.368325+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:28:51.368325+0000","last_undegraded":"2023-09-23T15:28:51.368325+0000","last_fullsized":"2023-09-23T15:28:51.368325+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":300,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T15:29:05.480834+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:29:05.480834+0000","last_peered":"2023-09-23T15:29:05.480834+0000","last_clean":"2023-09-23T15:29:05.480834+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:29:05.480834+0000","last_undegraded":"2023-09-23T15:29:05.480834+0000","last_fullsized":"2023-09-23T15:29:05.480834+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":557,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:00.382590+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:29:00.382590+0000","last_peered":"2023-09-23T15:29:00.382590+0000","last_clean":"2023-09-23T15:29:00.382590+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:29:00.382590+0000","last_undegraded":"2023-09-23T15:29:00.382590+0000","last_fullsized":"2023-09-23T15:29:00.382590+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":72,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334537+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:28:51.334537+0000","last_peered":"2023-09-23T15:28:51.334537+0000","last_clean":"2023-09-23T15:28:51.334537+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:28:51.334537+0000","last_undegraded":"2023-09-23T15:28:51.334537+0000","last_fullsized":"2023-09-23T15:28:51.334537+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334904+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:28:51.334904+0000","last_peered":"2023-09-23T15:28:51.334904+0000","last_clean":"2023-09-23T15:28:51.334904+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:28:51.334904+0000","last_undegraded":"2023-09-23T15:28:51.334904+0000","last_fullsized":"2023-09-23T15:28:51.334904+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":717,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437222+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:28:51.437222+0000","last_peered":"2023-09-23T15:28:51.437222+0000","last_clean":"2023-09-23T15:28:51.437222+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:28:51.437222+0000","last_undegraded":"2023-09-23T15:28:51.437222+0000","last_fullsized":"2023-09-23T15:28:51.437222+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":155,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.366088+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:28:51.366088+0000","last_peered":"2023-09-23T15:28:51.366088+0000","last_clean":"2023-09-23T15:28:51.366088+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:28:51.366088+0000","last_undegraded":"2023-09-23T15:28:51.366088+0000","last_fullsized":"2023-09-23T15:28:51.366088+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":404,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.371833+0000","last_change":"2023-09-23T15:27:41.426159+0000","last_active":"2023-09-23T15:28:51.371833+0000","last_peered":"2023-09-23T15:28:51.371833+0000","last_clean":"2023-09-23T15:28:51.371833+0000","last_became_active":"2023-09-23T15:27:41.425781+0000","last_became_peered":"2023-09-23T15:27:41.425781+0000","last_unstale":"2023-09-23T15:28:51.371833+0000","last_undegraded":"2023-09-23T15:28:51.371833+0000","last_fullsized":"2023-09-23T15:28:51.371833+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:16.375308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":781,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334192+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:28:51.334192+0000","last_peered":"2023-09-23T15:28:51.334192+0000","last_clean":"2023-09-23T15:28:51.334192+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:28:51.334192+0000","last_undegraded":"2023-09-23T15:28:51.334192+0000","last_fullsized":"2023-09-23T15:28:51.334192+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":400,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372651+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:28:51.372651+0000","last_peered":"2023-09-23T15:28:51.372651+0000","last_clean":"2023-09-23T15:28:51.372651+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:28:51.372651+0000","last_undegraded":"2023-09-23T15:28:51.372651+0000","last_fullsized":"2023-09-23T15:28:51.372651+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":408,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335228+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:28:51.335228+0000","last_peered":"2023-09-23T15:28:51.335228+0000","last_clean":"2023-09-23T15:28:51.335228+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:28:51.335228+0000","last_undegraded":"2023-09-23T15:28:51.335228+0000","last_fullsized":"2023-09-23T15:28:51.335228+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":733,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.408608+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:28:57.408608+0000","last_peered":"2023-09-23T15:28:57.408608+0000","last_clean":"2023-09-23T15:28:57.408608+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:28:57.408608+0000","last_undegraded":"2023-09-23T15:28:57.408608+0000","last_fullsized":"2023-09-23T15:28:57.408608+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":149,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.366516+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:28:51.366516+0000","last_peered":"2023-09-23T15:28:51.366516+0000","last_clean":"2023-09-23T15:28:51.366516+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:28:51.366516+0000","last_undegraded":"2023-09-23T15:28:51.366516+0000","last_fullsized":"2023-09-23T15:28:51.366516+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":389,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437117+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:28:51.437117+0000","last_peered":"2023-09-23T15:28:51.437117+0000","last_clean":"2023-09-23T15:28:51.437117+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:28:51.437117+0000","last_undegraded":"2023-09-23T15:28:51.437117+0000","last_fullsized":"2023-09-23T15:28:51.437117+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":722,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335315+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:28:51.335315+0000","last_peered":"2023-09-23T15:28:51.335315+0000","last_clean":"2023-09-23T15:28:51.335315+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:28:51.335315+0000","last_undegraded":"2023-09-23T15:28:51.335315+0000","last_fullsized":"2023-09-23T15:28:51.335315+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":342,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333386+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:28:51.333386+0000","last_peered":"2023-09-23T15:28:51.333386+0000","last_clean":"2023-09-23T15:28:51.333386+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:28:51.333386+0000","last_undegraded":"2023-09-23T15:28:51.333386+0000","last_fullsized":"2023-09-23T15:28:51.333386+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":405,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.376961+0000","last_change":"2023-09-23T15:27:41.442107+0000","last_active":"2023-09-23T15:28:51.376961+0000","last_peered":"2023-09-23T15:28:51.376961+0000","last_clean":"2023-09-23T15:28:51.376961+0000","last_became_active":"2023-09-23T15:27:41.441806+0000","last_became_peered":"2023-09-23T15:27:41.441806+0000","last_unstale":"2023-09-23T15:28:51.376961+0000","last_undegraded":"2023-09-23T15:28:51.376961+0000","last_fullsized":"2023-09-23T15:28:51.376961+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:31.229332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.5","version":"406'9221","reported_seq":9946,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:03.738431+0000","last_change":"2023-09-23T15:28:32.579291+0000","last_active":"2023-09-23T15:29:03.738431+0000","last_peered":"2023-09-23T15:29:03.738431+0000","last_clean":"2023-09-23T15:29:03.738431+0000","last_became_active":"2023-09-23T15:28:32.565882+0000","last_became_peered":"2023-09-23T15:28:32.565882+0000","last_unstale":"2023-09-23T15:29:03.738431+0000","last_undegraded":"2023-09-23T15:29:03.738431+0000","last_fullsized":"2023-09-23T15:29:03.738431+0000","mapping_epoch":395,"log_start":"346'2400","ondisk_log_start":"346'2400","created":339,"last_epoch_clean":396,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6821,"log_dups_size":0,"ondisk_log_size":6821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:41.421234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86441984,"num_objects":1319,"num_object_clones":0,"num_object_copies":3957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9221,"num_write_kb":337280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":310,"num_bytes_recovered":3342336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":702,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:56.386954+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:28:56.386954+0000","last_peered":"2023-09-23T15:28:56.386954+0000","last_clean":"2023-09-23T15:28:56.386954+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:28:56.386954+0000","last_undegraded":"2023-09-23T15:28:56.386954+0000","last_fullsized":"2023-09-23T15:28:56.386954+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":145,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368117+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:28:51.368117+0000","last_peered":"2023-09-23T15:28:51.368117+0000","last_clean":"2023-09-23T15:28:51.368117+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:28:51.368117+0000","last_undegraded":"2023-09-23T15:28:51.368117+0000","last_fullsized":"2023-09-23T15:28:51.368117+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":415,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333385+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:28:51.333385+0000","last_peered":"2023-09-23T15:28:51.333385+0000","last_clean":"2023-09-23T15:28:51.333385+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:28:51.333385+0000","last_undegraded":"2023-09-23T15:28:51.333385+0000","last_fullsized":"2023-09-23T15:28:51.333385+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"407'9370","reported_seq":15012,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T15:29:05.672126+0000","last_change":"2023-09-23T15:27:43.281637+0000","last_active":"2023-09-23T15:29:05.672126+0000","last_peered":"2023-09-23T15:29:05.672126+0000","last_clean":"2023-09-23T15:29:05.672126+0000","last_became_active":"2023-09-23T15:27:43.271323+0000","last_became_peered":"2023-09-23T15:27:43.271323+0000","last_unstale":"2023-09-23T15:29:05.672126+0000","last_undegraded":"2023-09-23T15:29:05.672126+0000","last_fullsized":"2023-09-23T15:29:05.672126+0000","mapping_epoch":376,"log_start":"351'3156","ondisk_log_start":"351'3156","created":339,"last_epoch_clean":377,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:59.795930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77987840,"num_objects":1190,"num_object_clones":0,"num_object_copies":3570,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1190,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9370,"num_write_kb":337920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2762,"num_bytes_recovered":181010432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":726,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.371905+0000","last_change":"2023-09-23T15:28:48.882800+0000","last_active":"2023-09-23T15:28:51.371905+0000","last_peered":"2023-09-23T15:28:51.371905+0000","last_clean":"2023-09-23T15:28:51.371905+0000","last_became_active":"2023-09-23T15:27:12.393499+0000","last_became_peered":"2023-09-23T15:27:12.393499+0000","last_unstale":"2023-09-23T15:28:51.371905+0000","last_undegraded":"2023-09-23T15:28:51.371905+0000","last_fullsized":"2023-09-23T15:28:51.371905+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.882635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.882635+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:38.002743+0000","scrub_duration":0.08250528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":145,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.366584+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:28:51.366584+0000","last_peered":"2023-09-23T15:28:51.366584+0000","last_clean":"2023-09-23T15:28:51.366584+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:28:51.366584+0000","last_undegraded":"2023-09-23T15:28:51.366584+0000","last_fullsized":"2023-09-23T15:28:51.366584+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334078+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:28:51.334078+0000","last_peered":"2023-09-23T15:28:51.334078+0000","last_clean":"2023-09-23T15:28:51.334078+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:28:51.334078+0000","last_undegraded":"2023-09-23T15:28:51.334078+0000","last_fullsized":"2023-09-23T15:28:51.334078+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"406'9055","reported_seq":9759,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.994646+0000","last_change":"2023-09-23T15:28:27.475997+0000","last_active":"2023-09-23T15:29:04.994646+0000","last_peered":"2023-09-23T15:29:04.994646+0000","last_clean":"2023-09-23T15:29:04.994646+0000","last_became_active":"2023-09-23T15:28:27.405233+0000","last_became_peered":"2023-09-23T15:28:27.405233+0000","last_unstale":"2023-09-23T15:29:04.994646+0000","last_undegraded":"2023-09-23T15:29:04.994646+0000","last_fullsized":"2023-09-23T15:29:04.994646+0000","mapping_epoch":390,"log_start":"345'1997","ondisk_log_start":"345'1997","created":339,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:17.717875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78839808,"num_objects":1203,"num_object_clones":0,"num_object_copies":3609,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1203,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9055,"num_write_kb":328256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":297,"num_bytes_recovered":2555904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":653,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333840+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:28:51.333840+0000","last_peered":"2023-09-23T15:28:51.333840+0000","last_clean":"2023-09-23T15:28:51.333840+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:28:51.333840+0000","last_undegraded":"2023-09-23T15:28:51.333840+0000","last_fullsized":"2023-09-23T15:28:51.333840+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":133,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368204+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:28:51.368204+0000","last_peered":"2023-09-23T15:28:51.368204+0000","last_clean":"2023-09-23T15:28:51.368204+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:28:51.368204+0000","last_undegraded":"2023-09-23T15:28:51.368204+0000","last_fullsized":"2023-09-23T15:28:51.368204+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":386,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333710+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:28:51.333710+0000","last_peered":"2023-09-23T15:28:51.333710+0000","last_clean":"2023-09-23T15:28:51.333710+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:28:51.333710+0000","last_undegraded":"2023-09-23T15:28:51.333710+0000","last_fullsized":"2023-09-23T15:28:51.333710+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"406'9166","reported_seq":9241,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:05.011829+0000","last_change":"2023-09-23T15:25:35.723494+0000","last_active":"2023-09-23T15:29:05.011829+0000","last_peered":"2023-09-23T15:29:05.011829+0000","last_clean":"2023-09-23T15:29:05.011829+0000","last_became_active":"2023-09-23T15:25:35.723097+0000","last_became_peered":"2023-09-23T15:25:35.723097+0000","last_unstale":"2023-09-23T15:29:05.011829+0000","last_undegraded":"2023-09-23T15:29:05.011829+0000","last_fullsized":"2023-09-23T15:29:05.011829+0000","mapping_epoch":339,"log_start":"345'2128","ondisk_log_start":"345'2128","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:58.863529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78249984,"num_objects":1194,"num_object_clones":0,"num_object_copies":3582,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1194,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9166,"num_write_kb":331520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":623,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334850+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:28:51.334850+0000","last_peered":"2023-09-23T15:28:51.334850+0000","last_clean":"2023-09-23T15:28:51.334850+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:28:51.334850+0000","last_undegraded":"2023-09-23T15:28:51.334850+0000","last_fullsized":"2023-09-23T15:28:51.334850+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":382,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334619+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:28:51.334619+0000","last_peered":"2023-09-23T15:28:51.334619+0000","last_clean":"2023-09-23T15:28:51.334619+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:28:51.334619+0000","last_undegraded":"2023-09-23T15:28:51.334619+0000","last_fullsized":"2023-09-23T15:28:51.334619+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":407,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372798+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:28:51.372798+0000","last_peered":"2023-09-23T15:28:51.372798+0000","last_clean":"2023-09-23T15:28:51.372798+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:28:51.372798+0000","last_undegraded":"2023-09-23T15:28:51.372798+0000","last_fullsized":"2023-09-23T15:28:51.372798+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.1","version":"406'9280","reported_seq":10019,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.814468+0000","last_change":"2023-09-23T15:28:25.337060+0000","last_active":"2023-09-23T15:29:04.814468+0000","last_peered":"2023-09-23T15:29:04.814468+0000","last_clean":"2023-09-23T15:29:04.814468+0000","last_became_active":"2023-09-23T15:28:25.321577+0000","last_became_peered":"2023-09-23T15:28:25.321577+0000","last_unstale":"2023-09-23T15:29:04.814468+0000","last_undegraded":"2023-09-23T15:29:04.814468+0000","last_fullsized":"2023-09-23T15:29:04.814468+0000","mapping_epoch":388,"log_start":"351'3358","ondisk_log_start":"351'3358","created":339,"last_epoch_clean":389,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5922,"log_dups_size":0,"ondisk_log_size":5922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:42.348789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80740352,"num_objects":1232,"num_object_clones":0,"num_object_copies":3696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9280,"num_write_kb":336384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":313,"num_bytes_recovered":2621440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":658,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:52.028830+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:28:52.028830+0000","last_peered":"2023-09-23T15:28:52.028830+0000","last_clean":"2023-09-23T15:28:52.028830+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:28:52.028830+0000","last_undegraded":"2023-09-23T15:28:52.028830+0000","last_fullsized":"2023-09-23T15:28:52.028830+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":401,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.370681+0000","last_change":"2023-09-23T15:27:23.323912+0000","last_active":"2023-09-23T15:28:51.370681+0000","last_peered":"2023-09-23T15:28:51.370681+0000","last_clean":"2023-09-23T15:28:51.370681+0000","last_became_active":"2023-09-23T15:27:23.316529+0000","last_became_peered":"2023-09-23T15:27:23.316529+0000","last_unstale":"2023-09-23T15:28:51.370681+0000","last_undegraded":"2023-09-23T15:28:51.370681+0000","last_fullsized":"2023-09-23T15:28:51.370681+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:55.689850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334443+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:28:51.334443+0000","last_peered":"2023-09-23T15:28:51.334443+0000","last_clean":"2023-09-23T15:28:51.334443+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:28:51.334443+0000","last_undegraded":"2023-09-23T15:28:51.334443+0000","last_fullsized":"2023-09-23T15:28:51.334443+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"407'9433","reported_seq":9511,"reported_epoch":407,"state":"active+clean","last_fresh":"2023-09-23T15:29:05.655669+0000","last_change":"2023-09-23T15:25:35.722919+0000","last_active":"2023-09-23T15:29:05.655669+0000","last_peered":"2023-09-23T15:29:05.655669+0000","last_clean":"2023-09-23T15:29:05.655669+0000","last_became_active":"2023-09-23T15:25:35.722667+0000","last_became_peered":"2023-09-23T15:25:35.722667+0000","last_unstale":"2023-09-23T15:29:05.655669+0000","last_undegraded":"2023-09-23T15:29:05.655669+0000","last_fullsized":"2023-09-23T15:29:05.655669+0000","mapping_epoch":339,"log_start":"351'3219","ondisk_log_start":"351'3219","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6214,"log_dups_size":0,"ondisk_log_size":6214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:51.409068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79626240,"num_objects":1215,"num_object_clones":0,"num_object_copies":3645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9433,"num_write_kb":340736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":730,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334285+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:28:51.334285+0000","last_peered":"2023-09-23T15:28:51.334285+0000","last_clean":"2023-09-23T15:28:51.334285+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:28:51.334285+0000","last_undegraded":"2023-09-23T15:28:51.334285+0000","last_fullsized":"2023-09-23T15:28:51.334285+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":373,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333801+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:28:51.333801+0000","last_peered":"2023-09-23T15:28:51.333801+0000","last_clean":"2023-09-23T15:28:51.333801+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:28:51.333801+0000","last_undegraded":"2023-09-23T15:28:51.333801+0000","last_fullsized":"2023-09-23T15:28:51.333801+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334817+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:28:51.334817+0000","last_peered":"2023-09-23T15:28:51.334817+0000","last_clean":"2023-09-23T15:28:51.334817+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:28:51.334817+0000","last_undegraded":"2023-09-23T15:28:51.334817+0000","last_fullsized":"2023-09-23T15:28:51.334817+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"406'9117","reported_seq":9560,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.414447+0000","last_change":"2023-09-23T15:28:30.734365+0000","last_active":"2023-09-23T15:29:04.414447+0000","last_peered":"2023-09-23T15:29:04.414447+0000","last_clean":"2023-09-23T15:29:04.414447+0000","last_became_active":"2023-09-23T15:28:30.700318+0000","last_became_peered":"2023-09-23T15:28:30.700318+0000","last_unstale":"2023-09-23T15:29:04.414447+0000","last_undegraded":"2023-09-23T15:29:04.414447+0000","last_fullsized":"2023-09-23T15:29:04.414447+0000","mapping_epoch":393,"log_start":"351'3256","ondisk_log_start":"351'3256","created":339,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:11.270292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9117,"num_write_kb":333120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":304,"num_bytes_recovered":3014656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":686,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.408520+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:28:57.408520+0000","last_peered":"2023-09-23T15:28:57.408520+0000","last_clean":"2023-09-23T15:28:57.408520+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:28:57.408520+0000","last_undegraded":"2023-09-23T15:28:57.408520+0000","last_fullsized":"2023-09-23T15:28:57.408520+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":62,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334186+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:28:51.334186+0000","last_peered":"2023-09-23T15:28:51.334186+0000","last_clean":"2023-09-23T15:28:51.334186+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:28:51.334186+0000","last_undegraded":"2023-09-23T15:28:51.334186+0000","last_fullsized":"2023-09-23T15:28:51.334186+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334936+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:28:51.334936+0000","last_peered":"2023-09-23T15:28:51.334936+0000","last_clean":"2023-09-23T15:28:51.334936+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:28:51.334936+0000","last_undegraded":"2023-09-23T15:28:51.334936+0000","last_fullsized":"2023-09-23T15:28:51.334936+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"406'9219","reported_seq":9302,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.843261+0000","last_change":"2023-09-23T15:27:12.390259+0000","last_active":"2023-09-23T15:29:04.843261+0000","last_peered":"2023-09-23T15:29:04.843261+0000","last_clean":"2023-09-23T15:29:04.843261+0000","last_became_active":"2023-09-23T15:27:12.384140+0000","last_became_peered":"2023-09-23T15:27:12.384140+0000","last_unstale":"2023-09-23T15:29:04.843261+0000","last_undegraded":"2023-09-23T15:29:04.843261+0000","last_fullsized":"2023-09-23T15:29:04.843261+0000","mapping_epoch":363,"log_start":"351'3284","ondisk_log_start":"351'3284","created":339,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5935,"log_dups_size":0,"ondisk_log_size":5935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:31.985761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85393408,"num_objects":1303,"num_object_clones":0,"num_object_copies":3909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9219,"num_write_kb":336704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":742,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335217+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:28:51.335217+0000","last_peered":"2023-09-23T15:28:51.335217+0000","last_clean":"2023-09-23T15:28:51.335217+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:28:51.335217+0000","last_undegraded":"2023-09-23T15:28:51.335217+0000","last_fullsized":"2023-09-23T15:28:51.335217+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":415,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.369702+0000","last_change":"2023-09-23T15:27:41.418214+0000","last_active":"2023-09-23T15:28:51.369702+0000","last_peered":"2023-09-23T15:28:51.369702+0000","last_clean":"2023-09-23T15:28:51.369702+0000","last_became_active":"2023-09-23T15:27:41.417924+0000","last_became_peered":"2023-09-23T15:27:41.417924+0000","last_unstale":"2023-09-23T15:28:51.369702+0000","last_undegraded":"2023-09-23T15:28:51.369702+0000","last_fullsized":"2023-09-23T15:28:51.369702+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:45.122620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":341,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.423818+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:28:57.423818+0000","last_peered":"2023-09-23T15:28:57.423818+0000","last_clean":"2023-09-23T15:28:57.423818+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:28:57.423818+0000","last_undegraded":"2023-09-23T15:28:57.423818+0000","last_fullsized":"2023-09-23T15:28:57.423818+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"406'9383","reported_seq":9458,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.994606+0000","last_change":"2023-09-23T15:25:35.716554+0000","last_active":"2023-09-23T15:29:04.994606+0000","last_peered":"2023-09-23T15:29:04.994606+0000","last_clean":"2023-09-23T15:29:04.994606+0000","last_became_active":"2023-09-23T15:25:35.716257+0000","last_became_peered":"2023-09-23T15:25:35.716257+0000","last_unstale":"2023-09-23T15:29:04.994606+0000","last_undegraded":"2023-09-23T15:29:04.994606+0000","last_fullsized":"2023-09-23T15:29:04.994606+0000","mapping_epoch":339,"log_start":"346'2382","ondisk_log_start":"346'2382","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":7001,"log_dups_size":0,"ondisk_log_size":7001,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:37.096030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80674816,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9383,"num_write_kb":339648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":646,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333744+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:28:51.333744+0000","last_peered":"2023-09-23T15:28:51.333744+0000","last_clean":"2023-09-23T15:28:51.333744+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:28:51.333744+0000","last_undegraded":"2023-09-23T15:28:51.333744+0000","last_fullsized":"2023-09-23T15:28:51.333744+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":623,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.407702+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:28:57.407702+0000","last_peered":"2023-09-23T15:28:57.407702+0000","last_clean":"2023-09-23T15:28:57.407702+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:28:57.407702+0000","last_undegraded":"2023-09-23T15:28:57.407702+0000","last_fullsized":"2023-09-23T15:28:57.407702+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":382,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335148+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:28:51.335148+0000","last_peered":"2023-09-23T15:28:51.335148+0000","last_clean":"2023-09-23T15:28:51.335148+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:28:51.335148+0000","last_undegraded":"2023-09-23T15:28:51.335148+0000","last_fullsized":"2023-09-23T15:28:51.335148+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":370,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.375269+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:28:51.375269+0000","last_peered":"2023-09-23T15:28:51.375269+0000","last_clean":"2023-09-23T15:28:51.375269+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:28:51.375269+0000","last_undegraded":"2023-09-23T15:28:51.375269+0000","last_fullsized":"2023-09-23T15:28:51.375269+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"9.9","version":"406'9156","reported_seq":9929,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:03.764012+0000","last_change":"2023-09-23T15:28:29.680984+0000","last_active":"2023-09-23T15:29:03.764012+0000","last_peered":"2023-09-23T15:29:03.764012+0000","last_clean":"2023-09-23T15:29:03.764012+0000","last_became_active":"2023-09-23T15:28:29.667811+0000","last_became_peered":"2023-09-23T15:28:29.667811+0000","last_unstale":"2023-09-23T15:29:03.764012+0000","last_undegraded":"2023-09-23T15:29:03.764012+0000","last_fullsized":"2023-09-23T15:29:03.764012+0000","mapping_epoch":392,"log_start":"346'2256","ondisk_log_start":"346'2256","created":339,"last_epoch_clean":393,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6900,"log_dups_size":0,"ondisk_log_size":6900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:29:51.355536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86769664,"num_objects":1324,"num_object_clones":0,"num_object_copies":3972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9156,"num_write_kb":335360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":336,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":703,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335462+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:28:51.335462+0000","last_peered":"2023-09-23T15:28:51.335462+0000","last_clean":"2023-09-23T15:28:51.335462+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:28:51.335462+0000","last_undegraded":"2023-09-23T15:28:51.335462+0000","last_fullsized":"2023-09-23T15:28:51.335462+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":941,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335472+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:28:51.335472+0000","last_peered":"2023-09-23T15:28:51.335472+0000","last_clean":"2023-09-23T15:28:51.335472+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:28:51.335472+0000","last_undegraded":"2023-09-23T15:28:51.335472+0000","last_fullsized":"2023-09-23T15:28:51.335472+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334686+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:28:51.334686+0000","last_peered":"2023-09-23T15:28:51.334686+0000","last_clean":"2023-09-23T15:28:51.334686+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:28:51.334686+0000","last_undegraded":"2023-09-23T15:28:51.334686+0000","last_fullsized":"2023-09-23T15:28:51.334686+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":305,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.407351+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:28:57.407351+0000","last_peered":"2023-09-23T15:28:57.407351+0000","last_clean":"2023-09-23T15:28:57.407351+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:28:57.407351+0000","last_undegraded":"2023-09-23T15:28:57.407351+0000","last_fullsized":"2023-09-23T15:28:57.407351+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"406'9151","reported_seq":9843,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:05.008447+0000","last_change":"2023-09-23T15:28:23.357956+0000","last_active":"2023-09-23T15:29:05.008447+0000","last_peered":"2023-09-23T15:29:05.008447+0000","last_clean":"2023-09-23T15:29:05.008447+0000","last_became_active":"2023-09-23T15:28:23.286114+0000","last_became_peered":"2023-09-23T15:28:23.286114+0000","last_unstale":"2023-09-23T15:29:05.008447+0000","last_undegraded":"2023-09-23T15:29:05.008447+0000","last_fullsized":"2023-09-23T15:29:05.008447+0000","mapping_epoch":386,"log_start":"345'2167","ondisk_log_start":"345'2167","created":339,"last_epoch_clean":387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6984,"log_dups_size":0,"ondisk_log_size":6984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:16.750949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80150528,"num_objects":1223,"num_object_clones":0,"num_object_copies":3669,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1223,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9151,"num_write_kb":331968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":294,"num_bytes_recovered":3145728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":769,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.369264+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:28:51.369264+0000","last_peered":"2023-09-23T15:28:51.369264+0000","last_clean":"2023-09-23T15:28:51.369264+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:28:51.369264+0000","last_undegraded":"2023-09-23T15:28:51.369264+0000","last_fullsized":"2023-09-23T15:28:51.369264+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1025,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.375008+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:28:51.375008+0000","last_peered":"2023-09-23T15:28:51.375008+0000","last_clean":"2023-09-23T15:28:51.375008+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:28:51.375008+0000","last_undegraded":"2023-09-23T15:28:51.375008+0000","last_fullsized":"2023-09-23T15:28:51.375008+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":301,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.407364+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:28:57.407364+0000","last_peered":"2023-09-23T15:28:57.407364+0000","last_clean":"2023-09-23T15:28:57.407364+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:28:57.407364+0000","last_undegraded":"2023-09-23T15:28:57.407364+0000","last_fullsized":"2023-09-23T15:28:57.407364+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":309,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.423744+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:28:57.423744+0000","last_peered":"2023-09-23T15:28:57.423744+0000","last_clean":"2023-09-23T15:28:57.423744+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:28:57.423744+0000","last_undegraded":"2023-09-23T15:28:57.423744+0000","last_fullsized":"2023-09-23T15:28:57.423744+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"406'9194","reported_seq":9277,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.815614+0000","last_change":"2023-09-23T15:27:12.391982+0000","last_active":"2023-09-23T15:29:04.815614+0000","last_peered":"2023-09-23T15:29:04.815614+0000","last_clean":"2023-09-23T15:29:04.815614+0000","last_became_active":"2023-09-23T15:27:12.386967+0000","last_became_peered":"2023-09-23T15:27:12.386967+0000","last_unstale":"2023-09-23T15:29:04.815614+0000","last_undegraded":"2023-09-23T15:29:04.815614+0000","last_fullsized":"2023-09-23T15:29:04.815614+0000","mapping_epoch":363,"log_start":"351'3296","ondisk_log_start":"351'3296","created":339,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5898,"log_dups_size":0,"ondisk_log_size":5898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:15:32.675846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9194,"num_write_kb":334912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":752,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335213+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:28:51.335213+0000","last_peered":"2023-09-23T15:28:51.335213+0000","last_clean":"2023-09-23T15:28:51.335213+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:28:51.335213+0000","last_undegraded":"2023-09-23T15:28:51.335213+0000","last_fullsized":"2023-09-23T15:28:51.335213+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":712,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.365940+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:28:51.365940+0000","last_peered":"2023-09-23T15:28:51.365940+0000","last_clean":"2023-09-23T15:28:51.365940+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:28:51.365940+0000","last_undegraded":"2023-09-23T15:28:51.365940+0000","last_fullsized":"2023-09-23T15:28:51.365940+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":329,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.371524+0000","last_change":"2023-09-23T15:27:12.392661+0000","last_active":"2023-09-23T15:28:51.371524+0000","last_peered":"2023-09-23T15:28:51.371524+0000","last_clean":"2023-09-23T15:28:51.371524+0000","last_became_active":"2023-09-23T15:27:12.389493+0000","last_became_peered":"2023-09-23T15:27:12.389493+0000","last_unstale":"2023-09-23T15:28:51.371524+0000","last_undegraded":"2023-09-23T15:28:51.371524+0000","last_fullsized":"2023-09-23T15:28:51.371524+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:49.162226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":398,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333630+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:28:51.333630+0000","last_peered":"2023-09-23T15:28:51.333630+0000","last_clean":"2023-09-23T15:28:51.333630+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:28:51.333630+0000","last_undegraded":"2023-09-23T15:28:51.333630+0000","last_fullsized":"2023-09-23T15:28:51.333630+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"406'9273","reported_seq":9349,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:03.783568+0000","last_change":"2023-09-23T15:25:35.708972+0000","last_active":"2023-09-23T15:29:03.783568+0000","last_peered":"2023-09-23T15:29:03.783568+0000","last_clean":"2023-09-23T15:29:03.783568+0000","last_became_active":"2023-09-23T15:25:35.708745+0000","last_became_peered":"2023-09-23T15:25:35.708745+0000","last_unstale":"2023-09-23T15:29:03.783568+0000","last_undegraded":"2023-09-23T15:29:03.783568+0000","last_fullsized":"2023-09-23T15:29:03.783568+0000","mapping_epoch":339,"log_start":"346'2400","ondisk_log_start":"346'2400","created":339,"last_epoch_clean":340,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":6873,"log_dups_size":0,"ondisk_log_size":6873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:43:56.572163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87359488,"num_objects":1333,"num_object_clones":0,"num_object_copies":3999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9273,"num_write_kb":339392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":749,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.408344+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:28:57.408344+0000","last_peered":"2023-09-23T15:28:57.408344+0000","last_clean":"2023-09-23T15:28:57.408344+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:28:57.408344+0000","last_undegraded":"2023-09-23T15:28:57.408344+0000","last_fullsized":"2023-09-23T15:28:57.408344+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":119,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.365850+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:28:51.365850+0000","last_peered":"2023-09-23T15:28:51.365850+0000","last_clean":"2023-09-23T15:28:51.365850+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:28:51.365850+0000","last_undegraded":"2023-09-23T15:28:51.365850+0000","last_fullsized":"2023-09-23T15:28:51.365850+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":393,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437127+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:28:51.437127+0000","last_peered":"2023-09-23T15:28:51.437127+0000","last_clean":"2023-09-23T15:28:51.437127+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:28:51.437127+0000","last_undegraded":"2023-09-23T15:28:51.437127+0000","last_fullsized":"2023-09-23T15:28:51.437127+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":387,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373225+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:28:51.373225+0000","last_peered":"2023-09-23T15:28:51.373225+0000","last_clean":"2023-09-23T15:28:51.373225+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:28:51.373225+0000","last_undegraded":"2023-09-23T15:28:51.373225+0000","last_fullsized":"2023-09-23T15:28:51.373225+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.e","version":"406'9043","reported_seq":19703,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.431053+0000","last_change":"2023-09-23T15:28:26.307366+0000","last_active":"2023-09-23T15:29:04.431053+0000","last_peered":"2023-09-23T15:29:04.431053+0000","last_clean":"2023-09-23T15:29:04.431053+0000","last_became_active":"2023-09-23T15:28:26.291301+0000","last_became_peered":"2023-09-23T15:28:26.291301+0000","last_unstale":"2023-09-23T15:29:04.431053+0000","last_undegraded":"2023-09-23T15:29:04.431053+0000","last_fullsized":"2023-09-23T15:29:04.431053+0000","mapping_epoch":389,"log_start":"351'3257","ondisk_log_start":"351'3257","created":339,"last_epoch_clean":390,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5786,"log_dups_size":0,"ondisk_log_size":5786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:13.182062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9043,"num_write_kb":330752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5269,"num_bytes_recovered":333905920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":732,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333704+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:28:51.333704+0000","last_peered":"2023-09-23T15:28:51.333704+0000","last_clean":"2023-09-23T15:28:51.333704+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:28:51.333704+0000","last_undegraded":"2023-09-23T15:28:51.333704+0000","last_fullsized":"2023-09-23T15:28:51.333704+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1145,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.368229+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:28:51.368229+0000","last_peered":"2023-09-23T15:28:51.368229+0000","last_clean":"2023-09-23T15:28:51.368229+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:28:51.368229+0000","last_undegraded":"2023-09-23T15:28:51.368229+0000","last_fullsized":"2023-09-23T15:28:51.368229+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":285,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.407578+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:28:57.407578+0000","last_peered":"2023-09-23T15:28:57.407578+0000","last_clean":"2023-09-23T15:28:57.407578+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:28:57.407578+0000","last_undegraded":"2023-09-23T15:28:57.407578+0000","last_fullsized":"2023-09-23T15:28:57.407578+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":386,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373010+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:28:51.373010+0000","last_peered":"2023-09-23T15:28:51.373010+0000","last_clean":"2023-09-23T15:28:51.373010+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:28:51.373010+0000","last_undegraded":"2023-09-23T15:28:51.373010+0000","last_fullsized":"2023-09-23T15:28:51.373010+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"9.d","version":"406'9229","reported_seq":9883,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.381672+0000","last_change":"2023-09-23T15:28:24.301029+0000","last_active":"2023-09-23T15:29:04.381672+0000","last_peered":"2023-09-23T15:29:04.381672+0000","last_clean":"2023-09-23T15:29:04.381672+0000","last_became_active":"2023-09-23T15:28:24.286192+0000","last_became_peered":"2023-09-23T15:28:24.286192+0000","last_unstale":"2023-09-23T15:29:04.381672+0000","last_undegraded":"2023-09-23T15:29:04.381672+0000","last_fullsized":"2023-09-23T15:29:04.381672+0000","mapping_epoch":387,"log_start":"351'3447","ondisk_log_start":"351'3447","created":339,"last_epoch_clean":388,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5782,"log_dups_size":0,"ondisk_log_size":5782,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:33.615267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80609304,"num_objects":1231,"num_object_clones":0,"num_object_copies":3693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1231,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9229,"num_write_kb":334657,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":300,"num_bytes_recovered":3014680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":697,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334973+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:28:51.334973+0000","last_peered":"2023-09-23T15:28:51.334973+0000","last_clean":"2023-09-23T15:28:51.334973+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:28:51.334973+0000","last_undegraded":"2023-09-23T15:28:51.334973+0000","last_fullsized":"2023-09-23T15:28:51.334973+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":178,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.375418+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:28:51.375418+0000","last_peered":"2023-09-23T15:28:51.375418+0000","last_clean":"2023-09-23T15:28:51.375418+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:28:51.375418+0000","last_undegraded":"2023-09-23T15:28:51.375418+0000","last_fullsized":"2023-09-23T15:28:51.375418+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1040,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.407696+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:28:57.407696+0000","last_peered":"2023-09-23T15:28:57.407696+0000","last_clean":"2023-09-23T15:28:57.407696+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:28:57.407696+0000","last_undegraded":"2023-09-23T15:28:57.407696+0000","last_fullsized":"2023-09-23T15:28:57.407696+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":395,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.370378+0000","last_change":"2023-09-23T15:27:41.419984+0000","last_active":"2023-09-23T15:28:51.370378+0000","last_peered":"2023-09-23T15:28:51.370378+0000","last_clean":"2023-09-23T15:28:51.370378+0000","last_became_active":"2023-09-23T15:27:41.419701+0000","last_became_peered":"2023-09-23T15:27:41.419701+0000","last_unstale":"2023-09-23T15:28:51.370378+0000","last_undegraded":"2023-09-23T15:28:51.370378+0000","last_fullsized":"2023-09-23T15:28:51.370378+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:55.086471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"9.c","version":"406'9139","reported_seq":14829,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:29:04.825272+0000","last_change":"2023-09-23T15:28:27.558580+0000","last_active":"2023-09-23T15:29:04.825272+0000","last_peered":"2023-09-23T15:29:04.825272+0000","last_clean":"2023-09-23T15:29:04.825272+0000","last_became_active":"2023-09-23T15:28:22.658649+0000","last_became_peered":"2023-09-23T15:28:22.658649+0000","last_unstale":"2023-09-23T15:29:04.825272+0000","last_undegraded":"2023-09-23T15:29:04.825272+0000","last_fullsized":"2023-09-23T15:29:04.825272+0000","mapping_epoch":385,"log_start":"351'3212","ondisk_log_start":"351'3212","created":339,"last_epoch_clean":391,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:25:34.682718+0000","last_clean_scrub_stamp":"2023-09-23T15:25:34.682718+0000","objects_scrubbed":0,"log_size":5927,"log_dups_size":0,"ondisk_log_size":5927,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:46.751262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84738048,"num_objects":1293,"num_object_clones":0,"num_object_copies":3879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9139,"num_write_kb":333824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2785,"num_bytes_recovered":166264832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":725,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.371723+0000","last_change":"2023-09-23T15:27:41.438036+0000","last_active":"2023-09-23T15:28:51.371723+0000","last_peered":"2023-09-23T15:28:51.371723+0000","last_clean":"2023-09-23T15:28:51.371723+0000","last_became_active":"2023-09-23T15:27:41.437722+0000","last_became_peered":"2023-09-23T15:27:41.437722+0000","last_unstale":"2023-09-23T15:28:51.371723+0000","last_undegraded":"2023-09-23T15:28:51.371723+0000","last_fullsized":"2023-09-23T15:28:51.371723+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:37.744292+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":859,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333644+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:28:51.333644+0000","last_peered":"2023-09-23T15:28:51.333644+0000","last_clean":"2023-09-23T15:28:51.333644+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:28:51.333644+0000","last_undegraded":"2023-09-23T15:28:51.333644+0000","last_fullsized":"2023-09-23T15:28:51.333644+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":446,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.373317+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:28:51.373317+0000","last_peered":"2023-09-23T15:28:51.373317+0000","last_clean":"2023-09-23T15:28:51.373317+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:28:51.373317+0000","last_undegraded":"2023-09-23T15:28:51.373317+0000","last_fullsized":"2023-09-23T15:28:51.373317+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334183+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:28:51.334183+0000","last_peered":"2023-09-23T15:28:51.334183+0000","last_clean":"2023-09-23T15:28:51.334183+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:28:51.334183+0000","last_undegraded":"2023-09-23T15:28:51.334183+0000","last_fullsized":"2023-09-23T15:28:51.334183+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":640,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.370450+0000","last_change":"2023-09-23T15:28:50.816286+0000","last_active":"2023-09-23T15:28:51.370450+0000","last_peered":"2023-09-23T15:28:51.370450+0000","last_clean":"2023-09-23T15:28:51.370450+0000","last_became_active":"2023-09-23T15:28:39.666756+0000","last_became_peered":"2023-09-23T15:28:39.666756+0000","last_unstale":"2023-09-23T15:28:51.370450+0000","last_undegraded":"2023-09-23T15:28:51.370450+0000","last_fullsized":"2023-09-23T15:28:51.370450+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.816141+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.816141+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:36.466168+0000","scrub_duration":0.017140743999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":425,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372937+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:28:51.372937+0000","last_peered":"2023-09-23T15:28:51.372937+0000","last_clean":"2023-09-23T15:28:51.372937+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:28:51.372937+0000","last_undegraded":"2023-09-23T15:28:51.372937+0000","last_fullsized":"2023-09-23T15:28:51.372937+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":411,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.375090+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:28:51.375090+0000","last_peered":"2023-09-23T15:28:51.375090+0000","last_clean":"2023-09-23T15:28:51.375090+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:28:51.375090+0000","last_undegraded":"2023-09-23T15:28:51.375090+0000","last_fullsized":"2023-09-23T15:28:51.375090+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":609,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.796932+0000","last_change":"2023-09-23T15:28:51.796932+0000","last_active":"2023-09-23T15:28:51.796932+0000","last_peered":"2023-09-23T15:28:51.796932+0000","last_clean":"2023-09-23T15:28:51.796932+0000","last_became_active":"2023-09-23T15:28:39.656548+0000","last_became_peered":"2023-09-23T15:28:39.656548+0000","last_unstale":"2023-09-23T15:28:51.796932+0000","last_undegraded":"2023-09-23T15:28:51.796932+0000","last_fullsized":"2023-09-23T15:28:51.796932+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.796761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.796761+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:04:26.204960+0000","scrub_duration":0.011035615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":447,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334714+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:28:51.334714+0000","last_peered":"2023-09-23T15:28:51.334714+0000","last_clean":"2023-09-23T15:28:51.334714+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:28:51.334714+0000","last_undegraded":"2023-09-23T15:28:51.334714+0000","last_fullsized":"2023-09-23T15:28:51.334714+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.334825+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:28:51.334825+0000","last_peered":"2023-09-23T15:28:51.334825+0000","last_clean":"2023-09-23T15:28:51.334825+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:28:51.334825+0000","last_undegraded":"2023-09-23T15:28:51.334825+0000","last_fullsized":"2023-09-23T15:28:51.334825+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":427,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335031+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:28:51.335031+0000","last_peered":"2023-09-23T15:28:51.335031+0000","last_clean":"2023-09-23T15:28:51.335031+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:28:51.335031+0000","last_undegraded":"2023-09-23T15:28:51.335031+0000","last_fullsized":"2023-09-23T15:28:51.335031+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.335122+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:28:51.335122+0000","last_peered":"2023-09-23T15:28:51.335122+0000","last_clean":"2023-09-23T15:28:51.335122+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:28:51.335122+0000","last_undegraded":"2023-09-23T15:28:51.335122+0000","last_fullsized":"2023-09-23T15:28:51.335122+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":593,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:58.405531+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:28:58.405531+0000","last_peered":"2023-09-23T15:28:58.405531+0000","last_clean":"2023-09-23T15:28:58.405531+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:28:58.405531+0000","last_undegraded":"2023-09-23T15:28:58.405531+0000","last_fullsized":"2023-09-23T15:28:58.405531+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":386,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.372923+0000","last_change":"2023-09-23T15:27:12.384840+0000","last_active":"2023-09-23T15:28:51.372923+0000","last_peered":"2023-09-23T15:28:51.372923+0000","last_clean":"2023-09-23T15:28:51.372923+0000","last_became_active":"2023-09-23T15:27:12.384518+0000","last_became_peered":"2023-09-23T15:27:12.384518+0000","last_unstale":"2023-09-23T15:28:51.372923+0000","last_undegraded":"2023-09-23T15:28:51.372923+0000","last_fullsized":"2023-09-23T15:28:51.372923+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:18.377550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":385,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.370772+0000","last_change":"2023-09-23T15:27:12.385884+0000","last_active":"2023-09-23T15:28:51.370772+0000","last_peered":"2023-09-23T15:28:51.370772+0000","last_clean":"2023-09-23T15:28:51.370772+0000","last_became_active":"2023-09-23T15:27:12.385612+0000","last_became_peered":"2023-09-23T15:27:12.385612+0000","last_unstale":"2023-09-23T15:28:51.370772+0000","last_undegraded":"2023-09-23T15:28:51.370772+0000","last_fullsized":"2023-09-23T15:28:51.370772+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:50.641495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":586,"reported_epoch":406,"state":"active+clean","last_fresh":"2023-09-23T15:28:57.434264+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:28:57.434264+0000","last_peered":"2023-09-23T15:28:57.434264+0000","last_clean":"2023-09-23T15:28:57.434264+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:28:57.434264+0000","last_undegraded":"2023-09-23T15:28:57.434264+0000","last_fullsized":"2023-09-23T15:28:57.434264+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":360,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.437035+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:28:51.437035+0000","last_peered":"2023-09-23T15:28:51.437035+0000","last_clean":"2023-09-23T15:28:51.437035+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:28:51.437035+0000","last_undegraded":"2023-09-23T15:28:51.437035+0000","last_fullsized":"2023-09-23T15:28:51.437035+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":48,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T15:28:51.333960+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:28:51.333960+0000","last_peered":"2023-09-23T15:28:51.333960+0000","last_clean":"2023-09-23T15:28:51.333960+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:28:51.333960+0000","last_undegraded":"2023-09-23T15:28:51.333960+0000","last_fullsized":"2023-09-23T15:28:51.333960+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1320419352,"num_objects":20149,"num_object_clones":0,"num_object_copies":60447,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":20149,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":147429,"num_write_kb":5362433,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12970,"num_bytes_recovered":701300760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4399443968,"data_stored":4399431752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":102214,"ondisk_log_size":102214,"up":48,"acting":48,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768722,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":833872,"kb_used_data":742472,"kb_used_omap":13,"kb_used_meta":91378,"kb_avail":92911280,"statfs":{"total":95995035648,"available":95141150720,"internally_reserved":0,"allocated":760291328,"data_stored":758981034,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":93571166},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":5.7919999999999998,"5min":5.7919999999999998,"15min":5.7919999999999998},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":36.941000000000003,"5min":36.941000000000003,"15min":36.941000000000003},"last":0.253},{"interface":"front","average":{"1min":5.9080000000000004,"5min":5.9080000000000004,"15min":5.9080000000000004},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":36.904000000000003,"5min":36.904000000000003,"15min":36.904000000000003},"last":0.31900000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":21.283000000000001,"5min":21.283000000000001,"15min":21.283000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":157.096,"5min":157.096,"15min":157.096},"last":9.0589999999999993},{"interface":"front","average":{"1min":21.431999999999999,"5min":21.431999999999999,"15min":21.431999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":157.011,"5min":157.011,"15min":157.011},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76300000000000001,"5min":0.76300000000000001,"15min":0.76300000000000001},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":2.8050000000000002,"5min":2.8050000000000002,"15min":2.8050000000000002},"last":8.8460000000000001},{"interface":"front","average":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":4.109,"5min":4.109,"15min":4.109},"last":0.34899999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":4.0709999999999997,"5min":4.0709999999999997,"15min":4.0709999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":32.847999999999999,"5min":32.847999999999999,"15min":32.847999999999999},"last":0.60499999999999998},{"interface":"front","average":{"1min":4.4809999999999999,"5min":4.4809999999999999,"15min":4.4809999999999999},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":33.064999999999998,"5min":33.064999999999998,"15min":33.064999999999998},"last":8.7550000000000008}]},{"osd":4,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56899999999999995,"5min":0.56899999999999995,"15min":0.56899999999999995},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":1.601,"5min":1.601,"15min":1.601},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.98699999999999999,"5min":0.98699999999999999,"15min":0.98699999999999999},"last":8.8089999999999993}]},{"osd":5,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":3.4279999999999999,"5min":3.4279999999999999,"15min":3.4279999999999999},"last":1.2529999999999999},{"interface":"front","average":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":2.2389999999999999,"5min":2.2389999999999999,"15min":2.2389999999999999},"last":1.2709999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.5390000000000001,"5min":3.5390000000000001,"15min":3.5390000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":1.6619999999999999,"5min":1.6619999999999999,"15min":1.6619999999999999},"last":0.42599999999999999}]}]},{"osd":6,"up_from":304,"seq":1305670058065,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":362496,"kb_used_data":247928,"kb_used_omap":13,"kb_used_meta":114546,"kb_avail":93382656,"statfs":{"total":95995035648,"available":95623839744,"internally_reserved":0,"allocated":253878272,"data_stored":252583667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":117295459},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.8560000000000001,"5min":4.7110000000000003,"15min":2.5569999999999999},"min":{"1min":0.27500000000000002,"5min":0.224,"15min":0.224},"max":{"1min":9.5350000000000001,"5min":184.53200000000001,"15min":184.53200000000001},"last":1.405},{"interface":"front","average":{"1min":1.583,"5min":3.3079999999999998,"15min":3.0209999999999999},"min":{"1min":0.29299999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":4.4329999999999998,"5min":86.227999999999994,"15min":86.227999999999994},"last":0.28299999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":3.1779999999999999,"15min":2.887},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":9.2910000000000004,"5min":86.534000000000006,"15min":86.534000000000006},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.998,"5min":2.8420000000000001,"15min":2.855},"min":{"1min":0.28000000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":6.5419999999999998,"5min":86.144000000000005,"15min":86.144000000000005},"last":0.67300000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":2.0590000000000002,"5min":1.99,"15min":1.6799999999999999},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":9.2379999999999995,"5min":9.8469999999999995,"15min":9.8469999999999995},"last":2.6389999999999998},{"interface":"front","average":{"1min":1.9399999999999999,"5min":1.9670000000000001,"15min":1.8069999999999999},"min":{"1min":0.38100000000000001,"5min":0.307,"15min":0.307},"max":{"1min":9.3650000000000002,"5min":9.3650000000000002,"15min":9.3650000000000002},"last":4.5709999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":10.295999999999999,"5min":3.742,"15min":2.7669999999999999},"min":{"1min":0.35499999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":142.67599999999999,"5min":142.67599999999999,"15min":142.67599999999999},"last":2.8069999999999999},{"interface":"front","average":{"1min":10.250999999999999,"5min":4.1340000000000003,"15min":2.8919999999999999},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":142.541,"5min":142.541,"15min":142.541},"last":0.83399999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.2569999999999999,"5min":1.075,"15min":0.98399999999999999},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":6.6479999999999997,"5min":8.1219999999999999,"15min":8.1219999999999999},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.036,"5min":1.391,"15min":1.677},"min":{"1min":0.317,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.0140000000000002,"5min":13.627000000000001,"15min":13.627000000000001},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":8.3190000000000008,"5min":3.4620000000000002,"15min":2.298},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":130.91300000000001,"5min":130.91300000000001,"15min":130.91300000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":8.4179999999999993,"5min":3.0030000000000001,"15min":1.99},"min":{"1min":0.36599999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":130.92599999999999,"5min":130.92599999999999,"15min":130.92599999999999},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.236,"15min":1.236},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":9.3209999999999997,"5min":9.3209999999999997,"15min":9.3209999999999997},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.4750000000000001,"15min":1.4750000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":6.5709999999999997,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.42499999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015366,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1268352,"kb_used_data":707640,"kb_used_omap":6,"kb_used_meta":560697,"kb_avail":92476800,"statfs":{"total":95995035648,"available":94696243200,"internally_reserved":0,"allocated":724623360,"data_stored":722598134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":574153803},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":5.2649999999999997,"5min":2.6059999999999999,"15min":3.1059999999999999},"min":{"1min":0.24199999999999999,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":57.329000000000001,"5min":57.329000000000001,"15min":60.817999999999998},"last":2.5830000000000002},{"interface":"front","average":{"1min":1.905,"5min":2.8500000000000001,"15min":3.6160000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":7.8200000000000003,"5min":17.128,"15min":63.594999999999999},"last":0.26600000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":5.7489999999999997,"5min":3.0499999999999998,"15min":3.7959999999999998},"min":{"1min":0.26200000000000001,"5min":0.221,"15min":0.221},"max":{"1min":54.091999999999999,"5min":54.091999999999999,"15min":109.375},"last":6.5049999999999999},{"interface":"front","average":{"1min":6.0119999999999996,"5min":2.5289999999999999,"15min":3.157},"min":{"1min":0.27700000000000002,"5min":0.216,"15min":0.216},"max":{"1min":57.279000000000003,"5min":57.279000000000003,"15min":105.16},"last":2.5409999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":3.5419999999999998,"5min":2.319,"15min":5.4379999999999997},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":15.191000000000001,"5min":15.191000000000001,"15min":211.405},"last":2.6899999999999999},{"interface":"front","average":{"1min":2.6800000000000002,"5min":2.3199999999999998,"15min":5.6139999999999999},"min":{"1min":0.26400000000000001,"5min":0.246,"15min":0.246},"max":{"1min":8.8949999999999996,"5min":15.637,"15min":210.988},"last":0.34300000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":8.4640000000000004,"5min":3.508,"15min":3.5640000000000001},"min":{"1min":0.372,"5min":0.33300000000000002,"15min":0.29799999999999999},"max":{"1min":72.602000000000004,"5min":72.602000000000004,"15min":80.570999999999998},"last":2.5640000000000001},{"interface":"front","average":{"1min":5.7869999999999999,"5min":2.7210000000000001,"15min":3.246},"min":{"1min":0.318,"5min":0.312,"15min":0.308},"max":{"1min":73.454999999999998,"5min":73.454999999999998,"15min":80.75},"last":2.633}]},{"osd":5,"last update":"Sat Sep 23 15:28:18 2023","interfaces":[{"interface":"back","average":{"1min":2.4550000000000001,"5min":5.5880000000000001,"15min":3.669},"min":{"1min":0.38400000000000001,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":5.6390000000000002,"5min":173.18799999999999,"15min":173.18799999999999},"last":0.374},{"interface":"front","average":{"1min":4.4400000000000004,"5min":6.0170000000000003,"15min":3.9039999999999999},"min":{"1min":0.42499999999999999,"5min":0.33700000000000002,"15min":0.311},"max":{"1min":54.167000000000002,"5min":172.822,"15min":172.822},"last":2.6070000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:28:37 2023","interfaces":[{"interface":"back","average":{"1min":4.8070000000000004,"5min":3.1000000000000001,"15min":3.0800000000000001},"min":{"1min":0.45500000000000002,"5min":0.378,"15min":0.378},"max":{"1min":54.198,"5min":54.198,"15min":54.198},"last":6.2270000000000003},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.9710000000000001,"15min":2.2290000000000001},"min":{"1min":0.45500000000000002,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":5.4459999999999997,"5min":15.605,"15min":15.605},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":2.1389999999999998,"5min":2.1389999999999998,"15min":2.1389999999999998},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":6.7309999999999999,"5min":6.7309999999999999,"15min":6.7309999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":5.0730000000000004,"5min":5.0730000000000004,"15min":5.0730000000000004},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":54.223999999999997,"5min":54.223999999999997,"15min":54.223999999999997},"last":6.2690000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893412,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1462036,"kb_used_data":799180,"kb_used_omap":8,"kb_used_meta":662775,"kb_avail":92283116,"statfs":{"total":95995035648,"available":94497910784,"internally_reserved":0,"allocated":818360320,"data_stored":816484900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":678682196},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:28:09 2023","interfaces":[{"interface":"back","average":{"1min":10.531000000000001,"5min":6.6459999999999999,"15min":4.3559999999999999},"min":{"1min":0.22,"5min":0.217,"15min":0.18099999999999999},"max":{"1min":141.52199999999999,"5min":141.52199999999999,"15min":141.52199999999999},"last":1.1599999999999999},{"interface":"front","average":{"1min":11.266,"5min":6.79,"15min":4.1719999999999997},"min":{"1min":0.23799999999999999,"5min":0.223,"15min":0.20999999999999999},"max":{"1min":141.62799999999999,"5min":141.62799999999999,"15min":141.62799999999999},"last":1.367}]},{"osd":2,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":2.6560000000000001,"5min":2.0699999999999998,"15min":3.5720000000000001},"min":{"1min":0.23400000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":13.968999999999999,"5min":14.994,"15min":93.873999999999995},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.7829999999999999,"5min":1.7809999999999999,"15min":3.488},"min":{"1min":0.20300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":10.577,"5min":10.874000000000001,"15min":94.652000000000001},"last":1.3089999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:28:58 2023","interfaces":[{"interface":"back","average":{"1min":7.7430000000000003,"5min":5.5499999999999998,"15min":5.9180000000000001},"min":{"1min":0.35499999999999998,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":80.415999999999997,"5min":162.154,"15min":197.78899999999999},"last":1.9610000000000001},{"interface":"front","average":{"1min":6.327,"5min":5.1310000000000002,"15min":6.5490000000000004},"min":{"1min":0.34899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":80.463999999999999,"5min":162.215,"15min":200.58799999999999},"last":1.131}]},{"osd":4,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.5819999999999999,"5min":2.113,"15min":2.1339999999999999},"min":{"1min":0.40699999999999997,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":11.887,"5min":15.038,"15min":31.341999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":3.1970000000000001,"5min":2.2360000000000002,"15min":2.1480000000000001},"min":{"1min":0.32900000000000001,"5min":0.314,"15min":0.29399999999999998},"max":{"1min":13.656000000000001,"5min":13.656000000000001,"15min":31.059999999999999},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":1.6120000000000001,"5min":2.5529999999999999,"15min":2.3370000000000002},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":10.489000000000001,"5min":44.57,"15min":64.349999999999994},"last":1.0629999999999999},{"interface":"front","average":{"1min":2.637,"5min":2.9380000000000002,"15min":2.3460000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.312},"max":{"1min":9.798,"5min":44.588000000000001,"15min":44.588000000000001},"last":0.64700000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:28:38 2023","interfaces":[{"interface":"back","average":{"1min":3.3100000000000001,"5min":2.5670000000000002,"15min":2.7290000000000001},"min":{"1min":0.433,"5min":0.379,"15min":0.379},"max":{"1min":11.974,"5min":32.097999999999999,"15min":32.097999999999999},"last":0.67200000000000004},{"interface":"front","average":{"1min":4.0190000000000001,"5min":2.927,"15min":3.2160000000000002},"min":{"1min":0.42099999999999999,"5min":0.371,"15min":0.371},"max":{"1min":14.563000000000001,"5min":32.188000000000002,"15min":32.188000000000002},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:28:44 2023","interfaces":[{"interface":"back","average":{"1min":2.2949999999999999,"5min":2.2949999999999999,"15min":2.2949999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":10.537000000000001,"5min":10.537000000000001,"15min":10.537000000000001},"last":1.1000000000000001},{"interface":"front","average":{"1min":4.1529999999999996,"5min":4.1529999999999996,"15min":4.1529999999999996},"min":{"1min":0.41799999999999998,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":12.651,"5min":12.651,"15min":12.651},"last":0.61699999999999999}]}]},{"osd":2,"up_from":105,"seq":450971566362,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":911368,"kb_used_data":256320,"kb_used_omap":11,"kb_used_meta":655028,"kb_avail":92833784,"statfs":{"total":95995035648,"available":95061794816,"internally_reserved":0,"allocated":262471680,"data_stored":260604996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":670749456},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.8360000000000001,"5min":6.9180000000000001,"15min":5.8639999999999999},"min":{"1min":0.252,"5min":0.21199999999999999,"15min":0.17199999999999999},"max":{"1min":18.739999999999998,"5min":235.517,"15min":235.517},"last":2.968},{"interface":"front","average":{"1min":1.3759999999999999,"5min":6.8529999999999998,"15min":5.5759999999999996},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.8030000000000008,"5min":235.33799999999999,"15min":235.33799999999999},"last":3.883}]},{"osd":1,"last update":"Sat Sep 23 15:28:16 2023","interfaces":[{"interface":"back","average":{"1min":2.6150000000000002,"5min":3.0659999999999998,"15min":3.9300000000000002},"min":{"1min":0.28299999999999997,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":18.510000000000002,"5min":71.819000000000003,"15min":84.641000000000005},"last":7.1699999999999999},{"interface":"front","average":{"1min":1.248,"5min":2.8490000000000002,"15min":3.4089999999999998},"min":{"1min":0.26000000000000001,"5min":0.24399999999999999,"15min":0.20899999999999999},"max":{"1min":5.7469999999999999,"5min":71.805000000000007,"15min":84.751999999999995},"last":4.2160000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:28:55 2023","interfaces":[{"interface":"back","average":{"1min":8.8209999999999997,"5min":3.4809999999999999,"15min":5.8440000000000003},"min":{"1min":0.26000000000000001,"5min":0.24399999999999999,"15min":0.23499999999999999},"max":{"1min":75.701999999999998,"5min":75.701999999999998,"15min":87.108000000000004},"last":3.9279999999999999},{"interface":"front","average":{"1min":8.641,"5min":3.5230000000000001,"15min":5.867},"min":{"1min":0.31,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":75.388999999999996,"5min":75.388999999999996,"15min":86.158000000000001},"last":3.9670000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":2.8109999999999999,"5min":1.7290000000000001,"15min":2.097},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28000000000000003},"max":{"1min":40.119999999999997,"5min":40.119999999999997,"15min":84.608000000000004},"last":6.0449999999999999},{"interface":"front","average":{"1min":3.4630000000000001,"5min":1.716,"15min":2.4060000000000001},"min":{"1min":0.31900000000000001,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":40.198,"5min":40.198,"15min":87.019999999999996},"last":6.4039999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:28:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7649999999999999,"5min":2.5070000000000001,"15min":2.6120000000000001},"min":{"1min":0.34799999999999998,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":9.1899999999999995,"5min":53.594999999999999,"15min":84.498000000000005},"last":3.3159999999999998},{"interface":"front","average":{"1min":1.3500000000000001,"5min":2.375,"15min":2.4460000000000002},"min":{"1min":0.36499999999999999,"5min":0.318,"15min":0.30399999999999999},"max":{"1min":8.7579999999999991,"5min":54.009999999999998,"15min":84.664000000000001},"last":3.1850000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:28:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.9039999999999999,"15min":2.157},"min":{"1min":0.41199999999999998,"5min":0.36699999999999999,"15min":0.35399999999999998},"max":{"1min":9.2289999999999992,"5min":9.4600000000000009,"15min":9.4600000000000009},"last":3.2850000000000001},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.373,"15min":1.825},"min":{"1min":0.46600000000000003,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":3.2290000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":2.4900000000000002,"5min":2.4900000000000002,"15min":2.4900000000000002},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":9.3559999999999999,"5min":9.3559999999999999,"15min":9.3559999999999999},"last":5.9480000000000004},{"interface":"front","average":{"1min":3.3039999999999998,"5min":3.3039999999999998,"15min":3.3039999999999998},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":10.964,"5min":10.964,"15min":10.964},"last":4.1109999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402834,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370460,"kb_used_data":673236,"kb_used_omap":3,"kb_used_meta":697212,"kb_avail":92374692,"statfs":{"total":95995035648,"available":94591684608,"internally_reserved":0,"allocated":689393664,"data_stored":687509367,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":713945257},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":3.4700000000000002,"15min":3.6200000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.183},"max":{"1min":7.5359999999999996,"5min":88.489000000000004,"15min":108.318},"last":0.49399999999999999},{"interface":"front","average":{"1min":2.3849999999999998,"5min":3.8079999999999998,"15min":3.9660000000000002},"min":{"1min":0.189,"5min":0.189,"15min":0.184},"max":{"1min":14.574999999999999,"5min":88.938000000000002,"15min":108.889},"last":0.63}]},{"osd":1,"last update":"Sat Sep 23 15:28:13 2023","interfaces":[{"interface":"back","average":{"1min":7.9800000000000004,"5min":4.6260000000000003,"15min":4.9790000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":121.26900000000001,"5min":121.26900000000001,"15min":126.875},"last":0.42699999999999999},{"interface":"front","average":{"1min":8.3140000000000001,"5min":4.899,"15min":5.25},"min":{"1min":0.25800000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":121.012,"5min":121.012,"15min":128.249},"last":0.745}]},{"osd":2,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.7490000000000001,"15min":2.8180000000000001},"min":{"1min":0.28699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.6980000000000004,"5min":14.853,"15min":61.732999999999997},"last":0.71299999999999997},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.0409999999999999,"15min":3.6720000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":14.343999999999999,"5min":17.091000000000001,"15min":119.449},"last":0.66200000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.5289999999999999,"15min":2.177},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":14.4,"5min":17.027999999999999,"15min":54.051000000000002},"last":1.923},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.8420000000000001,"15min":2.0790000000000002},"min":{"1min":0.28599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.1369999999999996,"5min":13.115,"15min":52.228999999999999},"last":0.46000000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.8130000000000002,"15min":2.9740000000000002},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":17.195,"5min":82.793000000000006,"15min":148.619},"last":0.84699999999999998},{"interface":"front","average":{"1min":2.121,"5min":2.996,"15min":2.8039999999999998},"min":{"1min":0.32300000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2539999999999996,"5min":82.875,"15min":148.56999999999999},"last":1.0229999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:28:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":3.6749999999999998,"15min":2.7080000000000002},"min":{"1min":0.36399999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.8440000000000003,"5min":84.665000000000006,"15min":84.665000000000006},"last":0.47899999999999998},{"interface":"front","average":{"1min":2.6150000000000002,"5min":3.5630000000000002,"15min":3.621},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":17.061,"5min":84.897999999999996,"15min":84.897999999999996},"last":1.8200000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:28:42 2023","interfaces":[{"interface":"back","average":{"1min":12.247999999999999,"5min":12.247999999999999,"15min":12.247999999999999},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":216.56899999999999,"5min":216.56899999999999,"15min":216.56899999999999},"last":3.5830000000000002},{"interface":"front","average":{"1min":13.422000000000001,"5min":13.422000000000001,"15min":13.422000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":216.548,"5min":216.548,"15min":216.548},"last":0.92200000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986601,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":643780,"kb_used_data":256316,"kb_used_omap":0,"kb_used_meta":387456,"kb_avail":93101372,"statfs":{"total":95995035648,"available":95335804928,"internally_reserved":0,"allocated":262467584,"data_stored":261168940,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":396754944},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:22 2023","interfaces":[{"interface":"back","average":{"1min":2.3769999999999998,"5min":4.1310000000000002,"15min":3.9980000000000002},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.219},"max":{"1min":29.326000000000001,"5min":159.624,"15min":162.75999999999999},"last":3.0259999999999998},{"interface":"front","average":{"1min":3.2149999999999999,"5min":4.0750000000000002,"15min":4.0579999999999998},"min":{"1min":0.30299999999999999,"5min":0.27200000000000002,"15min":0.221},"max":{"1min":29.721,"5min":159.71000000000001,"15min":162.34800000000001},"last":3.1840000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:28:11 2023","interfaces":[{"interface":"back","average":{"1min":7.5410000000000004,"5min":5.7249999999999996,"15min":3.8109999999999999},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.245},"max":{"1min":116.33499999999999,"5min":144.76900000000001,"15min":144.76900000000001},"last":0.434},{"interface":"front","average":{"1min":8.0540000000000003,"5min":5.8680000000000003,"15min":4.1150000000000002},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":115.91200000000001,"5min":144.58799999999999,"15min":144.58799999999999},"last":4.1520000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:28:22 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.5169999999999999,"15min":2.7829999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":5.0499999999999998,"5min":10.933,"15min":78.751000000000005},"last":3.5299999999999998},{"interface":"front","average":{"1min":1.482,"5min":1.7949999999999999,"15min":3.0419999999999998},"min":{"1min":0.315,"5min":0.30599999999999999,"15min":0.28999999999999998},"max":{"1min":7.4699999999999998,"5min":7.4699999999999998,"15min":77.930999999999997},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:29:03 2023","interfaces":[{"interface":"back","average":{"1min":6.452,"5min":2.6760000000000002,"15min":4.3159999999999998},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.30399999999999999},"max":{"1min":96.587000000000003,"5min":96.587000000000003,"15min":156.46000000000001},"last":3.359},{"interface":"front","average":{"1min":7.2000000000000002,"5min":2.5790000000000002,"15min":4.4219999999999997},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":96.784999999999997,"5min":96.784999999999997,"15min":156.74299999999999},"last":10.617000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:28:11 2023","interfaces":[{"interface":"back","average":{"1min":0.75700000000000001,"5min":1.3819999999999999,"15min":1.2450000000000001},"min":{"1min":0.26500000000000001,"5min":0.221,"15min":0.221},"max":{"1min":5.258,"5min":40.685000000000002,"15min":87.912999999999997},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.83399999999999996,"5min":1.375,"15min":1.361},"min":{"1min":0.28599999999999998,"5min":0.246,"15min":0.246},"max":{"1min":5.25,"5min":38.439,"15min":88.602999999999994},"last":4.1029999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.89400000000000002,"15min":1.262},"min":{"1min":0.311,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":5.3550000000000004,"5min":6.8010000000000002,"15min":7.6470000000000002},"last":4.1970000000000001},{"interface":"front","average":{"1min":0.68000000000000005,"5min":1.022,"15min":1.8240000000000001},"min":{"1min":0.34899999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.4220000000000002,"5min":7.5380000000000003,"15min":7.5380000000000003},"last":0.40200000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.401,"15min":1.401},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":5.7249999999999996,"5min":5.7249999999999996,"15min":5.7249999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.0569999999999999,"15min":1.0569999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.8300000000000001,"5min":4.8300000000000001,"15min":4.8300000000000001},"last":0.65400000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724964,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1206780,"kb_used_data":739892,"kb_used_omap":0,"kb_used_meta":466880,"kb_avail":92538372,"statfs":{"total":95995035648,"available":94759292928,"internally_reserved":0,"allocated":757649408,"data_stored":756355217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":478085120},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:28:28 2023","interfaces":[{"interface":"back","average":{"1min":1.3540000000000001,"5min":6.2350000000000003,"15min":4.5250000000000004},"min":{"1min":0.246,"5min":0.23799999999999999,"15min":0.217},"max":{"1min":7.444,"5min":196.77000000000001,"15min":196.77000000000001},"last":9.6500000000000004},{"interface":"front","average":{"1min":1.524,"5min":6.1219999999999999,"15min":4.6130000000000004},"min":{"1min":0.26400000000000001,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":10.554,"5min":196.91399999999999,"15min":196.91399999999999},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:28:19 2023","interfaces":[{"interface":"back","average":{"1min":2.5760000000000001,"5min":6.0720000000000001,"15min":4.0540000000000003},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":25.934000000000001,"5min":193.66499999999999,"15min":193.66499999999999},"last":9.7010000000000005},{"interface":"front","average":{"1min":2.552,"5min":4.8250000000000002,"15min":3.7440000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.25900000000000001},"max":{"1min":25.699000000000002,"5min":193.52099999999999,"15min":193.52099999999999},"last":0.25600000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:28:28 2023","interfaces":[{"interface":"back","average":{"1min":0.88200000000000001,"5min":1.7869999999999999,"15min":2.8039999999999998},"min":{"1min":0.308,"5min":0.29699999999999999,"15min":0.25},"max":{"1min":4.5350000000000001,"5min":11.754,"15min":193.00999999999999},"last":0.32500000000000001},{"interface":"front","average":{"1min":1.2290000000000001,"5min":1.9610000000000001,"15min":3.089},"min":{"1min":0.33500000000000002,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":7.3879999999999999,"5min":10.542999999999999,"15min":193.15700000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":5.0549999999999997,"15min":6.6459999999999999},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":18.257000000000001,"5min":150.75899999999999,"15min":266.70299999999997},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.339,"5min":4.9889999999999999,"15min":6.7670000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":18.271000000000001,"5min":150.62299999999999,"15min":266.80399999999997},"last":9.6799999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:28:19 2023","interfaces":[{"interface":"back","average":{"1min":0.83199999999999996,"5min":1.647,"15min":1.579},"min":{"1min":0.28599999999999998,"5min":0.249,"15min":0.249},"max":{"1min":3.7850000000000001,"5min":10.749000000000001,"15min":138.68600000000001},"last":9.7189999999999994},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6539999999999999,"15min":1.5740000000000001},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.27200000000000002},"max":{"1min":3.6349999999999998,"5min":10.516,"15min":138.49600000000001},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 15:28:35 2023","interfaces":[{"interface":"back","average":{"1min":1.526,"5min":1.554,"15min":2.1160000000000001},"min":{"1min":0.376,"5min":0.308,"15min":0.308},"max":{"1min":4.7569999999999997,"5min":10.577999999999999,"15min":10.577999999999999},"last":0.56799999999999995},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.6259999999999999,"15min":2.1480000000000001},"min":{"1min":0.38100000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.4720000000000004,"5min":13.257999999999999,"15min":13.257999999999999},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:28:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2829999999999999,"5min":1.2829999999999999,"15min":1.2829999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":3.9500000000000002,"5min":3.9500000000000002,"15min":3.9500000000000002},"last":9.7400000000000002},{"interface":"front","average":{"1min":1.6259999999999999,"5min":1.6259999999999999,"15min":1.6259999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.0060000000000002,"5min":6.0060000000000002,"15min":6.0060000000000002},"last":0.53500000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":805371904,"data_stored":805371904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":709496832,"data_stored":709492760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":250413056,"data_stored":250413056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":658964480,"data_stored":658964480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":249298944,"data_stored":249298944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":728043520,"data_stored":728039448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":248512512,"data_stored":248512512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":749342720,"data_stored":749338648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:29:08.572 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:29:08.572 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:29:08.573 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:29:08.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T15:29:08.573 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:08.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:08.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:08.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:08.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:29:08.740 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:08 smithi027 ceph-mon[195233]: pgmap v1178: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T15:29:08.740 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:08 smithi027 ceph-mon[195233]: from='client.56326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:08.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2156916219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:08.749 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:08.903 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:08.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:08.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:08.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[215646]: pgmap v1178: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T15:29:08.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[215646]: from='client.56326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:08.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2156916219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:08.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[222878]: pgmap v1178: 119 pgs: 119 active+clean; 1.2 GiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 791 op/s 2023-09-23T15:29:08.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[222878]: from='client.56326 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:29:08.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2156916219' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:08.950 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:29:09.010 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:09.079 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:09.178 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:09.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:09.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:09.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:09.354 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:29:09.369 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:09.487 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:09.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:09.596 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:09.647 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:09.655 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:09.716 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:09.800 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:09.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:09.905 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:09.979 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:10.112 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:10.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:10.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:10.222 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:10.250 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:10.296 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:10.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:10.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:10.400 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:10.478 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:10.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:10 smithi027 ceph-mon[195233]: pgmap v1179: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T15:29:10.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:10 smithi017 ceph-mon[222878]: pgmap v1179: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T15:29:10.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:10 smithi017 ceph-mon[215646]: pgmap v1179: 119 pgs: 119 active+clean; 1.2 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 756 op/s 2023-09-23T15:29:10.735 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:10.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:10.841 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:10.872 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:11.055 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:11.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:11.166 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:11.236 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:11.247 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:29:11.341 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:11.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:11.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:11.416 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:29:11.416 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:29:11.416 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:29:11.416 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:29:11.417 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:29:11.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:11.520 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:11.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:11 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:29:11.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:11 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:29:11.621 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:11 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:29:11.766 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:11.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:11.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:11.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:11.915 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:12.313 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:12.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:12.319 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:12.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:29:12.420 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:12.467 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:12.571 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:12.583 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:12 smithi027 ceph-mon[195233]: pgmap v1180: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:12.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:12.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:12.657 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:12.684 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:12.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:12 smithi017 ceph-mon[222878]: pgmap v1180: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:12.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:12 smithi017 ceph-mon[215646]: pgmap v1180: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:12.812 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:12.913 INFO:teuthology.orchestra.run.smithi017.stdout:moved 9 / 357 (2.52101%) 2023-09-23T15:29:12.913 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.625 2023-09-23T15:29:12.913 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.36273 -> 7.12281 (expected baseline 6.24875) 2023-09-23T15:29:12.913 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 31 -> 31 pgs (0.694678 -> 0.694678 * mean) 2023-09-23T15:29:12.913 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.7 with 52 -> 54 pgs (1.16527 -> 1.21008 * mean) 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0097 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0107 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1 weight 1.0000 -> 0.9500 2023-09-23T15:29:12.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5 weight 1.0000 -> 0.9500 2023-09-23T15:29:12.946 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:29:12.991 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:13.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:13.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:13.046 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:12 smithi017 conmon[202315]: 2023-09-23T15:29:12.904+0000 7fdcb8de7700 -1 mgr.server reply reply (4) Interrupted system call no change 2023-09-23T15:29:13.058 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:29:13.103 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:13.155 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:13.254 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:13.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:13.376 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:13.540 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:29:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1240596212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1240596212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:13.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:13 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1240596212' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:14.313 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:14.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:14.347 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:29:14.347 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T15:29:14.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:14.482 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:14.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: pgmap v1181: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: osdmap e408: 8 total, 8 up, 8 in 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[222878]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: pgmap v1181: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:14.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:14.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: osdmap e408: 8 total, 8 up, 8 in 2023-09-23T15:29:14.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:14 smithi017 ceph-mon[215646]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: pgmap v1181: 119 pgs: 119 active+clean; 1015 MiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: from='client.46424 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: osdmap e408: 8 total, 8 up, 8 in 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: failed to encode map e408 with expected crc 2023-09-23T15:29:14.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:14 smithi027 ceph-mon[195233]: failed to encode map e408 with expected crc 2023-09-23T15:29:15.134 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:15.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:15.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:15.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:15.303 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:15.383 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:15.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:15.400 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:15.500 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:15.589 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:15.696 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:15.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:15.712 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:15.812 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:15.824 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:15.889 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88039 9.d 9.b6231ccd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506738818000002, 2023-09-23T15:29:15.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515619899999996, 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88039 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409929+0000", 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409936+0000", 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409948+0000", 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1756999999999999e-05 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409957+0000", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3381999999999998e-05 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410021+0000", 2023-09-23T15:29:15.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.101e-05 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405075+0000", 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505363800000002 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405087+0000", 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1863e-05 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.928 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88041 9.d 9.a4d7176d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506698219, 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99514826899999997, 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.929 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88041 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409973+0000", 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6620000000000001e-06 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409980+0000", 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7689999999999999e-06 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409985+0000", 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6720000000000003e-06 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410051+0000", 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:29:15.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410058+0000", 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9380000000000003e-06 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405110+0000", 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505194299999999 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405119+0000", 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7710000000000001e-06 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88044 9.d 9.5d4f13fd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506661711999996, 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515036099999998, 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.934 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88044 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.935 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410007+0000", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5220000000000002e-06 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.936 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410018+0000", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6599999999999995e-06 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410023+0000", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3789999999999997e-06 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410129+0000", 2023-09-23T15:29:15.937 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010582300000000001 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410141+0000", 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2234e-05 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405148+0000", 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.995007164 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.938 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405158+0000", 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9410000000000002e-06 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88045 9.d 9.d99be86d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506638692999999, 2023-09-23T15:29:15.939 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515891099999998, 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88045 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.940 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410030+0000", 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.941 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410033+0000", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410039+0000", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0480000000000004e-06 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410043+0000", 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4689999999999999e-06 2023-09-23T15:29:15.942 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410178+0000", 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134657 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410185+0000", 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7519999999999999e-06 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.943 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405180+0000", 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.994995097 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405190+0000", 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7569999999999999e-06 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.944 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88046 9.3 9.1f14c703 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506621291000002, 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99103617399999999, 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88046 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.945 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:29:15.946 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410050+0000", 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8989999999999999e-06 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410055+0000", 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4700000000000001e-06 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.947 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410061+0000", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7860000000000002e-06 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410100+0000", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8804999999999999e-05 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410109+0000", 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8459999999999997e-06 2023-09-23T15:29:15.948 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401069+0000", 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99096060100000005 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401084+0000", 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5098e-05 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.949 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88047 9.3 9.c318e9c3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 95.506593257000006, 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99202290000000004, 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88047 2023-09-23T15:29:15.950 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.951 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410078+0000", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4839999999999998e-06 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410085+0000", 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7649999999999996e-06 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.952 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410090+0000", 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.899e-06 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410149+0000", 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9074999999999999e-05 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.953 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410159+0000", 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0394000000000001e-05 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402086+0000", 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99192703699999996 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402099+0000", 2023-09-23T15:29:15.954 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2700999999999999e-05 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118583 9.d 9:b22e1d7c:::benchmark_data_smithi017_543597_object34792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.552401748999998, 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.944761042, 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.955 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118583 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.956 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364266+0000", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364271+0000", 2023-09-23T15:29:15.957 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0320000000000003e-06 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364281+0000", 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8560000000000002e-06 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364286+0000", 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3299999999999998e-06 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.958 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364340+0000", 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3464999999999997e-05 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364349+0000", 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1099999999999992e-06 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.959 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298613+0000", 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93426368199999998 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298716+0000", 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010302999999999999 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298806+0000", 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.056e-05 2023-09-23T15:29:15.960 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305553+0000", 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067469649999999997 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305618+0000", 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5029000000000004e-05 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.961 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308931+0000", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033124449999999998 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308986+0000", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4809999999999999e-05 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309029+0000", 2023-09-23T15:29:15.962 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3118000000000003e-05 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118587 9.d 9:b46673d5:::benchmark_data_smithi017_543597_object34796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.545836104000003, 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93829474099999999, 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.963 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118587 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.964 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370832+0000", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370837+0000", 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5280000000000003e-06 2023-09-23T15:29:15.965 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370850+0000", 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2576e-05 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370856+0000", 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5419999999999996e-06 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.966 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370909+0000", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3155999999999997e-05 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370915+0000", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5300000000000002e-06 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301114+0000", 2023-09-23T15:29:15.967 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.930198525 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301253+0000", 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013897099999999999 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301389+0000", 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135814 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.968 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305605+0000", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042168529999999996 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308882+0000", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032768129999999999 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309075+0000", 2023-09-23T15:29:15.969 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000192296 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309105+0000", 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0505e-05 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309128+0000", 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2966999999999999e-05 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.970 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118593 9.d 9:b3ef2821:::benchmark_data_smithi017_543597_object34802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.537137592000001, 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.934446412, 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.971 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118593 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.972 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379530+0000", 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379535+0000", 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4610000000000001e-06 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.973 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379545+0000", 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379550+0000", 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2059999999999998e-06 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379683+0000", 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013354299999999999 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.974 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379695+0000", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1833e-05 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303454+0000", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92375848400000005 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303581+0000", 2023-09-23T15:29:15.975 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127126 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303717+0000", 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136534 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305643+0000", 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019254490000000001 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.976 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308885+0000", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032424519999999998 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313893+0000", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050071999999999998 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313940+0000", 2023-09-23T15:29:15.977 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6974000000000001e-05 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313978+0000", 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8679e-05 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118617 9.d 9:b0bafb86:::benchmark_data_smithi017_543597_object34826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.978 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.517373466999999, 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91924088900000001, 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118617 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.979 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399295+0000", 2023-09-23T15:29:15.980 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399305+0000", 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0543999999999999e-05 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399318+0000", 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.325e-05 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.981 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399326+0000", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9670000000000001e-06 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399399+0000", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2344999999999999e-05 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399412+0000", 2023-09-23T15:29:15.982 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3570999999999999e-05 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305695+0000", 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90628282500000001 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305769+0000", 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3960000000000003e-05 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.983 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305840+0000", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0525999999999995e-05 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309148+0000", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033085340000000001 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309185+0000", 2023-09-23T15:29:15.984 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6806e-05 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318447+0000", 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092622610000000008 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318496+0000", 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8952999999999997e-05 2023-09-23T15:29:15.985 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318537+0000", 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0771e-05 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118636 9.d 9:be5e9593:::benchmark_data_smithi017_543597_object34845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:29:15.986 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.492606856000002, 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89944873700000005, 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118636 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:29:15.987 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424061+0000", 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.988 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424066+0000", 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7740000000000004e-06 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424078+0000", 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2053e-05 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.989 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424084+0000", 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7379999999999999e-06 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424159+0000", 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5166999999999994e-05 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424170+0000", 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1090000000000001e-05 2023-09-23T15:29:15.990 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307472+0000", 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88330215099999998 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307554+0000", 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1878000000000004e-05 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.991 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307626+0000", 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1649000000000005e-05 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309193+0000", 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001566858 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313853+0000", 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046608609999999997 2023-09-23T15:29:15.992 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323430+0000", 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095760589999999996 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323476+0000", 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6399000000000001e-05 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:15.993 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323511+0000", 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5370000000000002e-05 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118669 9.d 9:b914bc60:::benchmark_data_smithi017_543597_object34878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.451419708000003, 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.85838970000000003, 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:15.994 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118669 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.995 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465248+0000", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465255+0000", 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9920000000000001e-06 2023-09-23T15:29:15.996 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465273+0000", 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8357999999999999e-05 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465282+0000", 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.997 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465349+0000", 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6883000000000003e-05 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465360+0000", 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1181e-05 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:15.998 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309219+0000", 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.84385860999999995 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309293+0000", 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3689000000000005e-05 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309370+0000", 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7571999999999995e-05 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:15.999 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313995+0000", 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004624548 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314027+0000", 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2759999999999998e-05 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.000 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323570+0000", 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095423469999999996 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323608+0000", 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8599000000000001e-05 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323639+0000", 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1059999999999997e-05 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.001 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118722 9.d 9:bde41141:::benchmark_data_smithi017_543597_object34931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.385233215, 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80511723000000002, 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.002 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118722 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.003 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531435+0000", 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531441+0000", 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:29:16.004 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531458+0000", 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7099e-05 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531466+0000", 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3810000000000001e-06 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.005 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532385+0000", 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00091978700000000004 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532394+0000", 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311491+0000", 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77909717099999998 2023-09-23T15:29:16.006 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311622+0000", 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013045400000000001 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311748+0000", 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012667700000000001 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.007 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314036+0000", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022880510000000001 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318410+0000", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043731940000000004 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336468+0000", 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018058586000000001 2023-09-23T15:29:16.008 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336516+0000", 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7856999999999999e-05 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336553+0000", 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7351999999999997e-05 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.009 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118729 9.d 9:b8e345b2:::benchmark_data_smithi017_543597_object34938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.378912286999999, 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.799062563, 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118729 2023-09-23T15:29:16.010 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.011 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537756+0000", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537763+0000", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0600000000000002e-06 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537777+0000", 2023-09-23T15:29:16.012 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4175999999999999e-05 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537782+0000", 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537847+0000", 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5366999999999999e-05 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.013 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537854+0000", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5139999999999998e-06 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314070+0000", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77621656999999999 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314182+0000", 2023-09-23T15:29:16.014 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011181200000000001 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314270+0000", 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7640999999999997e-05 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323357+0000", 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.009087108 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.015 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323363+0000", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3090000000000001e-06 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336742+0000", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013378594000000001 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336784+0000", 2023-09-23T15:29:16.016 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2333000000000003e-05 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336820+0000", 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5672e-05 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118744 9.d 9:b54a00db:::benchmark_data_smithi017_543597_object34953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.017 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.363667624999998, 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.78398909900000002, 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118744 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.018 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553000+0000", 2023-09-23T15:29:16.019 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553006+0000", 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4979999999999997e-06 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553019+0000", 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2855e-05 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.020 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553024+0000", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.271e-06 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553069+0000", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.494e-05 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553077+0000", 2023-09-23T15:29:16.021 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6000000000000001e-06 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.315967+0000", 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76288994200000004 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316067+0000", 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010011 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.022 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316183+0000", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0001165 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323368+0000", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071846490000000004 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323401+0000", 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2699e-05 2023-09-23T15:29:16.023 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336915+0000", 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013514884 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336958+0000", 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2064000000000002e-05 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.024 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336991+0000", 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3380999999999998e-05 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118748 9.d 9:b5a57405:::benchmark_data_smithi017_543597_object34957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.361822875999998, 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79383968999999999, 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.025 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118748 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.026 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554845+0000", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554853+0000", 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7870000000000002e-06 2023-09-23T15:29:16.027 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554866+0000", 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3534e-05 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554874+0000", 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3470000000000001e-06 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.028 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554929+0000", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5269999999999998e-05 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554938+0000", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318565+0000", 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76362680500000002 2023-09-23T15:29:16.029 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318662+0000", 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6918999999999998e-05 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318774+0000", 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112495 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.030 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323373+0000", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045984939999999998 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323543+0000", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017058199999999999 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348594+0000", 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025050833000000002 2023-09-23T15:29:16.031 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348644+0000", 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0266000000000003e-05 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348686+0000", 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1925999999999998e-05 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.032 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118765 9.d 9:b25beba1:::benchmark_data_smithi017_543597_object34974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.343343976, 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77546686300000001, 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.033 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118765 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.034 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573324+0000", 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573331+0000", 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4259999999999997e-06 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.035 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573342+0000", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0403e-05 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573348+0000", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1800000000000001e-06 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573421+0000", 2023-09-23T15:29:16.036 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3468999999999997e-05 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573431+0000", 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5540000000000003e-06 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321037+0000", 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.747606306 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.037 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321179+0000", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014166 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321251+0000", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2272000000000002e-05 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323671+0000", 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024195829999999999 2023-09-23T15:29:16.038 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327602+0000", 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039315790000000002 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348718+0000", 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021115821999999999 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.039 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348757+0000", 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9100999999999997e-05 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348792+0000", 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4980000000000001e-05 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118776 9.d 9:b496d689:::benchmark_data_smithi017_543597_object34985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.040 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.324994672999999, 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76259006200000001, 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118776 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.041 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591673+0000", 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.042 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591678+0000", 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2259999999999996e-06 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591693+0000", 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4647000000000001e-05 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.043 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591699+0000", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4659999999999996e-06 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591758+0000", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8990000000000003e-05 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591766+0000", 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1250000000000004e-06 2023-09-23T15:29:16.044 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323691+0000", 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.73192517599999996 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323786+0000", 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5377000000000005e-05 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.045 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323892+0000", 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105882 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.332104+0000", 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082115420000000005 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.046 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.333996+0000", 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018925509999999999 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.047 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354164+0000", 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020168175 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354220+0000", 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5327000000000001e-05 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354265+0000", 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5158999999999999e-05 2023-09-23T15:29:16.048 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118785 9.d 9:b48c076d:::benchmark_data_smithi017_543597_object34994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.312197079000001, 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74991449799999998, 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.049 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118785 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:29:16.050 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604471+0000", 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604477+0000", 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.806e-06 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.051 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604488+0000", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0818999999999999e-05 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604494+0000", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0739999999999998e-06 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604547+0000", 2023-09-23T15:29:16.052 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3557999999999998e-05 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604557+0000", 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7880000000000001e-06 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325681+0000", 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72112381699999994 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.053 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325761+0000", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0216000000000006e-05 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325859+0000", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8181000000000001e-05 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336439+0000", 2023-09-23T15:29:16.054 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010580001 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336654+0000", 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000215183 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354304+0000", 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017649116999999999 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.055 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354353+0000", 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9005000000000003e-05 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354387+0000", 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4282999999999997e-05 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.056 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118797 9.d 9:b088cc93:::benchmark_data_smithi017_543597_object35006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 52.284091381000003, 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72191496499999996, 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118797 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.057 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.058 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632576+0000", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632584+0000", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4419999999999998e-06 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632601+0000", 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7013000000000001e-05 2023-09-23T15:29:16.059 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632610+0000", 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632675+0000", 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4850000000000004e-05 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.060 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632689+0000", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4800000000000001e-05 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327685+0000", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69499560800000004 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327806+0000", 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012095999999999999 2023-09-23T15:29:16.061 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327937+0000", 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013145299999999999 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336643+0000", 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0087055399999999995 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.062 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336662+0000", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.933e-05 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354421+0000", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017759161999999998 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354461+0000", 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9267000000000002e-05 2023-09-23T15:29:16.063 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354493+0000", 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2258999999999997e-05 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:29:16.064 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:29:16.065 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:16.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:16.166 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:16.273 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:16.332 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:16.376 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:16.388 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:16.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:16.414 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:16.436 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:16.474 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:16.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:16.527 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:16.568 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:16.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:16.692 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:16.724 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.725 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:29:16.739 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:16.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:16.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:16.745 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:16 smithi017 ceph-mon[215646]: pgmap v1183: 119 pgs: 119 active+clean; 865 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T15:29:16.745 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1669271132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:16.750 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:16 smithi017 ceph-mon[222878]: pgmap v1183: 119 pgs: 119 active+clean; 865 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T15:29:16.750 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1669271132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:16.845 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:16 smithi027 ceph-mon[195233]: pgmap v1183: 119 pgs: 119 active+clean; 865 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2023-09-23T15:29:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1669271132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:16.915 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:17.177 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:29:17.418 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:17.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:17.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:17.533 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:17.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:29:17.580 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:17.673 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:17.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:17.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:17.817 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:18.100 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:18.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:18.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:18.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:18.285 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:18] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T15:29:18.286 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:18.442 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:18.457 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:18.558 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:18.559 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:18 smithi017 ceph-mon[215646]: pgmap v1184: 119 pgs: 119 active+clean; 780 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T15:29:18.607 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:18.689 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:18.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:18.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:18.710 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:18.760 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:18.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:18.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:18 smithi027 ceph-mon[195233]: pgmap v1184: 119 pgs: 119 active+clean; 780 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T15:29:18.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:18 smithi017 ceph-mon[222878]: pgmap v1184: 119 pgs: 119 active+clean; 780 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 773 op/s 2023-09-23T15:29:18.882 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:19.120 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:19.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:19.132 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:19.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:19.269 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:19.332 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:19.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:19.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:19.440 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:19.591 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.625 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.36273 -> 6.36273 (expected baseline 6.24875) 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 31 -> 31 pgs (0.694678 -> 0.694678 * mean) 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.7 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0086 2023-09-23T15:29:19.714 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0863 2023-09-23T15:29:19.726 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:29:19.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:19.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:19.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[215646]: pgmap v1185: 119 pgs: 119 active+clean; 737 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:19.776 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:19.776 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:19.776 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:19 smithi017 ceph-mon[222878]: pgmap v1185: 119 pgs: 119 active+clean; 737 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:20.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:29:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:19 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:19 smithi027 ceph-mon[195233]: pgmap v1185: 119 pgs: 119 active+clean; 737 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:21.090 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:20 smithi017 ceph-mon[222878]: from='client.36522 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:21.092 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:20 smithi017 ceph-mon[215646]: from='client.36522 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:20 smithi027 ceph-mon[195233]: from='client.36522 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:22.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:21 smithi017 ceph-mon[222878]: pgmap v1186: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:22.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:21 smithi017 ceph-mon[215646]: pgmap v1186: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:22.081 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:22.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:22.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:21 smithi027 ceph-mon[195233]: pgmap v1186: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:22.167 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:29:22.167 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T15:29:22.190 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:22.319 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:22.739 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:22.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:22.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:22.848 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:22.883 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:22.963 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:23.166 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:23.166 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:23.357 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:23.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:23.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:23.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:23.528 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:24.159 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:24.366 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:29:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:24 smithi027 ceph-mon[195233]: pgmap v1187: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1279137166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:24.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:24 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:24.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[215646]: pgmap v1187: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1279137166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[222878]: pgmap v1187: 119 pgs: 119 active+clean; 548 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 746 op/s 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:24.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1279137166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:24.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:24 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:29:25.233 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:25.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:25.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:25.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:29:25.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:25.439 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:25.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:25.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: osdmap e409: 8 total, 8 up, 8 in 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:29:25.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:29:25.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: osdmap e409: 8 total, 8 up, 8 in 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:25.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: osdmap e409: 8 total, 8 up, 8 in 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: failed to encode map e409 with expected crc 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:29:25.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:25.885 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:25.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:25.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:25.995 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:26.060 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:26.141 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:26.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:26.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:26.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:26.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:26.574 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:26.608 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:26.647 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:26.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:26.661 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:26.695 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:26.729 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:26.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:26.778 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:26.796 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:26.798 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:26.799 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.800 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:29:26.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:26 smithi017 ceph-mon[215646]: pgmap v1189: 119 pgs: 119 active+clean; 391 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T15:29:26.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3309771065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:26.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:26 smithi017 ceph-mon[222878]: pgmap v1189: 119 pgs: 119 active+clean; 391 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T15:29:26.823 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3309771065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:26.840 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:26.847 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:26 smithi027 ceph-mon[195233]: pgmap v1189: 119 pgs: 119 active+clean; 391 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 757 op/s 2023-09-23T15:29:26.847 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3309771065' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:26.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:26.948 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:27.026 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:27.825 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:27.899 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:27.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:27.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:28.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:28.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:29:28.089 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:28] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.43.0" 2023-09-23T15:29:28.094 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:28.420 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:28.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:28.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:28.541 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:28.580 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:28.661 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:28.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:28.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:28 smithi017 ceph-mon[222878]: pgmap v1190: 119 pgs: 119 active+clean; 300 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:28.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:28 smithi017 ceph-mon[215646]: pgmap v1190: 119 pgs: 119 active+clean; 300 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:28.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:28.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:28.834 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:28 smithi027 ceph-mon[195233]: pgmap v1190: 119 pgs: 119 active+clean; 300 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 768 op/s 2023-09-23T15:29:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:29.044 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:29:29.044 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.625 2023-09-23T15:29:29.044 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.36273 -> 6.36273 (expected baseline 6.24875) 2023-09-23T15:29:29.044 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 31 -> 31 pgs (0.694678 -> 0.694678 * mean) 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.7 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0066 2023-09-23T15:29:29.045 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0661 2023-09-23T15:29:29.083 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:29:29.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:29.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:29.274 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:29.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:29.456 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:29.564 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:29.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:29.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:29.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:29.712 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:29.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:29:30.108 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:30.123 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:29:30.123 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T15:29:30.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:30.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:30.225 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:30.267 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:30.335 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:30.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:30.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:30.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:30.501 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:30.570 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:30.628 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 83788 objects 2023-09-23T15:29:30.628 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :115.602 2023-09-23T15:29:30.645 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:29:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:29:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:29:30.645 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:29:30.645 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:29:30.732 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:30.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:30.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:30.835 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:30.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:30 smithi027 ceph-mon[195233]: pgmap v1191: 119 pgs: 119 active+clean; 253 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T15:29:30.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:30 smithi027 ceph-mon[195233]: from='client.46445 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:30.862 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:30.905 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:30.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:30.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:30.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:30 smithi017 ceph-mon[215646]: pgmap v1191: 119 pgs: 119 active+clean; 253 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T15:29:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:30 smithi017 ceph-mon[215646]: from='client.46445 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:30.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:30 smithi017 ceph-mon[222878]: pgmap v1191: 119 pgs: 119 active+clean; 253 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2023-09-23T15:29:30.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:30 smithi017 ceph-mon[222878]: from='client.46445 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:31.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:31.016 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T15:29:31.016 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T15:29:31.040 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:31.083 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:31.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:31.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:31.191 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:31.218 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:31.261 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:31.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:31.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:31.367 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:31.396 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:31.726 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:31.744 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:29:31.745 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:31.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:31.849 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:31.877 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:31.921 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:31.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:31.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:32.028 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:32.057 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:32.154 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:32.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:32.158 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:32.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:32.285 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:32.736 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:32.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:32.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:32.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:32.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:32.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:32 smithi027 ceph-mon[195233]: pgmap v1192: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:32.861 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:32.873 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:32.923 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:32.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:32 smithi017 ceph-mon[215646]: pgmap v1192: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:32.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:32 smithi017 ceph-mon[222878]: pgmap v1192: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:33.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:33.057 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:33.104 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:33.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:33.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:33.239 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:33.283 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:33.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:33.290 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:33.390 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:33.417 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:33.462 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:33.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:33.566 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:33.592 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:33.636 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:33.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:33.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:33.743 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:33.770 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:33.813 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:33.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:33.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:33.918 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:33.955 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:34.582 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:29:34.723 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:34.813 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:34.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[215646]: pgmap v1193: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:34.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1999966650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[222878]: pgmap v1193: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:34.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1999966650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:34 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:34.818 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:34.849 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:29:34 smithi027 conmon[174777]: logger=cleanup t=2023-09-23T15:29:34.483957118Z level=info msg="Completed cleanup jobs" duration=68.009781ms 2023-09-23T15:29:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:34 smithi027 ceph-mon[195233]: pgmap v1193: 119 pgs: 119 active+clean; 47 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:29:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1999966650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:34 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:34.918 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:34.920 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:34.944 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:34.951 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:34.959 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:34.981 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:35.001 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:35.002 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:35.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:35.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:35.020 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:35.039 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:35.059 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:35.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.060 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.061 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.061 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.061 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.061 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:29:35.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:35.139 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:35.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: osdmap e410: 8 total, 8 up, 8 in 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/845764133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:29:35.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: osdmap e410: 8 total, 8 up, 8 in 2023-09-23T15:29:35.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/845764133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:29:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:29:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: failed to encode map e410 with expected crc 2023-09-23T15:29:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: osdmap e410: 8 total, 8 up, 8 in 2023-09-23T15:29:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/845764133' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:36.651 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:36.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:36.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:36.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:36 smithi017 ceph-mon[215646]: failed to encode map e410 with expected crc 2023-09-23T15:29:36.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:36 smithi017 ceph-mon[215646]: pgmap v1195: 119 pgs: 119 active+clean; 20 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T15:29:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:36 smithi017 ceph-mon[222878]: failed to encode map e410 with expected crc 2023-09-23T15:29:36.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:36 smithi017 ceph-mon[222878]: pgmap v1195: 119 pgs: 119 active+clean; 20 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T15:29:36.756 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:36.763 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:29:36.763 INFO:teuthology.orchestra.run.smithi017.stdout:avg 44.625 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.36273 -> 6.36273 (expected baseline 6.24875) 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 31 -> 31 pgs (0.694678 -> 0.694678 * mean) 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.7 with 52 -> 52 pgs (1.16527 -> 1.16527 * mean) 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0055 2023-09-23T15:29:36.764 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0549 2023-09-23T15:29:36.766 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:29:36.789 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:36.839 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:36.845 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:36 smithi027 ceph-mon[195233]: failed to encode map e410 with expected crc 2023-09-23T15:29:36.845 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:36 smithi027 ceph-mon[195233]: pgmap v1195: 119 pgs: 119 active+clean; 20 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 592 op/s 2023-09-23T15:29:36.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:36.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:36.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:36.975 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:37.022 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:37.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:37.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:37.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:37.155 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:37.200 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:37.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:37.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:37.306 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:37.338 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:37.756 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:37.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:37.758 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:37.852 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:29:37.852 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T15:29:37.860 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:37.929 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:38.133 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:38.134 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:38.163 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:38] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.43.0" 2023-09-23T15:29:38.215 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:29:38.436 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:38.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:38.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:29:38.540 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:38.583 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_4' removed 2023-09-23T15:29:38.595 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:38.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[215646]: from='client.46463 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:38.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[215646]: pgmap v1196: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 471 op/s 2023-09-23T15:29:38.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:29:38.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[222878]: from='client.46463 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:38.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[222878]: pgmap v1196: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 471 op/s 2023-09-23T15:29:38.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:29:38.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:38 smithi027 ceph-mon[195233]: from='client.46463 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:38.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:38 smithi027 ceph-mon[195233]: pgmap v1196: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 471 op/s 2023-09-23T15:29:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:29:39.183 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:29:39.254 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:39.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:39.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:39.335 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:39.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:39.361 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T15:29:39.361 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T15:29:39.362 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T15:29:39.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T15:29:39.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:39.442 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:39.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:39.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:39.547 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:39.588 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:39.644 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:39.724 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:39.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:39.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:39.829 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:39.861 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:39.884 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[215646]: failed to encode map e411 with expected crc 2023-09-23T15:29:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[215646]: failed to encode map e411 with expected crc 2023-09-23T15:29:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[215646]: osdmap e411: 8 total, 8 up, 8 in 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[215646]: failed to encode map e411 with expected crc 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[222878]: failed to encode map e411 with expected crc 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[222878]: failed to encode map e411 with expected crc 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[222878]: osdmap e411: 8 total, 8 up, 8 in 2023-09-23T15:29:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:39 smithi017 ceph-mon[222878]: failed to encode map e411 with expected crc 2023-09-23T15:29:40.009 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:40.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:40.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:39 smithi027 ceph-mon[195233]: failed to encode map e411 with expected crc 2023-09-23T15:29:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:39 smithi027 ceph-mon[195233]: failed to encode map e411 with expected crc 2023-09-23T15:29:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:39 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4059832119' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:29:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:39 smithi027 ceph-mon[195233]: osdmap e411: 8 total, 8 up, 8 in 2023-09-23T15:29:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:39 smithi027 ceph-mon[195233]: failed to encode map e411 with expected crc 2023-09-23T15:29:40.115 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:40.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:40.191 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:40.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:40.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:40.295 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:40.323 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:40.510 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:40.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:40.517 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:40.618 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:40.647 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:40.694 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:40.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:40.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:40.798 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:40.829 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:40.929 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:40.931 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:40 smithi017 ceph-mon[215646]: pgmap v1198: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:40.932 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:40 smithi017 ceph-mon[222878]: pgmap v1198: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:40.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:40.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:41.033 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:41.061 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:41.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:40 smithi027 ceph-mon[195233]: pgmap v1198: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:41.517 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 309 (0%) 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:avg 38.625 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 5.09749 -> 5.09749 (expected baseline 5.81351) 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 28 -> 28 pgs (0.724919 -> 0.724919 * mean) 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 44 -> 44 pgs (1.13916 -> 1.13916 * mean) 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:29:42.032 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2023-09-23T15:29:42.033 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:29:42.033 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0053 2023-09-23T15:29:42.033 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0535 2023-09-23T15:29:42.034 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:29:42.266 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:42.274 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:42.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:42.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:42.380 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:42.415 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:42.619 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:42.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:42.622 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:42.723 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:42.754 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:42.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:42 smithi017 ceph-mon[215646]: pgmap v1199: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:42.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:42 smithi017 ceph-mon[222878]: pgmap v1199: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:42 smithi027 ceph-mon[195233]: pgmap v1199: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:43.574 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:43.575 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:43.575 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:43.575 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:43.575 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:43.584 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:43.604 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:43.624 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:43.625 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.626 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:29:43.648 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[222878]: from='client.36567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1185463690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[215646]: from='client.36567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:43.649 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1185463690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:43 smithi027 ceph-mon[195233]: from='client.36567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:29:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:44.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1185463690' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:44.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:44.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:44.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:44.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:44.443 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:44.642 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:44.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:44.655 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:29:44.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[215646]: pgmap v1200: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2596604554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[222878]: pgmap v1200: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2596604554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.746 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:44.778 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:44.830 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:44.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:44.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:44.936 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:44.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:44 smithi027 ceph-mon[195233]: pgmap v1200: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:44.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2596604554' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:29:44.964 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:45.011 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:45.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:45.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:45.117 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:45.144 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:45.189 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:45.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:45.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:45.324 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:45.368 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:45.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:45.373 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:45.474 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:45.501 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:45.544 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:45.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:45.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:45.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:45.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:45.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[215646]: failed to encode map e412 with expected crc 2023-09-23T15:29:45.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[215646]: failed to encode map e412 with expected crc 2023-09-23T15:29:45.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:45.658 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[215646]: osdmap e412: 8 total, 8 up, 8 in 2023-09-23T15:29:45.685 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:45.689 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:29:45.690 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:29:45.690 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:29:45.690 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:29:45.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:45.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[222878]: failed to encode map e412 with expected crc 2023-09-23T15:29:45.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[222878]: failed to encode map e412 with expected crc 2023-09-23T15:29:45.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:45.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:45 smithi017 ceph-mon[222878]: osdmap e412: 8 total, 8 up, 8 in 2023-09-23T15:29:46.002 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:46.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:46.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:45 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:45 smithi027 ceph-mon[195233]: failed to encode map e412 with expected crc 2023-09-23T15:29:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:45 smithi027 ceph-mon[195233]: failed to encode map e412 with expected crc 2023-09-23T15:29:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:45 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:29:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:45 smithi027 ceph-mon[195233]: osdmap e412: 8 total, 8 up, 8 in 2023-09-23T15:29:46.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:46.133 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:46.533 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:46.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:46.593 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:29:46.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:46.666 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:46.769 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:46.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:46.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:46.874 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:46.906 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:46 smithi017 ceph-mon[215646]: pgmap v1202: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:46.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:46 smithi017 ceph-mon[222878]: pgmap v1202: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:46.955 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:46.961 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:46 smithi027 ceph-mon[195233]: pgmap v1202: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:46.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:46.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:46.974 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:47.063 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:47.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:29:47.091 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:47.137 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:47.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:47.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:47.243 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:47.270 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:47.314 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:47.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:47.419 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:47.446 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:47.491 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:47.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:47.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:47.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:47.625 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:47.664 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T15:29:47.664 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T15:29:47.677 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' created 2023-09-23T15:29:48.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T15:29:48.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:29:48.365 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:48] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.43.0" 2023-09-23T15:29:48.435 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:48.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:48.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:48.538 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:29:48.568 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:48.620 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:48.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T15:29:48.686 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:29:48.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:48.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:29:48.727 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:29:48.727 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:29:48.788 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:48.793 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:48.826 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:48.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[215646]: pgmap v1203: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:48.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[215646]: failed to encode map e413 with expected crc 2023-09-23T15:29:48.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[215646]: failed to encode map e413 with expected crc 2023-09-23T15:29:48.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T15:29:48.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[215646]: osdmap e413: 8 total, 8 up, 8 in 2023-09-23T15:29:48.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[222878]: pgmap v1203: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:48.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[222878]: failed to encode map e413 with expected crc 2023-09-23T15:29:48.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[222878]: failed to encode map e413 with expected crc 2023-09-23T15:29:48.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T15:29:48.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:48 smithi017 ceph-mon[222878]: osdmap e413: 8 total, 8 up, 8 in 2023-09-23T15:29:48.881 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:48.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:48.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:48.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:49.016 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:48 smithi027 ceph-mon[195233]: pgmap v1203: 103 pgs: 103 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:49.016 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:48 smithi027 ceph-mon[195233]: failed to encode map e413 with expected crc 2023-09-23T15:29:49.017 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:48 smithi027 ceph-mon[195233]: failed to encode map e413 with expected crc 2023-09-23T15:29:49.017 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2902541366' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T15:29:49.017 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:48 smithi027 ceph-mon[195233]: osdmap e413: 8 total, 8 up, 8 in 2023-09-23T15:29:49.019 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:49.071 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:49.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:49.145 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:29:49.145 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:29:49.145 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:29:49.145 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:29:49.145 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:29:49.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:49.207 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:49.255 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:49.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:49.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:49.363 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:49.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:49.436 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:49.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:49.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:49.542 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:49.569 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:49.613 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:49.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:49.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:49.719 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:49.753 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: osdmap e414: 8 total, 8 up, 8 in 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[215646]: pgmap v1206: 119 pgs: 14 unknown, 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: osdmap e414: 8 total, 8 up, 8 in 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: failed to encode map e414 with expected crc 2023-09-23T15:29:49.857 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:49 smithi017 ceph-mon[222878]: pgmap v1206: 119 pgs: 14 unknown, 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:49.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:29:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: failed to encode map e414 with expected crc 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: failed to encode map e414 with expected crc 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: osdmap e414: 8 total, 8 up, 8 in 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: failed to encode map e414 with expected crc 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: failed to encode map e414 with expected crc 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: failed to encode map e414 with expected crc 2023-09-23T15:29:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:49 smithi027 ceph-mon[195233]: pgmap v1206: 119 pgs: 14 unknown, 105 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:50.359 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:50.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:50.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:50.463 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:50.492 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:50.691 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:50.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:50.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:50.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[215646]: failed to encode map e415 with expected crc 2023-09-23T15:29:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[215646]: failed to encode map e415 with expected crc 2023-09-23T15:29:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[215646]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T15:29:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[222878]: failed to encode map e415 with expected crc 2023-09-23T15:29:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[222878]: failed to encode map e415 with expected crc 2023-09-23T15:29:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:50 smithi017 ceph-mon[222878]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T15:29:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:50 smithi027 ceph-mon[195233]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:50 smithi027 ceph-mon[195233]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:50 smithi027 ceph-mon[195233]: osdmap e415: 8 total, 8 up, 8 in 2023-09-23T15:29:51.305 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:51.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:51.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:51.410 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:51.443 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:51.482 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:51.493 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:51.495 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:51.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:51.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:51.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:51.602 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:51.613 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:51.630 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:51.638 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:51.659 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:51.677 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:51.679 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[215646]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[222878]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[222878]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[222878]: pgmap v1208: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:51.681 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2735254541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:51.682 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:51.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:51.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:51.702 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:51.721 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:51.741 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.743 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.744 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.744 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:29:51.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:51.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:51 smithi027 ceph-mon[195233]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:51 smithi027 ceph-mon[195233]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:51 smithi027 ceph-mon[195233]: pgmap v1208: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:51.810 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2735254541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:51.812 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:51.856 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:51.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:51.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:51.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[215646]: failed to encode map e415 with expected crc 2023-09-23T15:29:51.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[215646]: pgmap v1208: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:51.933 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2735254541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:29:51.964 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:51.990 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:52.033 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:52.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:52.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:52.140 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:52.172 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:53.273 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:53.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:53.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:53.378 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:53.407 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:53.452 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:53.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:53.455 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:53.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:53.589 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:53.640 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:53.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:53.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:53.748 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:53.775 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:53.848 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:53.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:53.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:53.951 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:53.981 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:54.179 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:54.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:54.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:54.268 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:54.268 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:54.282 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:54.314 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:54.472 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:54.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:54.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:54.576 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:54.589 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:29:54.609 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:54.661 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:54.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:54.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:54.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:54 smithi017 ceph-mon[215646]: pgmap v1209: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:54.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:54.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:54 smithi017 ceph-mon[222878]: pgmap v1209: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:54.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:54.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:29:54.793 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:54 smithi027 ceph-mon[195233]: pgmap v1209: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:54.793 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:29:54.796 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:54.843 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:29:54.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:54.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:29:54.948 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:54.977 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:55.020 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:55.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:55.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:55.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:55.155 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:55.199 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:55.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:55.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:55.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:29:55.304 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:55.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:55.448 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:55.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:55.514 INFO:teuthology.orchestra.run.smithi017.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T15:29:55.555 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1378028596' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.585 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1378028596' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.586 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.589 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:55.785 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:55.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:55.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:55.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: failed to encode map e416 with expected crc 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/546445474' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: osdmap e416: 8 total, 8 up, 8 in 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1378028596' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:29:55.888 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:29:55.917 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:55.963 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:29:55.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:55.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:29:55.975 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:29:56.070 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:56.097 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:56.210 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:56.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:56.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:56.250 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:56.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:56.344 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:56.403 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:56.491 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:56.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:56.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:56.596 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: pgmap v1211: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: failed to encode map e416 with expected crc 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[215646]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.622 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: pgmap v1211: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: failed to encode map e416 with expected crc 2023-09-23T15:29:56.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:56 smithi017 ceph-mon[222878]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.624 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:56.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: pgmap v1211: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: osdmap e417: 8 total, 8 up, 8 in 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: failed to encode map e416 with expected crc 2023-09-23T15:29:56.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:56 smithi027 ceph-mon[195233]: failed to encode map e417 with expected crc 2023-09-23T15:29:56.874 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:29:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:56.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:29:56.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:29:57.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:57.062 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:29:57.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:57.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:29:57.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:57.199 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:57.211 DEBUG:teuthology.orchestra.run.smithi017:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T15:29:57.212 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T15:29:57.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:29:57.214 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T15:29:57.214 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T15:29:57.246 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:57.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:57.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:57.264 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:hints = 1 2023-09-23T15:29:57.265 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T15:29:57.265 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object prefix: benchmark_data_smithi017_604517 2023-09-23T15:29:57.265 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:29:57.265 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 0 0 0 0 0 0 - 0 2023-09-23T15:29:57.354 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:29:57.400 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:57.485 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:29:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:57.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:29:57.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:29:57.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:29:57.654 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:58.115 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:29:58.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:58.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:29:58.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:58.270 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 1 16 3357 3341 207.677 208.812 0.00484058 0.00479571 2023-09-23T15:29:58.311 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:29:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:29:58] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.43.0" 2023-09-23T15:29:58.324 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:58.417 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:58.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:58.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:58.510 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:29:58.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:58.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[222878]: pgmap v1213: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:58.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:58.598 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:58.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[215646]: pgmap v1213: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:58.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:58.599 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:58.602 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:58.706 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:29:58.706 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:29:58.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:58 smithi027 ceph-mon[195233]: pgmap v1213: 119 pgs: 119 active+clean; 4.6 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:29:58.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:29:58.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:29:58.912 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:29:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:29:58.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:29:59.018 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:29:59.098 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:29:59.239 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:29:59.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 2 16 6127 6111 190.44 173.125 0.00308776 0.00524122 2023-09-23T15:29:59.273 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:29:59.315 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:29:59.357 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:29:59.395 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:29:59.432 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:29:59.472 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:29:59.511 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:29:59.520 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:30:00.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:59 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:00.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:59 smithi027 ceph-mon[195233]: pgmap v1214: 119 pgs: 119 active+clean; 39 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-09-23T15:30:00.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:29:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2201793104' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[215646]: pgmap v1214: 119 pgs: 119 active+clean; 39 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2201793104' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[222878]: pgmap v1214: 119 pgs: 119 active+clean; 39 MiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 4.3 MiB/s wr, 69 op/s 2023-09-23T15:30:00.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:29:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2201793104' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:00.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 3 16 9400 9384 195.129 204.562 0.00125195 0.00506548 2023-09-23T15:30:01.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:00 smithi017 ceph-mon[215646]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:00 smithi017 ceph-mon[215646]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:00 smithi017 ceph-mon[222878]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:00 smithi017 ceph-mon[222878]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 4 16 12351 12335 192.455 184.438 0.00403966 0.00518793 2023-09-23T15:30:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:00 smithi027 ceph-mon[195233]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:00 smithi027 ceph-mon[195233]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T15:30:01.659 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:01.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:01.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:01.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:01.817 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:01.862 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:01.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:01.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:01.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:02.028 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:01 smithi027 ceph-mon[195233]: pgmap v1215: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T15:30:02.170 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:01 smithi017 ceph-mon[215646]: pgmap v1215: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T15:30:02.170 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:01 smithi017 ceph-mon[222878]: pgmap v1215: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 912 op/s 2023-09-23T15:30:02.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:30:02.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 5 16 14685 14669 183.146 145.875 0.00216144 0.00537103 2023-09-23T15:30:02.875 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:02.948 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:02.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:02.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:03.060 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:03.111 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:03.202 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:03.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:03.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 6 16 16600 16584 172.578 119.688 0.00848561 0.00578806 2023-09-23T15:30:03.310 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:03.358 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:03.450 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:03.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:03.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:03.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:03.625 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:03.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:03.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:03.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:03.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:03.984 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:04.072 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:04.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:04.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:04.202 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:30:04.204 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:30:04.207 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:04.252 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:04.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:04.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:04.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 7 16 18493 18477 164.829 118.312 0.00150203 0.00588944 2023-09-23T15:30:04.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:04.386 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:04.431 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:04.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:04.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:04.535 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:04.581 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:04 smithi027 ceph-mon[195233]: pgmap v1216: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T15:30:04.586 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:04.691 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:04.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:04 smithi017 ceph-mon[215646]: pgmap v1216: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T15:30:04.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:04 smithi017 ceph-mon[222878]: pgmap v1216: 119 pgs: 119 active+clean; 461 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 54 MiB/s wr, 869 op/s 2023-09-23T15:30:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:04.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:04.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:04.856 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:04.914 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:30:05.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 8 16 20256 20240 158.002 110.188 0.00457421 0.00626211 2023-09-23T15:30:05.474 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:05.549 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:05.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:05.602 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1216,"stamp":"2023-09-23T15:30:02.905355+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":483334067,"num_objects":7330,"num_object_clones":0,"num_object_copies":21990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7330,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":8209,"num_write_kb":480053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":159,"num_bytes_recovered":4466766,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":13326,"ondisk_log_size":13326,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":359,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5062196,"kb_used_data":1346996,"kb_used_omap":57,"kb_used_meta":3715078,"kb_avail":744899020,"statfs":{"total":767960285184,"available":762776596480,"internally_reserved":0,"allocated":1379323904,"data_stored":1366045088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":3804239897},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":478543872,"num_objects":7302,"num_object_clones":0,"num_object_copies":21906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7302,"num_write_kb":467328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.402746"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":764,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.476622+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:29:50.476622+0000","last_peered":"2023-09-23T15:29:50.476622+0000","last_clean":"2023-09-23T15:29:50.476622+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:29:50.476622+0000","last_undegraded":"2023-09-23T15:29:50.476622+0000","last_fullsized":"2023-09-23T15:29:50.476622+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506214+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:29:54.506214+0000","last_peered":"2023-09-23T15:29:54.506214+0000","last_clean":"2023-09-23T15:29:54.506214+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:29:54.506214+0000","last_undegraded":"2023-09-23T15:29:54.506214+0000","last_fullsized":"2023-09-23T15:29:54.506214+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":372,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225407+0000","last_change":"2023-09-23T15:27:23.313341+0000","last_active":"2023-09-23T15:29:55.225407+0000","last_peered":"2023-09-23T15:29:55.225407+0000","last_clean":"2023-09-23T15:29:55.225407+0000","last_became_active":"2023-09-23T15:27:23.313012+0000","last_became_peered":"2023-09-23T15:27:23.313012+0000","last_unstale":"2023-09-23T15:29:55.225407+0000","last_undegraded":"2023-09-23T15:29:55.225407+0000","last_fullsized":"2023-09-23T15:29:55.225407+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:17:06.833473+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":498,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477368+0000","last_change":"2023-09-23T15:28:55.922068+0000","last_active":"2023-09-23T15:29:50.477368+0000","last_peered":"2023-09-23T15:29:50.477368+0000","last_clean":"2023-09-23T15:29:50.477368+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:29:50.477368+0000","last_undegraded":"2023-09-23T15:29:50.477368+0000","last_fullsized":"2023-09-23T15:29:50.477368+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:55.921999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:55.921999+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:38.044680+0000","scrub_duration":0.0074994720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.504077+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:29:54.504077+0000","last_peered":"2023-09-23T15:29:54.504077+0000","last_clean":"2023-09-23T15:29:54.504077+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:29:54.504077+0000","last_undegraded":"2023-09-23T15:29:54.504077+0000","last_fullsized":"2023-09-23T15:29:54.504077+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":412,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.366069+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:29:50.366069+0000","last_peered":"2023-09-23T15:29:50.366069+0000","last_clean":"2023-09-23T15:29:50.366069+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:29:50.366069+0000","last_undegraded":"2023-09-23T15:29:50.366069+0000","last_fullsized":"2023-09-23T15:29:50.366069+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":552,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506280+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:29:54.506280+0000","last_peered":"2023-09-23T15:29:54.506280+0000","last_clean":"2023-09-23T15:29:54.506280+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:29:54.506280+0000","last_undegraded":"2023-09-23T15:29:54.506280+0000","last_fullsized":"2023-09-23T15:29:54.506280+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":408,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224719+0000","last_change":"2023-09-23T15:27:12.402468+0000","last_active":"2023-09-23T15:29:55.224719+0000","last_peered":"2023-09-23T15:29:55.224719+0000","last_clean":"2023-09-23T15:29:55.224719+0000","last_became_active":"2023-09-23T15:27:12.402003+0000","last_became_peered":"2023-09-23T15:27:12.402003+0000","last_unstale":"2023-09-23T15:29:55.224719+0000","last_undegraded":"2023-09-23T15:29:55.224719+0000","last_fullsized":"2023-09-23T15:29:55.224719+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:45.443108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":367,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.508187+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:29:54.508187+0000","last_peered":"2023-09-23T15:29:54.508187+0000","last_clean":"2023-09-23T15:29:54.508187+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:29:54.508187+0000","last_undegraded":"2023-09-23T15:29:54.508187+0000","last_fullsized":"2023-09-23T15:29:54.508187+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":685,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283340+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:29:55.283340+0000","last_peered":"2023-09-23T15:29:55.283340+0000","last_clean":"2023-09-23T15:29:55.283340+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:29:55.283340+0000","last_undegraded":"2023-09-23T15:29:55.283340+0000","last_fullsized":"2023-09-23T15:29:55.283340+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":366,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365819+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:29:50.365819+0000","last_peered":"2023-09-23T15:29:50.365819+0000","last_clean":"2023-09-23T15:29:50.365819+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:29:50.365819+0000","last_undegraded":"2023-09-23T15:29:50.365819+0000","last_fullsized":"2023-09-23T15:29:50.365819+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":406,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224768+0000","last_change":"2023-09-23T15:27:12.402484+0000","last_active":"2023-09-23T15:29:55.224768+0000","last_peered":"2023-09-23T15:29:55.224768+0000","last_clean":"2023-09-23T15:29:55.224768+0000","last_became_active":"2023-09-23T15:27:12.402209+0000","last_became_peered":"2023-09-23T15:27:12.402209+0000","last_unstale":"2023-09-23T15:29:55.224768+0000","last_undegraded":"2023-09-23T15:29:55.224768+0000","last_fullsized":"2023-09-23T15:29:55.224768+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T14:59:07.497544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":826,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.394764+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:29:50.394764+0000","last_peered":"2023-09-23T15:29:50.394764+0000","last_clean":"2023-09-23T15:29:50.394764+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:29:50.394764+0000","last_undegraded":"2023-09-23T15:29:50.394764+0000","last_fullsized":"2023-09-23T15:29:50.394764+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":433,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477360+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:29:50.477360+0000","last_peered":"2023-09-23T15:29:50.477360+0000","last_clean":"2023-09-23T15:29:50.477360+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:29:50.477360+0000","last_undegraded":"2023-09-23T15:29:50.477360+0000","last_fullsized":"2023-09-23T15:29:50.477360+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":366,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224961+0000","last_change":"2023-09-23T15:27:41.424467+0000","last_active":"2023-09-23T15:29:55.224961+0000","last_peered":"2023-09-23T15:29:55.224961+0000","last_clean":"2023-09-23T15:29:55.224961+0000","last_became_active":"2023-09-23T15:27:41.424098+0000","last_became_peered":"2023-09-23T15:27:41.424098+0000","last_unstale":"2023-09-23T15:29:55.224961+0000","last_undegraded":"2023-09-23T15:29:55.224961+0000","last_fullsized":"2023-09-23T15:29:55.224961+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:46.478911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":519,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477414+0000","last_change":"2023-09-23T15:28:59.984242+0000","last_active":"2023-09-23T15:29:50.477414+0000","last_peered":"2023-09-23T15:29:50.477414+0000","last_clean":"2023-09-23T15:29:50.477414+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:29:50.477414+0000","last_undegraded":"2023-09-23T15:29:50.477414+0000","last_fullsized":"2023-09-23T15:29:50.477414+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:59.984131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:59.984131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:34.209362+0000","scrub_duration":0.015840911999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":162,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283742+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:29:55.283742+0000","last_peered":"2023-09-23T15:29:55.283742+0000","last_clean":"2023-09-23T15:29:55.283742+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:29:55.283742+0000","last_undegraded":"2023-09-23T15:29:55.283742+0000","last_fullsized":"2023-09-23T15:29:55.283742+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":310,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.504656+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:29:54.504656+0000","last_peered":"2023-09-23T15:29:54.504656+0000","last_clean":"2023-09-23T15:29:54.504656+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:29:54.504656+0000","last_undegraded":"2023-09-23T15:29:54.504656+0000","last_fullsized":"2023-09-23T15:29:54.504656+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":567,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505439+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:29:54.505439+0000","last_peered":"2023-09-23T15:29:54.505439+0000","last_clean":"2023-09-23T15:29:54.505439+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:29:54.505439+0000","last_undegraded":"2023-09-23T15:29:54.505439+0000","last_fullsized":"2023-09-23T15:29:54.505439+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":83,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506766+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:29:54.506766+0000","last_peered":"2023-09-23T15:29:54.506766+0000","last_clean":"2023-09-23T15:29:54.506766+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:29:54.506766+0000","last_undegraded":"2023-09-23T15:29:54.506766+0000","last_fullsized":"2023-09-23T15:29:54.506766+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506956+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:29:54.506956+0000","last_peered":"2023-09-23T15:29:54.506956+0000","last_clean":"2023-09-23T15:29:54.506956+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:29:54.506956+0000","last_undegraded":"2023-09-23T15:29:54.506956+0000","last_fullsized":"2023-09-23T15:29:54.506956+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":727,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365111+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:29:50.365111+0000","last_peered":"2023-09-23T15:29:50.365111+0000","last_clean":"2023-09-23T15:29:50.365111+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:29:50.365111+0000","last_undegraded":"2023-09-23T15:29:50.365111+0000","last_fullsized":"2023-09-23T15:29:50.365111+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":166,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283297+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:29:55.283297+0000","last_peered":"2023-09-23T15:29:55.283297+0000","last_clean":"2023-09-23T15:29:55.283297+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:29:55.283297+0000","last_undegraded":"2023-09-23T15:29:55.283297+0000","last_fullsized":"2023-09-23T15:29:55.283297+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":416,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225200+0000","last_change":"2023-09-23T15:27:41.426159+0000","last_active":"2023-09-23T15:29:55.225200+0000","last_peered":"2023-09-23T15:29:55.225200+0000","last_clean":"2023-09-23T15:29:55.225200+0000","last_became_active":"2023-09-23T15:27:41.425781+0000","last_became_peered":"2023-09-23T15:27:41.425781+0000","last_unstale":"2023-09-23T15:29:55.225200+0000","last_undegraded":"2023-09-23T15:29:55.225200+0000","last_fullsized":"2023-09-23T15:29:55.225200+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:16.375308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":791,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.476915+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:29:50.476915+0000","last_peered":"2023-09-23T15:29:50.476915+0000","last_clean":"2023-09-23T15:29:50.476915+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:29:50.476915+0000","last_undegraded":"2023-09-23T15:29:50.476915+0000","last_fullsized":"2023-09-23T15:29:50.476915+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":410,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.364991+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:29:50.364991+0000","last_peered":"2023-09-23T15:29:50.364991+0000","last_clean":"2023-09-23T15:29:50.364991+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:29:50.364991+0000","last_undegraded":"2023-09-23T15:29:50.364991+0000","last_fullsized":"2023-09-23T15:29:50.364991+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":418,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477786+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:29:50.477786+0000","last_peered":"2023-09-23T15:29:50.477786+0000","last_clean":"2023-09-23T15:29:50.477786+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:29:50.477786+0000","last_undegraded":"2023-09-23T15:29:50.477786+0000","last_fullsized":"2023-09-23T15:29:50.477786+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":743,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505057+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:29:54.505057+0000","last_peered":"2023-09-23T15:29:54.505057+0000","last_clean":"2023-09-23T15:29:54.505057+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:29:54.505057+0000","last_undegraded":"2023-09-23T15:29:54.505057+0000","last_fullsized":"2023-09-23T15:29:54.505057+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":160,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283339+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:29:55.283339+0000","last_peered":"2023-09-23T15:29:55.283339+0000","last_clean":"2023-09-23T15:29:55.283339+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:29:55.283339+0000","last_undegraded":"2023-09-23T15:29:55.283339+0000","last_fullsized":"2023-09-23T15:29:55.283339+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":399,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.366142+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:29:50.366142+0000","last_peered":"2023-09-23T15:29:50.366142+0000","last_clean":"2023-09-23T15:29:50.366142+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:29:50.366142+0000","last_undegraded":"2023-09-23T15:29:50.366142+0000","last_fullsized":"2023-09-23T15:29:50.366142+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":732,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477087+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:29:50.477087+0000","last_peered":"2023-09-23T15:29:50.477087+0000","last_clean":"2023-09-23T15:29:50.477087+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:29:50.477087+0000","last_undegraded":"2023-09-23T15:29:50.477087+0000","last_fullsized":"2023-09-23T15:29:50.477087+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":353,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507519+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:29:54.507519+0000","last_peered":"2023-09-23T15:29:54.507519+0000","last_clean":"2023-09-23T15:29:54.507519+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:29:54.507519+0000","last_undegraded":"2023-09-23T15:29:54.507519+0000","last_fullsized":"2023-09-23T15:29:54.507519+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":416,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225524+0000","last_change":"2023-09-23T15:27:41.442107+0000","last_active":"2023-09-23T15:29:55.225524+0000","last_peered":"2023-09-23T15:29:55.225524+0000","last_clean":"2023-09-23T15:29:55.225524+0000","last_became_active":"2023-09-23T15:27:41.441806+0000","last_became_peered":"2023-09-23T15:27:41.441806+0000","last_unstale":"2023-09-23T15:29:55.225524+0000","last_undegraded":"2023-09-23T15:29:55.225524+0000","last_fullsized":"2023-09-23T15:29:55.225524+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:31.229332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"417'557","reported_seq":568,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:30:00.026755+0000","last_change":"2023-09-23T15:29:48.688598+0000","last_active":"2023-09-23T15:30:00.026755+0000","last_peered":"2023-09-23T15:30:00.026755+0000","last_clean":"2023-09-23T15:30:00.026755+0000","last_became_active":"2023-09-23T15:29:48.688227+0000","last_became_peered":"2023-09-23T15:29:48.688227+0000","last_unstale":"2023-09-23T15:30:00.026755+0000","last_undegraded":"2023-09-23T15:30:00.026755+0000","last_fullsized":"2023-09-23T15:30:00.026755+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":557,"log_dups_size":0,"ondisk_log_size":557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:04.960560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36503552,"num_objects":557,"num_object_clones":0,"num_object_copies":1671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":557,"num_write_kb":35648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":713,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283779+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:29:55.283779+0000","last_peered":"2023-09-23T15:29:55.283779+0000","last_clean":"2023-09-23T15:29:55.283779+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:29:55.283779+0000","last_undegraded":"2023-09-23T15:29:55.283779+0000","last_fullsized":"2023-09-23T15:29:55.283779+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":156,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283935+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:29:55.283935+0000","last_peered":"2023-09-23T15:29:55.283935+0000","last_clean":"2023-09-23T15:29:55.283935+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:29:55.283935+0000","last_undegraded":"2023-09-23T15:29:55.283935+0000","last_fullsized":"2023-09-23T15:29:55.283935+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":425,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.476730+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:29:50.476730+0000","last_peered":"2023-09-23T15:29:50.476730+0000","last_clean":"2023-09-23T15:29:50.476730+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:29:50.476730+0000","last_undegraded":"2023-09-23T15:29:50.476730+0000","last_fullsized":"2023-09-23T15:29:50.476730+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"417'403","reported_seq":415,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.560576+0000","last_change":"2023-09-23T15:29:48.696982+0000","last_active":"2023-09-23T15:29:59.560576+0000","last_peered":"2023-09-23T15:29:59.560576+0000","last_clean":"2023-09-23T15:29:59.560576+0000","last_became_active":"2023-09-23T15:29:48.696666+0000","last_became_peered":"2023-09-23T15:29:48.696666+0000","last_unstale":"2023-09-23T15:29:59.560576+0000","last_undegraded":"2023-09-23T15:29:59.560576+0000","last_fullsized":"2023-09-23T15:29:59.560576+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":403,"log_dups_size":0,"ondisk_log_size":403,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:34.479018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26411008,"num_objects":403,"num_object_clones":0,"num_object_copies":1209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":403,"num_write_kb":25792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":738,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225087+0000","last_change":"2023-09-23T15:28:48.882800+0000","last_active":"2023-09-23T15:29:55.225087+0000","last_peered":"2023-09-23T15:29:55.225087+0000","last_clean":"2023-09-23T15:29:55.225087+0000","last_became_active":"2023-09-23T15:27:12.393499+0000","last_became_peered":"2023-09-23T15:27:12.393499+0000","last_unstale":"2023-09-23T15:29:55.225087+0000","last_undegraded":"2023-09-23T15:29:55.225087+0000","last_fullsized":"2023-09-23T15:29:55.225087+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.882635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.882635+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:38.002743+0000","scrub_duration":0.08250528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":156,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283721+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:29:55.283721+0000","last_peered":"2023-09-23T15:29:55.283721+0000","last_clean":"2023-09-23T15:29:55.283721+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:29:55.283721+0000","last_undegraded":"2023-09-23T15:29:55.283721+0000","last_fullsized":"2023-09-23T15:29:55.283721+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506238+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:29:54.506238+0000","last_peered":"2023-09-23T15:29:54.506238+0000","last_clean":"2023-09-23T15:29:54.506238+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:29:54.506238+0000","last_undegraded":"2023-09-23T15:29:54.506238+0000","last_fullsized":"2023-09-23T15:29:54.506238+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"417'265","reported_seq":277,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:58.809864+0000","last_change":"2023-09-23T15:29:48.697177+0000","last_active":"2023-09-23T15:29:58.809864+0000","last_peered":"2023-09-23T15:29:58.809864+0000","last_clean":"2023-09-23T15:29:58.809864+0000","last_became_active":"2023-09-23T15:29:48.696842+0000","last_became_peered":"2023-09-23T15:29:48.696842+0000","last_unstale":"2023-09-23T15:29:58.809864+0000","last_undegraded":"2023-09-23T15:29:58.809864+0000","last_fullsized":"2023-09-23T15:29:58.809864+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":265,"log_dups_size":0,"ondisk_log_size":265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:18.816926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17367040,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":265,"num_write_kb":16960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":664,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.508074+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:29:54.508074+0000","last_peered":"2023-09-23T15:29:54.508074+0000","last_clean":"2023-09-23T15:29:54.508074+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:29:54.508074+0000","last_undegraded":"2023-09-23T15:29:54.508074+0000","last_fullsized":"2023-09-23T15:29:54.508074+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":144,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283436+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:29:55.283436+0000","last_peered":"2023-09-23T15:29:55.283436+0000","last_clean":"2023-09-23T15:29:55.283436+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:29:55.283436+0000","last_undegraded":"2023-09-23T15:29:55.283436+0000","last_fullsized":"2023-09-23T15:29:55.283436+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":397,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507632+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:29:54.507632+0000","last_peered":"2023-09-23T15:29:54.507632+0000","last_clean":"2023-09-23T15:29:54.507632+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:29:54.507632+0000","last_undegraded":"2023-09-23T15:29:54.507632+0000","last_fullsized":"2023-09-23T15:29:54.507632+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"417'440","reported_seq":452,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.377668+0000","last_change":"2023-09-23T15:29:48.698992+0000","last_active":"2023-09-23T15:29:59.377668+0000","last_peered":"2023-09-23T15:29:59.377668+0000","last_clean":"2023-09-23T15:29:59.377668+0000","last_became_active":"2023-09-23T15:29:48.698673+0000","last_became_peered":"2023-09-23T15:29:48.698673+0000","last_unstale":"2023-09-23T15:29:59.377668+0000","last_undegraded":"2023-09-23T15:29:59.377668+0000","last_fullsized":"2023-09-23T15:29:59.377668+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":440,"log_dups_size":0,"ondisk_log_size":440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:12.184760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28835840,"num_objects":440,"num_object_clones":0,"num_object_copies":1320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":440,"num_write_kb":28160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":634,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507008+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:29:54.507008+0000","last_peered":"2023-09-23T15:29:54.507008+0000","last_clean":"2023-09-23T15:29:54.507008+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:29:54.507008+0000","last_undegraded":"2023-09-23T15:29:54.507008+0000","last_fullsized":"2023-09-23T15:29:54.507008+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":392,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477010+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:29:50.477010+0000","last_peered":"2023-09-23T15:29:50.477010+0000","last_clean":"2023-09-23T15:29:50.477010+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:29:50.477010+0000","last_undegraded":"2023-09-23T15:29:50.477010+0000","last_fullsized":"2023-09-23T15:29:50.477010+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":418,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365211+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:29:50.365211+0000","last_peered":"2023-09-23T15:29:50.365211+0000","last_clean":"2023-09-23T15:29:50.365211+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:29:50.365211+0000","last_undegraded":"2023-09-23T15:29:50.365211+0000","last_fullsized":"2023-09-23T15:29:50.365211+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"417'662","reported_seq":675,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:30:00.690286+0000","last_change":"2023-09-23T15:29:48.679906+0000","last_active":"2023-09-23T15:30:00.690286+0000","last_peered":"2023-09-23T15:30:00.690286+0000","last_clean":"2023-09-23T15:30:00.690286+0000","last_became_active":"2023-09-23T15:29:48.679584+0000","last_became_peered":"2023-09-23T15:29:48.679584+0000","last_unstale":"2023-09-23T15:30:00.690286+0000","last_undegraded":"2023-09-23T15:30:00.690286+0000","last_fullsized":"2023-09-23T15:30:00.690286+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":662,"log_dups_size":0,"ondisk_log_size":662,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:53.805715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43384832,"num_objects":662,"num_object_clones":0,"num_object_copies":1986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":662,"num_write_kb":42368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":669,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283227+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:29:55.283227+0000","last_peered":"2023-09-23T15:29:55.283227+0000","last_clean":"2023-09-23T15:29:55.283227+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:29:55.283227+0000","last_undegraded":"2023-09-23T15:29:55.283227+0000","last_fullsized":"2023-09-23T15:29:55.283227+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":413,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224873+0000","last_change":"2023-09-23T15:27:23.323912+0000","last_active":"2023-09-23T15:29:55.224873+0000","last_peered":"2023-09-23T15:29:55.224873+0000","last_clean":"2023-09-23T15:29:55.224873+0000","last_became_active":"2023-09-23T15:27:23.316529+0000","last_became_peered":"2023-09-23T15:27:23.316529+0000","last_unstale":"2023-09-23T15:29:55.224873+0000","last_undegraded":"2023-09-23T15:29:55.224873+0000","last_fullsized":"2023-09-23T15:29:55.224873+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:55.689850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506706+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:29:54.506706+0000","last_peered":"2023-09-23T15:29:54.506706+0000","last_clean":"2023-09-23T15:29:54.506706+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:29:54.506706+0000","last_undegraded":"2023-09-23T15:29:54.506706+0000","last_fullsized":"2023-09-23T15:29:54.506706+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"417'410","reported_seq":422,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.381951+0000","last_change":"2023-09-23T15:29:48.698981+0000","last_active":"2023-09-23T15:29:59.381951+0000","last_peered":"2023-09-23T15:29:59.381951+0000","last_clean":"2023-09-23T15:29:59.381951+0000","last_became_active":"2023-09-23T15:29:48.698707+0000","last_became_peered":"2023-09-23T15:29:48.698707+0000","last_unstale":"2023-09-23T15:29:59.381951+0000","last_undegraded":"2023-09-23T15:29:59.381951+0000","last_fullsized":"2023-09-23T15:29:59.381951+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":410,"log_dups_size":0,"ondisk_log_size":410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:28.109594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26869760,"num_objects":410,"num_object_clones":0,"num_object_copies":1230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":410,"num_write_kb":26240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":740,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477028+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:29:50.477028+0000","last_peered":"2023-09-23T15:29:50.477028+0000","last_clean":"2023-09-23T15:29:50.477028+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:29:50.477028+0000","last_undegraded":"2023-09-23T15:29:50.477028+0000","last_fullsized":"2023-09-23T15:29:50.477028+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":384,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507792+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:29:54.507792+0000","last_peered":"2023-09-23T15:29:54.507792+0000","last_clean":"2023-09-23T15:29:54.507792+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:29:54.507792+0000","last_undegraded":"2023-09-23T15:29:54.507792+0000","last_fullsized":"2023-09-23T15:29:54.507792+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506959+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:29:54.506959+0000","last_peered":"2023-09-23T15:29:54.506959+0000","last_clean":"2023-09-23T15:29:54.506959+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:29:54.506959+0000","last_undegraded":"2023-09-23T15:29:54.506959+0000","last_fullsized":"2023-09-23T15:29:54.506959+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"417'426","reported_seq":438,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.566620+0000","last_change":"2023-09-23T15:29:48.683338+0000","last_active":"2023-09-23T15:29:59.566620+0000","last_peered":"2023-09-23T15:29:59.566620+0000","last_clean":"2023-09-23T15:29:59.566620+0000","last_became_active":"2023-09-23T15:29:48.682970+0000","last_became_peered":"2023-09-23T15:29:48.682970+0000","last_unstale":"2023-09-23T15:29:59.566620+0000","last_undegraded":"2023-09-23T15:29:59.566620+0000","last_fullsized":"2023-09-23T15:29:59.566620+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":426,"log_dups_size":0,"ondisk_log_size":426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:20.273166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":27918336,"num_objects":426,"num_object_clones":0,"num_object_copies":1278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":426,"num_write_kb":27264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":696,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.504795+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:29:54.504795+0000","last_peered":"2023-09-23T15:29:54.504795+0000","last_clean":"2023-09-23T15:29:54.504795+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:29:54.504795+0000","last_undegraded":"2023-09-23T15:29:54.504795+0000","last_fullsized":"2023-09-23T15:29:54.504795+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506263+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:29:54.506263+0000","last_peered":"2023-09-23T15:29:54.506263+0000","last_clean":"2023-09-23T15:29:54.506263+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:29:54.506263+0000","last_undegraded":"2023-09-23T15:29:54.506263+0000","last_fullsized":"2023-09-23T15:29:54.506263+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507063+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:29:54.507063+0000","last_peered":"2023-09-23T15:29:54.507063+0000","last_clean":"2023-09-23T15:29:54.507063+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:29:54.507063+0000","last_undegraded":"2023-09-23T15:29:54.507063+0000","last_fullsized":"2023-09-23T15:29:54.507063+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"417'502","reported_seq":514,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.847392+0000","last_change":"2023-09-23T15:29:48.680338+0000","last_active":"2023-09-23T15:29:59.847392+0000","last_peered":"2023-09-23T15:29:59.847392+0000","last_clean":"2023-09-23T15:29:59.847392+0000","last_became_active":"2023-09-23T15:29:48.679979+0000","last_became_peered":"2023-09-23T15:29:48.679979+0000","last_unstale":"2023-09-23T15:29:59.847392+0000","last_undegraded":"2023-09-23T15:29:59.847392+0000","last_fullsized":"2023-09-23T15:29:59.847392+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":502,"log_dups_size":0,"ondisk_log_size":502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:54.650904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":502,"num_write_kb":32128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":752,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.476967+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:29:50.476967+0000","last_peered":"2023-09-23T15:29:50.476967+0000","last_clean":"2023-09-23T15:29:50.476967+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:29:50.476967+0000","last_undegraded":"2023-09-23T15:29:50.476967+0000","last_fullsized":"2023-09-23T15:29:50.476967+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":427,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224648+0000","last_change":"2023-09-23T15:27:41.418214+0000","last_active":"2023-09-23T15:29:55.224648+0000","last_peered":"2023-09-23T15:29:55.224648+0000","last_clean":"2023-09-23T15:29:55.224648+0000","last_became_active":"2023-09-23T15:27:41.417924+0000","last_became_peered":"2023-09-23T15:27:41.417924+0000","last_unstale":"2023-09-23T15:29:55.224648+0000","last_undegraded":"2023-09-23T15:29:55.224648+0000","last_fullsized":"2023-09-23T15:29:55.224648+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:45.122620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":351,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506045+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:29:54.506045+0000","last_peered":"2023-09-23T15:29:54.506045+0000","last_clean":"2023-09-23T15:29:54.506045+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:29:54.506045+0000","last_undegraded":"2023-09-23T15:29:54.506045+0000","last_fullsized":"2023-09-23T15:29:54.506045+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"417'434","reported_seq":446,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.445388+0000","last_change":"2023-09-23T15:29:48.682296+0000","last_active":"2023-09-23T15:29:59.445388+0000","last_peered":"2023-09-23T15:29:59.445388+0000","last_clean":"2023-09-23T15:29:59.445388+0000","last_became_active":"2023-09-23T15:29:48.681890+0000","last_became_peered":"2023-09-23T15:29:48.681890+0000","last_unstale":"2023-09-23T15:29:59.445388+0000","last_undegraded":"2023-09-23T15:29:59.445388+0000","last_fullsized":"2023-09-23T15:29:59.445388+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":434,"log_dups_size":0,"ondisk_log_size":434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:34.635613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28442624,"num_objects":434,"num_object_clones":0,"num_object_copies":1302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":434,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":434,"num_write_kb":27776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":657,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.508111+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:29:54.508111+0000","last_peered":"2023-09-23T15:29:54.508111+0000","last_clean":"2023-09-23T15:29:54.508111+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:29:54.508111+0000","last_undegraded":"2023-09-23T15:29:54.508111+0000","last_fullsized":"2023-09-23T15:29:54.508111+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":634,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505420+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:29:54.505420+0000","last_peered":"2023-09-23T15:29:54.505420+0000","last_clean":"2023-09-23T15:29:54.505420+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:29:54.505420+0000","last_undegraded":"2023-09-23T15:29:54.505420+0000","last_fullsized":"2023-09-23T15:29:54.505420+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":392,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477772+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:29:50.477772+0000","last_peered":"2023-09-23T15:29:50.477772+0000","last_clean":"2023-09-23T15:29:50.477772+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:29:50.477772+0000","last_undegraded":"2023-09-23T15:29:50.477772+0000","last_fullsized":"2023-09-23T15:29:50.477772+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":380,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.395074+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:29:50.395074+0000","last_peered":"2023-09-23T15:29:50.395074+0000","last_clean":"2023-09-23T15:29:50.395074+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:29:50.395074+0000","last_undegraded":"2023-09-23T15:29:50.395074+0000","last_fullsized":"2023-09-23T15:29:50.395074+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"417'447","reported_seq":458,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.557520+0000","last_change":"2023-09-23T15:29:48.686875+0000","last_active":"2023-09-23T15:29:59.557520+0000","last_peered":"2023-09-23T15:29:59.557520+0000","last_clean":"2023-09-23T15:29:59.557520+0000","last_became_active":"2023-09-23T15:29:48.686520+0000","last_became_peered":"2023-09-23T15:29:48.686520+0000","last_unstale":"2023-09-23T15:29:59.557520+0000","last_undegraded":"2023-09-23T15:29:59.557520+0000","last_fullsized":"2023-09-23T15:29:59.557520+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":447,"log_dups_size":0,"ondisk_log_size":447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:52.851173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29294592,"num_objects":447,"num_object_clones":0,"num_object_copies":1341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":447,"num_write_kb":28608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":713,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.476379+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:29:50.476379+0000","last_peered":"2023-09-23T15:29:50.476379+0000","last_clean":"2023-09-23T15:29:50.476379+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:29:50.476379+0000","last_undegraded":"2023-09-23T15:29:50.476379+0000","last_fullsized":"2023-09-23T15:29:50.476379+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":951,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477389+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:29:50.477389+0000","last_peered":"2023-09-23T15:29:50.477389+0000","last_clean":"2023-09-23T15:29:50.477389+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:29:50.477389+0000","last_undegraded":"2023-09-23T15:29:50.477389+0000","last_fullsized":"2023-09-23T15:29:50.477389+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506694+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:29:54.506694+0000","last_peered":"2023-09-23T15:29:54.506694+0000","last_clean":"2023-09-23T15:29:54.506694+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:29:54.506694+0000","last_undegraded":"2023-09-23T15:29:54.506694+0000","last_fullsized":"2023-09-23T15:29:54.506694+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":316,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505215+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:29:54.505215+0000","last_peered":"2023-09-23T15:29:54.505215+0000","last_clean":"2023-09-23T15:29:54.505215+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:29:54.505215+0000","last_undegraded":"2023-09-23T15:29:54.505215+0000","last_fullsized":"2023-09-23T15:29:54.505215+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"417'576","reported_seq":586,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:30:00.043218+0000","last_change":"2023-09-23T15:29:48.686854+0000","last_active":"2023-09-23T15:30:00.043218+0000","last_peered":"2023-09-23T15:30:00.043218+0000","last_clean":"2023-09-23T15:30:00.043218+0000","last_became_active":"2023-09-23T15:29:48.686539+0000","last_became_peered":"2023-09-23T15:29:48.686539+0000","last_unstale":"2023-09-23T15:30:00.043218+0000","last_undegraded":"2023-09-23T15:30:00.043218+0000","last_fullsized":"2023-09-23T15:30:00.043218+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":576,"log_dups_size":0,"ondisk_log_size":576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:33.671531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37748736,"num_objects":576,"num_object_clones":0,"num_object_copies":1728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":576,"num_write_kb":36864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":780,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283467+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:29:55.283467+0000","last_peered":"2023-09-23T15:29:55.283467+0000","last_clean":"2023-09-23T15:29:55.283467+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:29:55.283467+0000","last_undegraded":"2023-09-23T15:29:55.283467+0000","last_fullsized":"2023-09-23T15:29:55.283467+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1035,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.395085+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:29:50.395085+0000","last_peered":"2023-09-23T15:29:50.395085+0000","last_clean":"2023-09-23T15:29:50.395085+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:29:50.395085+0000","last_undegraded":"2023-09-23T15:29:50.395085+0000","last_fullsized":"2023-09-23T15:29:50.395085+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":311,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.504180+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:29:54.504180+0000","last_peered":"2023-09-23T15:29:54.504180+0000","last_clean":"2023-09-23T15:29:54.504180+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:29:54.504180+0000","last_undegraded":"2023-09-23T15:29:54.504180+0000","last_fullsized":"2023-09-23T15:29:54.504180+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":319,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505963+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:29:54.505963+0000","last_peered":"2023-09-23T15:29:54.505963+0000","last_clean":"2023-09-23T15:29:54.505963+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:29:54.505963+0000","last_undegraded":"2023-09-23T15:29:54.505963+0000","last_fullsized":"2023-09-23T15:29:54.505963+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"417'450","reported_seq":462,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.567405+0000","last_change":"2023-09-23T15:29:48.684663+0000","last_active":"2023-09-23T15:29:59.567405+0000","last_peered":"2023-09-23T15:29:59.567405+0000","last_clean":"2023-09-23T15:29:59.567405+0000","last_became_active":"2023-09-23T15:29:48.684293+0000","last_became_peered":"2023-09-23T15:29:48.684293+0000","last_unstale":"2023-09-23T15:29:59.567405+0000","last_undegraded":"2023-09-23T15:29:59.567405+0000","last_fullsized":"2023-09-23T15:29:59.567405+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":450,"log_dups_size":0,"ondisk_log_size":450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:18.182468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29491200,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":450,"num_write_kb":28800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":762,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477592+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:29:50.477592+0000","last_peered":"2023-09-23T15:29:50.477592+0000","last_clean":"2023-09-23T15:29:50.477592+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:29:50.477592+0000","last_undegraded":"2023-09-23T15:29:50.477592+0000","last_fullsized":"2023-09-23T15:29:50.477592+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":723,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283277+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:29:55.283277+0000","last_peered":"2023-09-23T15:29:55.283277+0000","last_clean":"2023-09-23T15:29:55.283277+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:29:55.283277+0000","last_undegraded":"2023-09-23T15:29:55.283277+0000","last_fullsized":"2023-09-23T15:29:55.283277+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":341,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224788+0000","last_change":"2023-09-23T15:27:12.392661+0000","last_active":"2023-09-23T15:29:55.224788+0000","last_peered":"2023-09-23T15:29:55.224788+0000","last_clean":"2023-09-23T15:29:55.224788+0000","last_became_active":"2023-09-23T15:27:12.389493+0000","last_became_peered":"2023-09-23T15:27:12.389493+0000","last_unstale":"2023-09-23T15:29:55.224788+0000","last_undegraded":"2023-09-23T15:29:55.224788+0000","last_fullsized":"2023-09-23T15:29:55.224788+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:49.162226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":409,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507879+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:29:54.507879+0000","last_peered":"2023-09-23T15:29:54.507879+0000","last_clean":"2023-09-23T15:29:54.507879+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:29:54.507879+0000","last_undegraded":"2023-09-23T15:29:54.507879+0000","last_fullsized":"2023-09-23T15:29:54.507879+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"417'289","reported_seq":301,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:58.778709+0000","last_change":"2023-09-23T15:29:48.679894+0000","last_active":"2023-09-23T15:29:58.778709+0000","last_peered":"2023-09-23T15:29:58.778709+0000","last_clean":"2023-09-23T15:29:58.778709+0000","last_became_active":"2023-09-23T15:29:48.679648+0000","last_became_peered":"2023-09-23T15:29:48.679648+0000","last_unstale":"2023-09-23T15:29:58.778709+0000","last_undegraded":"2023-09-23T15:29:58.778709+0000","last_fullsized":"2023-09-23T15:29:58.778709+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":289,"log_dups_size":0,"ondisk_log_size":289,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:23.079554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18939904,"num_objects":289,"num_object_clones":0,"num_object_copies":867,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":289,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":289,"num_write_kb":18496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":760,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.504139+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:29:54.504139+0000","last_peered":"2023-09-23T15:29:54.504139+0000","last_clean":"2023-09-23T15:29:54.504139+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:29:54.504139+0000","last_undegraded":"2023-09-23T15:29:54.504139+0000","last_fullsized":"2023-09-23T15:29:54.504139+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":130,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283150+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:29:55.283150+0000","last_peered":"2023-09-23T15:29:55.283150+0000","last_clean":"2023-09-23T15:29:55.283150+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:29:55.283150+0000","last_undegraded":"2023-09-23T15:29:55.283150+0000","last_fullsized":"2023-09-23T15:29:55.283150+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":404,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365002+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:29:50.365002+0000","last_peered":"2023-09-23T15:29:50.365002+0000","last_clean":"2023-09-23T15:29:50.365002+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:29:50.365002+0000","last_undegraded":"2023-09-23T15:29:50.365002+0000","last_fullsized":"2023-09-23T15:29:50.365002+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":397,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365907+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:29:50.365907+0000","last_peered":"2023-09-23T15:29:50.365907+0000","last_clean":"2023-09-23T15:29:50.365907+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:29:50.365907+0000","last_undegraded":"2023-09-23T15:29:50.365907+0000","last_fullsized":"2023-09-23T15:29:50.365907+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"417'448","reported_seq":460,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.565775+0000","last_change":"2023-09-23T15:29:48.683852+0000","last_active":"2023-09-23T15:29:59.565775+0000","last_peered":"2023-09-23T15:29:59.565775+0000","last_clean":"2023-09-23T15:29:59.565775+0000","last_became_active":"2023-09-23T15:29:48.683560+0000","last_became_peered":"2023-09-23T15:29:48.683560+0000","last_unstale":"2023-09-23T15:29:59.565775+0000","last_undegraded":"2023-09-23T15:29:59.565775+0000","last_fullsized":"2023-09-23T15:29:59.565775+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":448,"log_dups_size":0,"ondisk_log_size":448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:38.866091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29360128,"num_objects":448,"num_object_clones":0,"num_object_copies":1344,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":448,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":448,"num_write_kb":28672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":743,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507992+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:29:54.507992+0000","last_peered":"2023-09-23T15:29:54.507992+0000","last_clean":"2023-09-23T15:29:54.507992+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:29:54.507992+0000","last_undegraded":"2023-09-23T15:29:54.507992+0000","last_fullsized":"2023-09-23T15:29:54.507992+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1156,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283682+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:29:55.283682+0000","last_peered":"2023-09-23T15:29:55.283682+0000","last_clean":"2023-09-23T15:29:55.283682+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:29:55.283682+0000","last_undegraded":"2023-09-23T15:29:55.283682+0000","last_fullsized":"2023-09-23T15:29:55.283682+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":296,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505325+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:29:54.505325+0000","last_peered":"2023-09-23T15:29:54.505325+0000","last_clean":"2023-09-23T15:29:54.505325+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:29:54.505325+0000","last_undegraded":"2023-09-23T15:29:54.505325+0000","last_fullsized":"2023-09-23T15:29:54.505325+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":397,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365512+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:29:50.365512+0000","last_peered":"2023-09-23T15:29:50.365512+0000","last_clean":"2023-09-23T15:29:50.365512+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:29:50.365512+0000","last_undegraded":"2023-09-23T15:29:50.365512+0000","last_fullsized":"2023-09-23T15:29:50.365512+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"417'511","reported_seq":523,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.848086+0000","last_change":"2023-09-23T15:29:48.685734+0000","last_active":"2023-09-23T15:29:59.848086+0000","last_peered":"2023-09-23T15:29:59.848086+0000","last_clean":"2023-09-23T15:29:59.848086+0000","last_became_active":"2023-09-23T15:29:48.684906+0000","last_became_peered":"2023-09-23T15:29:48.684906+0000","last_unstale":"2023-09-23T15:29:59.848086+0000","last_undegraded":"2023-09-23T15:29:59.848086+0000","last_fullsized":"2023-09-23T15:29:59.848086+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":511,"log_dups_size":0,"ondisk_log_size":511,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:06.890491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33488896,"num_objects":511,"num_object_clones":0,"num_object_copies":1533,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":511,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":511,"num_write_kb":32704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":708,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.507130+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:29:54.507130+0000","last_peered":"2023-09-23T15:29:54.507130+0000","last_clean":"2023-09-23T15:29:54.507130+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:29:54.507130+0000","last_undegraded":"2023-09-23T15:29:54.507130+0000","last_fullsized":"2023-09-23T15:29:54.507130+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":189,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.395491+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:29:50.395491+0000","last_peered":"2023-09-23T15:29:50.395491+0000","last_clean":"2023-09-23T15:29:50.395491+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:29:50.395491+0000","last_undegraded":"2023-09-23T15:29:50.395491+0000","last_fullsized":"2023-09-23T15:29:50.395491+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1050,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505799+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:29:54.505799+0000","last_peered":"2023-09-23T15:29:54.505799+0000","last_clean":"2023-09-23T15:29:54.505799+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:29:54.505799+0000","last_undegraded":"2023-09-23T15:29:54.505799+0000","last_fullsized":"2023-09-23T15:29:54.505799+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":407,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224800+0000","last_change":"2023-09-23T15:27:41.419984+0000","last_active":"2023-09-23T15:29:55.224800+0000","last_peered":"2023-09-23T15:29:55.224800+0000","last_clean":"2023-09-23T15:29:55.224800+0000","last_became_active":"2023-09-23T15:27:41.419701+0000","last_became_peered":"2023-09-23T15:27:41.419701+0000","last_unstale":"2023-09-23T15:29:55.224800+0000","last_undegraded":"2023-09-23T15:29:55.224800+0000","last_fullsized":"2023-09-23T15:29:55.224800+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:55.086471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"417'482","reported_seq":494,"reported_epoch":417,"state":"active+clean","last_fresh":"2023-09-23T15:29:59.568869+0000","last_change":"2023-09-23T15:29:48.697095+0000","last_active":"2023-09-23T15:29:59.568869+0000","last_peered":"2023-09-23T15:29:59.568869+0000","last_clean":"2023-09-23T15:29:59.568869+0000","last_became_active":"2023-09-23T15:29:48.696636+0000","last_became_peered":"2023-09-23T15:29:48.696636+0000","last_unstale":"2023-09-23T15:29:59.568869+0000","last_undegraded":"2023-09-23T15:29:59.568869+0000","last_fullsized":"2023-09-23T15:29:59.568869+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":482,"log_dups_size":0,"ondisk_log_size":482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:52.284501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31588352,"num_objects":482,"num_object_clones":0,"num_object_copies":1446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":482,"num_write_kb":30848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":737,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225055+0000","last_change":"2023-09-23T15:27:41.438036+0000","last_active":"2023-09-23T15:29:55.225055+0000","last_peered":"2023-09-23T15:29:55.225055+0000","last_clean":"2023-09-23T15:29:55.225055+0000","last_became_active":"2023-09-23T15:27:41.437722+0000","last_became_peered":"2023-09-23T15:27:41.437722+0000","last_unstale":"2023-09-23T15:29:55.225055+0000","last_undegraded":"2023-09-23T15:29:55.225055+0000","last_fullsized":"2023-09-23T15:29:55.225055+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:37.744292+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":870,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.508349+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:29:54.508349+0000","last_peered":"2023-09-23T15:29:54.508349+0000","last_clean":"2023-09-23T15:29:54.508349+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:29:54.508349+0000","last_undegraded":"2023-09-23T15:29:54.508349+0000","last_fullsized":"2023-09-23T15:29:54.508349+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":456,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365995+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:29:50.365995+0000","last_peered":"2023-09-23T15:29:50.365995+0000","last_clean":"2023-09-23T15:29:50.365995+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:29:50.365995+0000","last_undegraded":"2023-09-23T15:29:50.365995+0000","last_fullsized":"2023-09-23T15:29:50.365995+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506391+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:29:54.506391+0000","last_peered":"2023-09-23T15:29:54.506391+0000","last_clean":"2023-09-23T15:29:54.506391+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:29:54.506391+0000","last_undegraded":"2023-09-23T15:29:54.506391+0000","last_fullsized":"2023-09-23T15:29:54.506391+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":652,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224708+0000","last_change":"2023-09-23T15:28:50.816286+0000","last_active":"2023-09-23T15:29:55.224708+0000","last_peered":"2023-09-23T15:29:55.224708+0000","last_clean":"2023-09-23T15:29:55.224708+0000","last_became_active":"2023-09-23T15:28:39.666756+0000","last_became_peered":"2023-09-23T15:28:39.666756+0000","last_unstale":"2023-09-23T15:29:55.224708+0000","last_undegraded":"2023-09-23T15:29:55.224708+0000","last_fullsized":"2023-09-23T15:29:55.224708+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.816141+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.816141+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:36.466168+0000","scrub_duration":0.017140743999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":436,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365377+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:29:50.365377+0000","last_peered":"2023-09-23T15:29:50.365377+0000","last_clean":"2023-09-23T15:29:50.365377+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:29:50.365377+0000","last_undegraded":"2023-09-23T15:29:50.365377+0000","last_fullsized":"2023-09-23T15:29:50.365377+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":421,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.395200+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:29:50.395200+0000","last_peered":"2023-09-23T15:29:50.395200+0000","last_clean":"2023-09-23T15:29:50.395200+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:29:50.395200+0000","last_undegraded":"2023-09-23T15:29:50.395200+0000","last_fullsized":"2023-09-23T15:29:50.395200+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":621,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225215+0000","last_change":"2023-09-23T15:28:51.796932+0000","last_active":"2023-09-23T15:29:55.225215+0000","last_peered":"2023-09-23T15:29:55.225215+0000","last_clean":"2023-09-23T15:29:55.225215+0000","last_became_active":"2023-09-23T15:28:39.656548+0000","last_became_peered":"2023-09-23T15:28:39.656548+0000","last_unstale":"2023-09-23T15:29:55.225215+0000","last_undegraded":"2023-09-23T15:29:55.225215+0000","last_fullsized":"2023-09-23T15:29:55.225215+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.796761+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.796761+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:04:26.204960+0000","scrub_duration":0.011035615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":457,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477123+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:29:50.477123+0000","last_peered":"2023-09-23T15:29:50.477123+0000","last_clean":"2023-09-23T15:29:50.477123+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:29:50.477123+0000","last_undegraded":"2023-09-23T15:29:50.477123+0000","last_fullsized":"2023-09-23T15:29:50.477123+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506820+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:29:54.506820+0000","last_peered":"2023-09-23T15:29:54.506820+0000","last_clean":"2023-09-23T15:29:54.506820+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:29:54.506820+0000","last_undegraded":"2023-09-23T15:29:54.506820+0000","last_fullsized":"2023-09-23T15:29:54.506820+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":437,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.477682+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:29:50.477682+0000","last_peered":"2023-09-23T15:29:50.477682+0000","last_clean":"2023-09-23T15:29:50.477682+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:29:50.477682+0000","last_undegraded":"2023-09-23T15:29:50.477682+0000","last_fullsized":"2023-09-23T15:29:50.477682+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506363+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:29:54.506363+0000","last_peered":"2023-09-23T15:29:54.506363+0000","last_clean":"2023-09-23T15:29:54.506363+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:29:54.506363+0000","last_undegraded":"2023-09-23T15:29:54.506363+0000","last_fullsized":"2023-09-23T15:29:54.506363+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":604,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.283531+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:29:55.283531+0000","last_peered":"2023-09-23T15:29:55.283531+0000","last_clean":"2023-09-23T15:29:55.283531+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:29:55.283531+0000","last_undegraded":"2023-09-23T15:29:55.283531+0000","last_fullsized":"2023-09-23T15:29:55.283531+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":398,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.225512+0000","last_change":"2023-09-23T15:27:12.384840+0000","last_active":"2023-09-23T15:29:55.225512+0000","last_peered":"2023-09-23T15:29:55.225512+0000","last_clean":"2023-09-23T15:29:55.225512+0000","last_became_active":"2023-09-23T15:27:12.384518+0000","last_became_peered":"2023-09-23T15:27:12.384518+0000","last_unstale":"2023-09-23T15:29:55.225512+0000","last_undegraded":"2023-09-23T15:29:55.225512+0000","last_fullsized":"2023-09-23T15:29:55.225512+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:18.377550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":396,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:55.224868+0000","last_change":"2023-09-23T15:27:12.385884+0000","last_active":"2023-09-23T15:29:55.224868+0000","last_peered":"2023-09-23T15:29:55.224868+0000","last_clean":"2023-09-23T15:29:55.224868+0000","last_became_active":"2023-09-23T15:27:12.385612+0000","last_became_peered":"2023-09-23T15:27:12.385612+0000","last_unstale":"2023-09-23T15:29:55.224868+0000","last_undegraded":"2023-09-23T15:29:55.224868+0000","last_fullsized":"2023-09-23T15:29:55.224868+0000","mapping_epoch":363,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":364,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:50.641495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":597,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.505435+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:29:54.505435+0000","last_peered":"2023-09-23T15:29:54.505435+0000","last_clean":"2023-09-23T15:29:54.505435+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:29:54.505435+0000","last_undegraded":"2023-09-23T15:29:54.505435+0000","last_fullsized":"2023-09-23T15:29:54.505435+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":371,"reported_epoch":415,"state":"active+clean","last_fresh":"2023-09-23T15:29:50.365881+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:29:50.365881+0000","last_peered":"2023-09-23T15:29:50.365881+0000","last_clean":"2023-09-23T15:29:50.365881+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:29:50.365881+0000","last_undegraded":"2023-09-23T15:29:50.365881+0000","last_fullsized":"2023-09-23T15:29:50.365881+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":59,"reported_epoch":416,"state":"active+clean","last_fresh":"2023-09-23T15:29:54.506001+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:29:54.506001+0000","last_peered":"2023-09-23T15:29:54.506001+0000","last_clean":"2023-09-23T15:29:54.506001+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:29:54.506001+0000","last_undegraded":"2023-09-23T15:29:54.506001+0000","last_fullsized":"2023-09-23T15:29:54.506001+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":478543872,"num_objects":7302,"num_object_clones":0,"num_object_copies":21906,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7302,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7302,"num_write_kb":467328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1419247616,"data_stored":1419247616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7302,"ondisk_log_size":7302,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768733,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":165068,"kb_used_data":70980,"kb_used_omap":13,"kb_used_meta":94066,"kb_avail":93580084,"statfs":{"total":95995035648,"available":95826006016,"internally_reserved":0,"allocated":72683520,"data_stored":71336227,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":96323678},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5919999999999996,"5min":5.7519999999999998,"15min":5.7779999999999996},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":80,"5min":80,"15min":80},"last":0.83699999999999997},{"interface":"front","average":{"1min":5.4880000000000004,"5min":5.8239999999999998,"15min":5.8799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":80.117000000000004,"5min":80.117000000000004,"15min":80.117000000000004},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":6.2830000000000004,"5min":18.283000000000001,"15min":20.283000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.632000000000005,"5min":157.096,"15min":157.096},"last":0.41799999999999998},{"interface":"front","average":{"1min":5.3289999999999997,"5min":18.210999999999999,"15min":20.358000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":76.489000000000004,"5min":157.011,"15min":157.011},"last":0.52400000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":1.0649999999999999,"15min":0.86299999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.1439999999999999,"5min":0.996,"15min":0.97099999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.6930000000000001,"5min":4.109,"15min":4.109},"last":0.46999999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":5.6980000000000004,"5min":4.3959999999999999,"15min":4.1790000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":76.019000000000005,"5min":76.019000000000005,"15min":76.019000000000005},"last":0.72899999999999998},{"interface":"front","average":{"1min":6.7709999999999999,"5min":4.9390000000000001,"15min":4.633},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":76.317999999999998,"5min":76.317999999999998,"15min":76.317999999999998},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.65800000000000003,"15min":0.59799999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3710000000000004,"5min":4.3710000000000004,"15min":4.3710000000000004},"last":0.79400000000000004},{"interface":"front","average":{"1min":1.3740000000000001,"5min":0.67000000000000004,"15min":0.55300000000000005},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.8089999999999993,"5min":8.8089999999999993,"15min":8.8089999999999993},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.79100000000000004,"15min":0.80200000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.1840000000000002,"5min":3.4279999999999999,"15min":3.4279999999999999},"last":0.622},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.80800000000000005,"15min":0.754},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.3369999999999997,"5min":4.3369999999999997,"15min":4.3369999999999997},"last":0.878}]},{"osd":6,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.75600000000000001,"15min":0.73699999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.85499999999999998,"15min":0.71699999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.2810000000000006,"5min":9.2810000000000006,"15min":9.2810000000000006},"last":1.0589999999999999}]}]},{"osd":6,"up_from":304,"seq":1305670058076,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":300544,"kb_used_data":175928,"kb_used_omap":13,"kb_used_meta":124594,"kb_avail":93444608,"statfs":{"total":95995035648,"available":95687278592,"internally_reserved":0,"allocated":180150272,"data_stored":178814596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":127584611},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.879,"15min":2.593},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":10.827,"5min":46.740000000000002,"15min":184.53200000000001},"last":0.999},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.865,"15min":2.9169999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.2460000000000004,"5min":46.549999999999997,"15min":86.227999999999994},"last":0.91300000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.6459999999999999,"15min":2.8090000000000002},"min":{"1min":0.28299999999999997,"5min":0.221,"15min":0.221},"max":{"1min":9.9710000000000001,"5min":9.9710000000000001,"15min":86.534000000000006},"last":0.55900000000000005},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.351,"15min":2.7349999999999999},"min":{"1min":0.34200000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.8100000000000001,"5min":6.8710000000000004,"15min":86.144000000000005},"last":1.4570000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.7949999999999999,"15min":1.694},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.1980000000000004,"5min":9.2379999999999995,"15min":9.8469999999999995},"last":0.873},{"interface":"front","average":{"1min":1.224,"5min":1.603,"15min":1.7729999999999999},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.307},"max":{"1min":4.5709999999999997,"5min":9.3650000000000002,"15min":9.3650000000000002},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":3.1800000000000002,"15min":2.7229999999999999},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9569999999999999,"5min":142.67599999999999,"15min":142.67599999999999},"last":1.1200000000000001},{"interface":"front","average":{"1min":2.7410000000000001,"5min":3.6509999999999998,"15min":2.923},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":18.149000000000001,"5min":142.541,"15min":142.541},"last":4.4619999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.78000000000000003,"15min":0.96699999999999997},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.8460000000000001,"5min":6.6479999999999997,"15min":8.1219999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.272,"5min":1.095,"15min":1.641},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.569,"5min":5.569,"15min":13.627000000000001},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.7839999999999998,"15min":2.258},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":4.2030000000000003,"5min":130.91300000000001,"15min":130.91300000000001},"last":0.433},{"interface":"front","average":{"1min":1.26,"5min":2.6160000000000001,"15min":1.9750000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.6470000000000002,"5min":130.92599999999999,"15min":130.92599999999999},"last":0.50800000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.196,"15min":1.222},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7009999999999996,"5min":9.3209999999999997,"15min":9.3209999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.4430000000000001,"15min":1.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.492,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.54700000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015377,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":690448,"kb_used_data":121544,"kb_used_omap":6,"kb_used_meta":568889,"kb_avail":93054704,"statfs":{"total":95995035648,"available":95288016896,"internally_reserved":0,"allocated":124461056,"data_stored":122374432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":582542411},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0830000000000002,"5min":2.6299999999999999,"15min":2.9910000000000001},"min":{"1min":0.20499999999999999,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":12.093999999999999,"5min":57.329000000000001,"15min":60.817999999999998},"last":1.173},{"interface":"front","average":{"1min":2.6230000000000002,"5min":2.5659999999999998,"15min":3.484},"min":{"1min":0.26600000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":11.613,"5min":17.128,"15min":63.594999999999999},"last":1.7569999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.903,"5min":3.016,"15min":3.714},"min":{"1min":0.252,"5min":0.221,"15min":0.221},"max":{"1min":8.7289999999999992,"5min":54.091999999999999,"15min":109.375},"last":0.75900000000000001},{"interface":"front","average":{"1min":2.407,"5min":2.6379999999999999,"15min":3.036},"min":{"1min":0.28000000000000003,"5min":0.216,"15min":0.216},"max":{"1min":8.8949999999999996,"5min":57.279000000000003,"15min":105.16},"last":1.2050000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.133,"5min":2.3199999999999998,"15min":4.8600000000000003},"min":{"1min":0.40100000000000002,"5min":0.27100000000000002,"15min":0.23899999999999999},"max":{"1min":5.5789999999999997,"5min":15.191000000000001,"15min":211.405},"last":1.3260000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":2.2970000000000002,"15min":4.9299999999999997},"min":{"1min":0.34300000000000003,"5min":0.26400000000000001,"15min":0.246},"max":{"1min":8.6379999999999999,"5min":15.637,"15min":210.988},"last":1.8009999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:29:21 2023","interfaces":[{"interface":"back","average":{"1min":2.7029999999999998,"5min":3.4399999999999999,"15min":3.5649999999999999},"min":{"1min":0.35799999999999998,"5min":0.34399999999999997,"15min":0.29799999999999999},"max":{"1min":7.0190000000000001,"5min":72.602000000000004,"15min":80.570999999999998},"last":0.81799999999999995},{"interface":"front","average":{"1min":1.829,"5min":2.6829999999999998,"15min":3.1299999999999999},"min":{"1min":0.315,"5min":0.312,"15min":0.308},"max":{"1min":7.8630000000000004,"5min":73.454999999999998,"15min":80.75},"last":1.246}]},{"osd":5,"last update":"Sat Sep 23 15:29:21 2023","interfaces":[{"interface":"back","average":{"1min":1.8200000000000001,"5min":3.9969999999999999,"15min":3.5379999999999998},"min":{"1min":0.374,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":8.5250000000000004,"5min":173.18799999999999,"15min":173.18799999999999},"last":1.885},{"interface":"front","average":{"1min":2.8239999999999998,"5min":4.5869999999999997,"15min":3.899},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.7889999999999997,"5min":172.822,"15min":172.822},"last":0.87}]},{"osd":6,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":2.7250000000000001,"15min":3.044},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.0490000000000004,"5min":54.198,"15min":54.198},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.619,"5min":1.8939999999999999,"15min":2.1800000000000002},"min":{"1min":0.42699999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.5800000000000001,"5min":15.605,"15min":15.605},"last":1.9279999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":2.0990000000000002,"15min":2.125},"min":{"1min":0.46800000000000003,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":8.6120000000000001,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":1.8460000000000001},{"interface":"front","average":{"1min":2.3279999999999998,"5min":4.524,"15min":4.8899999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.7709999999999999,"5min":54.223999999999997,"15min":54.223999999999997},"last":0.89100000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893423,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":973748,"kb_used_data":298668,"kb_used_omap":8,"kb_used_meta":675063,"kb_avail":92771404,"statfs":{"total":95995035648,"available":94997917696,"internally_reserved":0,"allocated":305836032,"data_stored":303891072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":691265108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:29:12 2023","interfaces":[{"interface":"back","average":{"1min":6.9459999999999997,"5min":7.4850000000000003,"15min":4.532},"min":{"1min":0.216,"5min":0.216,"15min":0.18099999999999999},"max":{"1min":95.445999999999998,"5min":141.52199999999999,"15min":141.52199999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":7.6470000000000002,"5min":7.6539999999999999,"15min":4.3680000000000003},"min":{"1min":0.216,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":95.680000000000007,"5min":141.62799999999999,"15min":141.62799999999999},"last":0.63100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.0710000000000002,"15min":3.2650000000000001},"min":{"1min":0.23699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.75,"5min":14.994,"15min":93.873999999999995},"last":0.66500000000000004},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":3.2589999999999999},"min":{"1min":0.27300000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.5300000000000002,"5min":10.874000000000001,"15min":94.652000000000001},"last":0.68500000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:28:58 2023","interfaces":[{"interface":"back","average":{"1min":7.7430000000000003,"5min":5.5499999999999998,"15min":5.9180000000000001},"min":{"1min":0.35499999999999998,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":80.415999999999997,"5min":162.154,"15min":197.78899999999999},"last":0.58899999999999997},{"interface":"front","average":{"1min":6.327,"5min":5.1310000000000002,"15min":6.5490000000000004},"min":{"1min":0.34899999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":80.463999999999999,"5min":162.215,"15min":200.58799999999999},"last":0.71299999999999997}]},{"osd":4,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.8640000000000001,"15min":1.895},"min":{"1min":0.36399999999999999,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.832000000000001,"5min":11.887,"15min":15.358000000000001},"last":0.73899999999999999},{"interface":"front","average":{"1min":2.4359999999999999,"5min":2.258,"15min":2.0070000000000001},"min":{"1min":0.371,"5min":0.314,"15min":0.29399999999999998},"max":{"1min":9.7769999999999992,"5min":13.656000000000001,"15min":13.656000000000001},"last":0.81899999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":2.1539999999999999,"15min":2.1539999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5039999999999996,"5min":44.57,"15min":64.349999999999994},"last":0.79300000000000004},{"interface":"front","average":{"1min":2.0659999999999998,"5min":2.5920000000000001,"15min":2.2149999999999999},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.312},"max":{"1min":10.797000000000001,"5min":44.588000000000001,"15min":44.588000000000001},"last":0.84099999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":2.3180000000000001,"15min":2.6720000000000002},"min":{"1min":0.44400000000000001,"5min":0.379,"15min":0.379},"max":{"1min":11.615,"5min":32.097999999999999,"15min":32.097999999999999},"last":0.80400000000000005},{"interface":"front","average":{"1min":2.1909999999999998,"5min":2.6720000000000002,"15min":3.1379999999999999},"min":{"1min":0.45000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":9.7409999999999997,"5min":32.188000000000002,"15min":32.188000000000002},"last":0.72799999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.0590000000000002,"15min":2.2160000000000002},"min":{"1min":0.45100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.0899999999999999,"5min":10.537000000000001,"15min":10.537000000000001},"last":0.755},{"interface":"front","average":{"1min":1.119,"5min":3.5459999999999998,"15min":3.9500000000000002},"min":{"1min":0.44800000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":3.948,"5min":12.651,"15min":12.651},"last":0.873}]}]},{"osd":2,"up_from":105,"seq":450971566373,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":772712,"kb_used_data":113568,"kb_used_omap":11,"kb_used_meta":659124,"kb_avail":92972440,"statfs":{"total":95995035648,"available":95203778560,"internally_reserved":0,"allocated":116293632,"data_stored":114357408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":674943760},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:28 2023","interfaces":[{"interface":"back","average":{"1min":6.2430000000000003,"5min":5.5640000000000001,"15min":5.8899999999999997},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.17199999999999999},"max":{"1min":70.307000000000002,"5min":235.517,"15min":235.517},"last":4.6440000000000001},{"interface":"front","average":{"1min":5.8799999999999999,"5min":5.4569999999999999,"15min":5.6600000000000001},"min":{"1min":0.251,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":70.286000000000001,"5min":235.33799999999999,"15min":235.33799999999999},"last":5.9390000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:29:19 2023","interfaces":[{"interface":"back","average":{"1min":6.3380000000000001,"5min":2.9249999999999998,"15min":4.016},"min":{"1min":0.24399999999999999,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":61.506,"5min":61.506,"15min":84.641000000000005},"last":1.0509999999999999},{"interface":"front","average":{"1min":6.6619999999999999,"5min":2.766,"15min":3.6829999999999998},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":61.524999999999999,"5min":61.524999999999999,"15min":84.751999999999995},"last":0.315}]},{"osd":3,"last update":"Sat Sep 23 15:29:59 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":3.3919999999999999,"15min":5.25},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":5.2119999999999997,"5min":75.701999999999998,"15min":86.332999999999998},"last":2.21},{"interface":"front","average":{"1min":0.89300000000000002,"5min":3.198,"15min":5.2990000000000004},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":3.9670000000000001,"5min":75.388999999999996,"15min":86.158000000000001},"last":1.0189999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:29:28 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.528,"15min":2.0329999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":6.056,"5min":40.119999999999997,"15min":84.608000000000004},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.712,"15min":2.359},"min":{"1min":0.38300000000000001,"5min":0.28399999999999997,"15min":0.26200000000000001},"max":{"1min":8.4390000000000001,"5min":40.198,"15min":87.019999999999996},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:29:28 2023","interfaces":[{"interface":"back","average":{"1min":6.9889999999999999,"5min":3.472,"15min":2.8809999999999998},"min":{"1min":0.371,"5min":0.33500000000000002,"15min":0.28899999999999998},"max":{"1min":107.021,"5min":107.021,"15min":107.021},"last":0.90500000000000003},{"interface":"front","average":{"1min":6.7110000000000003,"5min":3.2610000000000001,"15min":2.7120000000000002},"min":{"1min":0.318,"5min":0.318,"15min":0.317},"max":{"1min":107.136,"5min":107.136,"15min":107.136},"last":0.73999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":1.7889999999999999,"15min":2.1339999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":8.359,"5min":9.2289999999999992,"15min":9.4600000000000009},"last":1.099},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.5960000000000001,"15min":1.827},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.5069999999999997,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":8.2170000000000005,"5min":3.6349999999999998,"15min":2.871},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":112.70699999999999,"5min":112.70699999999999,"15min":112.70699999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":7.5679999999999996,"5min":4.1559999999999997,"15min":3.5880000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":109.402,"5min":109.402,"15min":109.402},"last":0.85699999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402845,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":864320,"kb_used_data":153144,"kb_used_omap":3,"kb_used_meta":711164,"kb_avail":92880832,"statfs":{"total":95995035648,"available":95109971968,"internally_reserved":0,"allocated":156819456,"data_stored":154862322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":728232105},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0630000000000002,"5min":3.4700000000000002,"15min":3.6200000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.183},"max":{"1min":7.5359999999999996,"5min":88.489000000000004,"15min":108.318},"last":0.38700000000000001},{"interface":"front","average":{"1min":2.3849999999999998,"5min":3.8079999999999998,"15min":3.9660000000000002},"min":{"1min":0.189,"5min":0.189,"15min":0.184},"max":{"1min":14.574999999999999,"5min":88.938000000000002,"15min":108.889},"last":0.69299999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:29:18 2023","interfaces":[{"interface":"back","average":{"1min":2.927,"5min":4.6900000000000004,"15min":4.8689999999999998},"min":{"1min":0.26500000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":10.702,"5min":121.26900000000001,"15min":126.875},"last":1.165},{"interface":"front","average":{"1min":3.4900000000000002,"5min":5.1689999999999996,"15min":5.1589999999999998},"min":{"1min":0.218,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":10.023999999999999,"5min":121.012,"15min":128.249},"last":0.47399999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.915,"5min":1.7490000000000001,"15min":2.8180000000000001},"min":{"1min":0.28699999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.6980000000000004,"5min":14.853,"15min":61.732999999999997},"last":0.50800000000000001},{"interface":"front","average":{"1min":2.2250000000000001,"5min":2.0409999999999999,"15min":3.6720000000000002},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":14.343999999999999,"5min":17.091000000000001,"15min":119.449},"last":0.73099999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.5289999999999999,"15min":2.177},"min":{"1min":0.36199999999999999,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":14.4,"5min":17.027999999999999,"15min":54.051000000000002},"last":1.0169999999999999},{"interface":"front","average":{"1min":1.9099999999999999,"5min":1.8420000000000001,"15min":2.0790000000000002},"min":{"1min":0.28599999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":6.1369999999999996,"5min":13.115,"15min":52.228999999999999},"last":1.6839999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.6070000000000002,"5min":2.8130000000000002,"15min":2.9740000000000002},"min":{"1min":0.33400000000000002,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":17.195,"5min":82.793000000000006,"15min":148.619},"last":0.753},{"interface":"front","average":{"1min":2.121,"5min":2.996,"15min":2.8039999999999998},"min":{"1min":0.32300000000000001,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.2539999999999996,"5min":82.875,"15min":148.56999999999999},"last":1.1240000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:29:32 2023","interfaces":[{"interface":"back","average":{"1min":2.2559999999999998,"5min":3.6800000000000002,"15min":2.71},"min":{"1min":0.35799999999999998,"5min":0.33800000000000002,"15min":0.32700000000000001},"max":{"1min":8.8130000000000006,"5min":84.665000000000006,"15min":84.665000000000006},"last":1.198},{"interface":"front","average":{"1min":3.1539999999999999,"5min":3.7850000000000001,"15min":3.5880000000000001},"min":{"1min":0.45700000000000002,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":14.432,"5min":84.897999999999996,"15min":84.897999999999996},"last":0.78900000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":10.141999999999999,"15min":11.545999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":6.7679999999999998,"5min":216.56899999999999,"15min":216.56899999999999},"last":0.70899999999999996},{"interface":"front","average":{"1min":2.9289999999999998,"5min":11.323,"15min":12.722},"min":{"1min":0.49199999999999999,"5min":0.432,"15min":0.432},"max":{"1min":17.300000000000001,"5min":216.548,"15min":216.548},"last":0.81799999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986612,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":646340,"kb_used_data":243580,"kb_used_omap":0,"kb_used_meta":402752,"kb_avail":93098812,"statfs":{"total":95995035648,"available":95333183488,"internally_reserved":0,"allocated":249425920,"data_stored":248086205,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":412418048},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:24 2023","interfaces":[{"interface":"back","average":{"1min":1.133,"5min":3.3039999999999998,"15min":3.988},"min":{"1min":0.29199999999999998,"5min":0.26300000000000001,"15min":0.219},"max":{"1min":3.0259999999999998,"5min":159.624,"15min":162.75999999999999},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.681,"5min":3.5910000000000002,"15min":4.0919999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.221},"max":{"1min":4.9349999999999996,"5min":159.71000000000001,"15min":162.34800000000001},"last":2.1920000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:29:12 2023","interfaces":[{"interface":"back","average":{"1min":2.5329999999999999,"5min":5.2009999999999996,"15min":3.677},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.245},"max":{"1min":20.564,"5min":144.76900000000001,"15min":144.76900000000001},"last":2.645},{"interface":"front","average":{"1min":3.778,"5min":5.5110000000000001,"15min":4.0279999999999996},"min":{"1min":0.25700000000000001,"5min":0.25,"15min":0.25},"max":{"1min":20.498999999999999,"5min":144.58799999999999,"15min":144.58799999999999},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:29:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.46,"15min":2.7090000000000001},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.0019999999999998,"5min":10.933,"15min":78.751000000000005},"last":2.2549999999999999},{"interface":"front","average":{"1min":1.2410000000000001,"5min":1.5629999999999999,"15min":2.847},"min":{"1min":0.38500000000000001,"5min":0.315,"15min":0.28999999999999998},"max":{"1min":3.915,"5min":7.4699999999999998,"15min":77.930999999999997},"last":2.8039999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:29:03 2023","interfaces":[{"interface":"back","average":{"1min":6.452,"5min":2.6760000000000002,"15min":4.3159999999999998},"min":{"1min":0.371,"5min":0.34300000000000003,"15min":0.30399999999999999},"max":{"1min":96.587000000000003,"5min":96.587000000000003,"15min":156.46000000000001},"last":1.0369999999999999},{"interface":"front","average":{"1min":7.2000000000000002,"5min":2.5790000000000002,"15min":4.4219999999999997},"min":{"1min":0.33300000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":96.784999999999997,"5min":96.784999999999997,"15min":156.74299999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:29:12 2023","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":1.2989999999999999,"15min":1.272},"min":{"1min":0.33500000000000002,"5min":0.221,"15min":0.221},"max":{"1min":3.613,"5min":40.685000000000002,"15min":87.912999999999997},"last":1.7589999999999999},{"interface":"front","average":{"1min":1.4139999999999999,"5min":1.3660000000000001,"15min":1.417},"min":{"1min":0.30599999999999999,"5min":0.246,"15min":0.246},"max":{"1min":4.1029999999999998,"5min":38.439,"15min":88.602999999999994},"last":0.54700000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.88200000000000001,"15min":1.258},"min":{"1min":0.32900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.508,"5min":7.508,"15min":7.6470000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.85199999999999998,"15min":1.7330000000000001},"min":{"1min":0.318,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.806,"5min":5.1589999999999998,"15min":7.5380000000000003},"last":2.2250000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.2949999999999999,"15min":1.365},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.8839999999999999,"5min":5.7249999999999996,"15min":5.7249999999999996},"last":2.1499999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.0069999999999999,"15min":1.04},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.5700000000000003,"5min":4.8300000000000001,"15min":4.8300000000000001},"last":1.071}]}]},{"osd":5,"up_from":39,"seq":167503724975,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":649016,"kb_used_data":169584,"kb_used_omap":0,"kb_used_meta":479424,"kb_avail":93096136,"statfs":{"total":95995035648,"available":95330443264,"internally_reserved":0,"allocated":173654016,"data_stored":172322826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":490930176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:30 2023","interfaces":[{"interface":"back","average":{"1min":3.109,"5min":6.0570000000000004,"15min":4.6319999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":18.937000000000001,"5min":196.77000000000001,"15min":196.77000000000001},"last":0.871},{"interface":"front","average":{"1min":2.9820000000000002,"5min":5.9109999999999996,"15min":4.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":18.562000000000001,"5min":196.91399999999999,"15min":196.91399999999999},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 15:29:19 2023","interfaces":[{"interface":"back","average":{"1min":3.71,"5min":4.7300000000000004,"15min":4.1900000000000004},"min":{"1min":0.27800000000000002,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":22.463999999999999,"5min":193.66499999999999,"15min":193.66499999999999},"last":0.91200000000000003},{"interface":"front","average":{"1min":4.5700000000000003,"5min":4.6870000000000003,"15min":3.8969999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":28.59,"5min":193.52099999999999,"15min":193.52099999999999},"last":0.58199999999999996}]},{"osd":2,"last update":"Sat Sep 23 15:29:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.4450000000000001,"15min":2.6920000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25},"max":{"1min":6.0570000000000004,"5min":6.2069999999999999,"15min":193.00999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.98,"5min":1.7789999999999999,"15min":2.9199999999999999},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":8.8010000000000002,"5min":9.125,"15min":193.15700000000001},"last":0.66200000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:29:02 2023","interfaces":[{"interface":"back","average":{"1min":2.8980000000000001,"5min":5.0549999999999997,"15min":6.6459999999999999},"min":{"1min":0.38300000000000001,"5min":0.32300000000000001,"15min":0.29199999999999998},"max":{"1min":18.257000000000001,"5min":150.75899999999999,"15min":266.70299999999997},"last":0.73199999999999998},{"interface":"front","average":{"1min":3.339,"5min":4.9889999999999999,"15min":6.7670000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":18.271000000000001,"5min":150.62299999999999,"15min":266.80399999999997},"last":0.81799999999999995}]},{"osd":4,"last update":"Sat Sep 23 15:29:19 2023","interfaces":[{"interface":"back","average":{"1min":1.6870000000000001,"5min":1.3500000000000001,"15min":1.6319999999999999},"min":{"1min":0.32100000000000001,"5min":0.249,"15min":0.249},"max":{"1min":9.7189999999999994,"5min":9.7189999999999994,"15min":138.68600000000001},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.288,"5min":1.2669999999999999,"15min":1.619},"min":{"1min":0.33700000000000002,"5min":0.33200000000000002,"15min":0.27200000000000002},"max":{"1min":4.7140000000000004,"5min":8.6349999999999998,"15min":138.49600000000001},"last":0.79900000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.2250000000000001,"15min":2.02},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":2.992,"5min":8.6989999999999998,"15min":10.577999999999999},"last":0.433},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.1559999999999999,"15min":2.0659999999999998},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.944,"5min":6.242,"15min":13.257999999999999},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.319,"15min":1.2949999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.7400000000000002,"5min":9.7400000000000002,"15min":9.7400000000000002},"last":0.84699999999999998},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.5009999999999999,"15min":1.5840000000000001},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.081,"5min":6.0060000000000002,"15min":6.0060000000000002},"last":0.47499999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":199032832,"data_stored":199032832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":129368064,"data_stored":129368064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":165019648,"data_stored":165019648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":168624128,"data_stored":168624128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":263651328,"data_stored":263651328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":179372032,"data_stored":179372032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":203948032,"data_stored":203948032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":110231552,"data_stored":110231552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:30:05.610 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T15:30:05.610 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T15:30:05.660 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:05.739 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:05.822 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:05 smithi017 ceph-mon[222878]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:05.822 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:05 smithi017 ceph-mon[215646]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:05.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:05 smithi027 ceph-mon[195233]: from='client.36624 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:05.894 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:05.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:05.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:06.008 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:06.052 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:06.138 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:06.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:06.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:06.248 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:06.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 9 16 22149 22133 153.593 118.312 0.00844396 0.00650527 2023-09-23T15:30:06.305 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:06.560 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:06.562 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:06 smithi017 ceph-mon[215646]: pgmap v1217: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:30:06.563 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:06 smithi017 ceph-mon[222878]: pgmap v1217: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:30:06.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:06.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:06.664 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:06.707 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:06.752 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:06.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:06.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:06.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:06 smithi027 ceph-mon[195233]: pgmap v1217: 119 pgs: 119 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T15:30:06.856 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:06.882 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:06.938 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:06.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:06.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:07.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:07.098 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:07.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 10 16 23779 23763 148.424 101.875 0.00516337 0.00673333 2023-09-23T15:30:07.425 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:07.450 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:07.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:07.555 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:07.573 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:30:07.611 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:07.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:07.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:07.705 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:07.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:07.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:07.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:07.875 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:07.962 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:07.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:07.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:08.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:08.132 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:08.272 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 11 16 25831 25815 146.59 128.25 0.00455443 0.00681643 2023-09-23T15:30:08.384 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:08.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:08.389 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:08] "GET /metrics HTTP/1.1" 200 36350 "" "Prometheus/2.43.0" 2023-09-23T15:30:08.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:08.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:08.518 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: pgmap v1218: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[215646]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: pgmap v1218: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:08 smithi017 ceph-mon[222878]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: pgmap v1218: 119 pgs: 119 active+clean; 1.2 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 104 MiB/s wr, 1.66k op/s 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1874332753' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: osdmap e418: 8 total, 8 up, 8 in 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: failed to encode map e418 with expected crc 2023-09-23T15:30:08.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:08 smithi027 ceph-mon[195233]: failed to encode map e418 with expected crc 2023-09-23T15:30:09.028 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:09.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:09.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:09.138 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:09.199 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:09.271 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 12 16 27545 27529 143.302 107.125 0.00423166 0.00697201 2023-09-23T15:30:09.548 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:09.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:09.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:09.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:09.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:09.786 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:30:09.786 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:30:09.786 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:30:09.786 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:30:09.786 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:30:10.272 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 13 16 29250 29234 140.476 106.562 0.00786673 0.00711293 2023-09-23T15:30:10.656 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:10.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:10 smithi027 ceph-mon[195233]: pgmap v1220: 119 pgs: 119 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-09-23T15:30:10.876 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:10.884 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:10 smithi017 ceph-mon[215646]: pgmap v1220: 119 pgs: 119 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-09-23T15:30:10.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:10 smithi017 ceph-mon[222878]: pgmap v1220: 119 pgs: 119 active+clean; 1.2 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 122 MiB/s wr, 1.94k op/s 2023-09-23T15:30:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:10.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:10.986 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:11.020 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:30:11.133 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:11.267 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:11.272 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 14 16 30889 30873 137.759 102.438 0.00383858 0.0072524 2023-09-23T15:30:11.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:11.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:11.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:11.449 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:11.625 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:11.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:11.633 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:11.733 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:11.773 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:11.781 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:11.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:11.833 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:11.838 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:11.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:11.860 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:11.895 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:11.934 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:11.939 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:11.971 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:12.006 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:12.007 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:12.013 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.014 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:30:12.116 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:12.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:12.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:12.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:12.272 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 15 16 32598 32582 135.696 106.812 0.0076197 0.00736551 2023-09-23T15:30:12.291 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:12.683 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:30:12.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:12 smithi027 ceph-mon[195233]: pgmap v1221: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:12.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/773632370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:12.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:12 smithi017 ceph-mon[222878]: pgmap v1221: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/773632370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:12 smithi017 ceph-mon[215646]: pgmap v1221: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/773632370' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:13.273 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 16 16 33758 33742 131.738 72.5 0.00949781 0.00758593 2023-09-23T15:30:13.403 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:13.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:13.508 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:13.555 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:13.685 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:13.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:13.693 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:13.776 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:13.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:13.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:13.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:13.890 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:13.905 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:14.099 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:14.180 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:14.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:14.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:14.273 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 17 16 35514 35498 130.445 109.75 0.0104809 0.0076597 2023-09-23T15:30:14.289 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:14.350 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:14.577 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:14 smithi017 ceph-mon[215646]: pgmap v1222: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:14.577 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:14 smithi017 ceph-mon[222878]: pgmap v1222: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:14.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:14 smithi027 ceph-mon[195233]: pgmap v1222: 119 pgs: 119 active+clean; 1.7 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 130 MiB/s wr, 2.08k op/s 2023-09-23T15:30:15.222 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T15:30:15.273 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 18 16 36919 36903 128.076 87.8125 0.0173017 0.00770172 2023-09-23T15:30:15.590 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2814295900' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T15:30:15.626 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:15.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:15.632 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:15.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:30:15.733 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:15.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2814295900' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T15:30:15.850 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2814295900' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T15:30:15.850 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:16.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 19 16 38244 38228 125.694 82.8125 0.00584154 0.00795161 2023-09-23T15:30:16.476 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T15:30:16.727 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:16 smithi017 ceph-mon[215646]: pgmap v1223: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T15:30:16.728 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:16 smithi017 ceph-mon[222878]: pgmap v1223: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T15:30:16.761 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:16.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:16.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:16.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:16 smithi027 ceph-mon[195233]: pgmap v1223: 119 pgs: 119 active+clean; 2.1 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T15:30:16.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:16.922 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:17.048 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:17.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:17.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:17.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:17.199 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:17.242 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:17.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:17.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:17.273 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:30:17.274496+0000 min lat: 0.00117915 max lat: 0.408537 avg lat: 0.00800433 2023-09-23T15:30:17.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:30:17.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 20 16 39788 39772 124.235 96.5 0.00480159 0.00800433 2023-09-23T15:30:17.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:17.419 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:17.649 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:17.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:17.708 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:17.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:17.854 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:17.924 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:30:17.924 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:30:18.083 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:18.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:18.092 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:18.107 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:18] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T15:30:18.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:18.246 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:18.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 21 16 41337 41321 122.927 96.8125 0.0049788 0.00813044 2023-09-23T15:30:18.333 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:18.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:18.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:18.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:18.473 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:18.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:18 smithi017 ceph-mon[222878]: pgmap v1224: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T15:30:18.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:18 smithi017 ceph-mon[215646]: pgmap v1224: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T15:30:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:18 smithi027 ceph-mon[195233]: pgmap v1224: 119 pgs: 119 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2023-09-23T15:30:19.138 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:19.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:19.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:19.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:30:19.245 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:19.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 22 16 42654 42638 121.081 82.3125 0.00984136 0.00825378 2023-09-23T15:30:19.312 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:19.405 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:19.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:19.413 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:19.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:19.578 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:19.643 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:30:19.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3135077166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:19.678 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:19.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3135077166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:19.680 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:19.693 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:19.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:19.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:19.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:19.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:19.956 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:20.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3135077166' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:20.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:20.274 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 23 16 44197 44181 120.009 96.4375 0.00583115 0.00832656 2023-09-23T15:30:20.625 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: pgmap v1225: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T15:30:20.626 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[222878]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: pgmap v1225: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T15:30:20.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T15:30:20.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:20 smithi027 ceph-mon[195233]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.910 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:20.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:20.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:20.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: pgmap v1225: 119 pgs: 119 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s 2023-09-23T15:30:20.917 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: failed to encode map e419 with expected crc 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: osdmap e419: 8 total, 8 up, 8 in 2023-09-23T15:30:20.918 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:20 smithi017 ceph-mon[215646]: failed to encode map e419 with expected crc 2023-09-23T15:30:21.018 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:21.135 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:21.222 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:21.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:21.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:21.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 24 16 45752 45736 119.04 97.1875 0.011 0.00839647 2023-09-23T15:30:21.327 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:21.387 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:21.477 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:30:21.477 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:30:21.477 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T15:30:21.477 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2023-09-23T15:30:21.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0 2023-09-23T15:30:21.500 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:21.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:21.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:21.606 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:21.712 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:21.733 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:21.770 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:21.817 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:21.851 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:21.854 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:21.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:21.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:21.895 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:21.931 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:21.962 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:21.969 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:22.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.028 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:30:22.030 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:22.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 25 16 47165 47149 117.811 88.3125 0.00570559 0.00848295 2023-09-23T15:30:22.325 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:22.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:22.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:22.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:22.632 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:22.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:22.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:22.740 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:22.787 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:22 smithi027 ceph-mon[195233]: pgmap v1227: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:22.787 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2028431304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:22.791 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:22.928 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:30:22.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:22 smithi017 ceph-mon[222878]: pgmap v1227: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:22.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2028431304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:22.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:22 smithi017 ceph-mon[215646]: pgmap v1227: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:22.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2028431304' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:23.031 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:23.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:23.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:23.134 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:23.138 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:23.210 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:23.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 26 16 48678 48662 116.917 94.5625 0.0117658 0.00854789 2023-09-23T15:30:23.412 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:23.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:23.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:23.523 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:23.596 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:23.705 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:23.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:23.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:49273 10.4 e419/413 10:2196f6f5:::benchmark_data_smithi017_604517_object49272:head v 419'2975, mlcod=419'2973)", 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:23.711129+0000", 2023-09-23T15:30:23.717 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0037925089999999999, 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003820094, 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711129+0000", 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:23.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711129+0000", 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711128+0000", 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711180+0000", 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1765999999999999e-05 2023-09-23T15:30:23.719 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711192+0000", 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1783999999999999e-05 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711198+0000", 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2879999999999998e-06 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:23.720 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711241+0000", 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2710000000000003e-05 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.711258+0000", 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7147999999999999e-05 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:49270 10.e 10:75eee3b8:::benchmark_data_smithi017_604517_object49269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T15:30:23.721 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:23.712371+0000", 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0025507559999999999, 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0026584080000000001, 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 49270 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.722 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712371+0000", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712371+0000", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712369+0000", 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:23.723 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712412+0000", 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2985000000000003e-05 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712424+0000", 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2181000000000001e-05 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.724 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712431+0000", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4930000000000003e-06 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712565+0000", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134387 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712694+0000", 2023-09-23T15:30:23.725 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129004 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712836+0000", 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014185700000000001 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.715017+0000", 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021812609999999999 2023-09-23T15:30:23.726 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:49276 10.1 10:829231ed:::benchmark_data_smithi017_604517_object49275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:23.712466+0000", 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0024553309999999998, 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002651156, 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:23.727 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 49276 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712466+0000", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712466+0000", 2023-09-23T15:30:23.728 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712465+0000", 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712500+0000", 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4501000000000001e-05 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.729 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712512+0000", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2211e-05 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712518+0000", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1469999999999998e-06 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712687+0000", 2023-09-23T15:30:23.730 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016905499999999999 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712798+0000", 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011086000000000001 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.712934+0000", 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013551500000000001 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.731 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.715008+0000", 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020742830000000001 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:49269 10.e 10:77c75077:::benchmark_data_smithi017_604517_object49268:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:23.708266+0000", 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0066555410000000001, 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069449070000000002, 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:23.732 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 49269 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708266+0000", 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.733 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708266+0000", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708264+0000", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708345+0000", 2023-09-23T15:30:23.734 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1599000000000003e-05 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708363+0000", 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7093999999999999e-05 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708370+0000", 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9100000000000005e-06 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.735 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708421+0000", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0473999999999998e-05 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708603+0000", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000181717 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.708737+0000", 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013466899999999999 2023-09-23T15:30:23.736 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.714584+0000", 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0058463330000000004 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:23.714593+0000", 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1789999999999997e-06 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.737 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:23.738 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:30:23.738 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 4 2023-09-23T15:30:23.738 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:30:23.811 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:23.877 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:23.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:23.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:23.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:24.042 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:24.069 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:24.112 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:24.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:24.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:24.216 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:24.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 27 16 49725 49709 115.011 65.4375 0.01431 0.00869 2023-09-23T15:30:24.280 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:24.491 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:24.492 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:24.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:24.498 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:24.599 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:24.640 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:24 smithi017 ceph-mon[215646]: pgmap v1228: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:24.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:24 smithi017 ceph-mon[222878]: pgmap v1228: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:24.659 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:25.043 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:24 smithi027 ceph-mon[195233]: pgmap v1228: 119 pgs: 119 active+clean; 2.6 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.51k op/s 2023-09-23T15:30:25.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:25.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:25.082 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:30:25.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:25.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:30:25.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:25.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 28 16 50947 50931 113.632 76.375 0.0156454 0.00879466 2023-09-23T15:30:25.314 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:25.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:25 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:30:25.661 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:30:25.662 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 0 (802) 2023-09-23T15:30:25.809 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:25.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:25 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:30:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:30:26.091 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:26.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:26.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:25 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:30:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]: dispatch 2023-09-23T15:30:26.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:26.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 29 16 51774 51758 111.496 51.6875 0.00520453 0.00896325 2023-09-23T15:30:26.285 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:26.615 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:26.854 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: pgmap v1229: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[215646]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: pgmap v1229: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: failed to encode map e420 with expected crc 2023-09-23T15:30:26.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:26 smithi017 ceph-mon[222878]: failed to encode map e420 with expected crc 2023-09-23T15:30:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: pgmap v1229: 119 pgs: 119 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 90 MiB/s wr, 1.44k op/s 2023-09-23T15:30:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: failed to encode map e420 with expected crc 2023-09-23T15:30:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/246179395' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2023-09-23T15:30:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: osdmap e420: 8 total, 8 up, 8 in 2023-09-23T15:30:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: failed to encode map e420 with expected crc 2023-09-23T15:30:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: failed to encode map e420 with expected crc 2023-09-23T15:30:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:26 smithi027 ceph-mon[195233]: failed to encode map e420 with expected crc 2023-09-23T15:30:27.278 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 30 16 52875 52859 110.074 68.8125 0.0128758 0.00907996 2023-09-23T15:30:27.332 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:27.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:27.440 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:27.572 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:27.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T15:30:27.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: 1.19 scrub starts 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[215646]: 1.19 scrub ok 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.687 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: 1.19 scrub starts 2023-09-23T15:30:27.688 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:27 smithi017 ceph-mon[222878]: 1.19 scrub ok 2023-09-23T15:30:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: osdmap e421: 8 total, 8 up, 8 in 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: failed to encode map e421 with expected crc 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: 1.19 scrub starts 2023-09-23T15:30:27.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:27 smithi027 ceph-mon[195233]: 1.19 scrub ok 2023-09-23T15:30:27.958 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:27.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:28.063 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:28.122 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:28.279 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 31 16 53695 53679 108.177 51.25 0.0113973 0.00923912 2023-09-23T15:30:28.445 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:28] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.43.0" 2023-09-23T15:30:28.491 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:28.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:28.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:28.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:28.653 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:28.669 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:30:28.670 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:30:28.670 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:30:28.670 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:30:28.670 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:30:28.735 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[222878]: pgmap v1232: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[215646]: pgmap v1232: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T15:30:28.763 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:30:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:30:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:28.764 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:28.769 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:28.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:28.875 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:28.943 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:28 smithi027 ceph-mon[195233]: pgmap v1232: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2023-09-23T15:30:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:28 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T15:30:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:30:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:29.279 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 32 16 54667 54651 106.692 60.75 0.00688155 0.00936767 2023-09-23T15:30:29.567 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:29.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:29.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:29.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:29.729 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:29.815 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:30:29.831 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:29.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:29.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: 1.1e scrub starts 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: 1.1e scrub ok 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: failed to encode map e422 with expected crc 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: failed to encode map e422 with expected crc 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[215646]: pgmap v1234: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T15:30:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: 1.1e scrub starts 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: 1.1e scrub ok 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: failed to encode map e422 with expected crc 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: failed to encode map e422 with expected crc 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T15:30:29.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:29 smithi017 ceph-mon[222878]: pgmap v1234: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T15:30:29.952 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:30.024 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:30.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: 1.1e scrub starts 2023-09-23T15:30:30.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: 1.1e scrub ok 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: failed to encode map e422 with expected crc 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: failed to encode map e422 with expected crc 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2995283201' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: osdmap e422: 8 total, 8 up, 8 in 2023-09-23T15:30:30.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:29 smithi027 ceph-mon[195233]: pgmap v1234: 119 pgs: 3 peering, 116 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2023-09-23T15:30:30.108 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:30.161 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:30.180 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:30.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:30.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:30.200 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:30.242 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:30.279 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 33 16 55777 55761 105.562 69.375 0.00614791 0.00946829 2023-09-23T15:30:30.284 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:30.291 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:30.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:30.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:30.386 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:30.410 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.412 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:30:30.490 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:30.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:30.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:30.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:30.629 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:30.660 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4235993978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:31.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4235993978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4235993978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:31.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 34 16 56823 56807 104.37 65.375 0.0156219 0.00957683 2023-09-23T15:30:31.398 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:31.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:31.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:31.566 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:31.569 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:30:31.569 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:30:31.569 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T15:30:31.569 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:30:31.735 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:31 smithi027 ceph-mon[195233]: pgmap v1235: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:30:31.809 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:30:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:31 smithi017 ceph-mon[222878]: pgmap v1235: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:30:32.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:31 smithi017 ceph-mon[215646]: pgmap v1235: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 62 MiB/s wr, 987 op/s 2023-09-23T15:30:32.247 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:32.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 35 16 57777 57761 103.092 59.625 0.0232567 0.00969274 2023-09-23T15:30:32.386 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:32.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:32.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:32.491 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:32.567 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:32.749 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:32 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:30:32.779 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:32.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:32.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:32.884 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:32.884 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:32 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:30:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:32 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 3 pgs inactive, 3 pgs peering) 2023-09-23T15:30:32.957 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:33.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 36 16 58914 58898 102.203 71.0625 0.00299489 0.00976823 2023-09-23T15:30:33.377 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:30:33.377 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:30:33.485 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:33.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:33.492 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:33.590 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:30:33.592 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:33.668 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:33.800 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:33.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:33.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:33.909 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:33.938 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:34.045 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:34.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:34.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:34 smithi027 ceph-mon[195233]: pgmap v1236: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T15:30:34.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:34 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.151 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:34.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[215646]: pgmap v1236: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[222878]: pgmap v1236: 119 pgs: 119 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 44 MiB/s wr, 697 op/s 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:34 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:34.197 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:34.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 37 16 59809 59793 100.953 55.9375 0.00834904 0.00990125 2023-09-23T15:30:34.442 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:34.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:30:34.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:34.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:34.557 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:34.617 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:34.664 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:34.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:34.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:34.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:34.811 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:34.924 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:34.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:34.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:35.023 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:35.031 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:35.097 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:30:35.097 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:30:35.097 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:35.106 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:35.244 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:35.250 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:35.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 38 16 60831 60815 99.9773 63.875 0.0174032 0.00999655 2023-09-23T15:30:35.342 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:35.350 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:35.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:30:35.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:30:35.361 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:35.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:30:35.361 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:30:35.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:35.408 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:35.491 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:35.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:35.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:35.597 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:35.665 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:35.854 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:35.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:35.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:35.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:36.011 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:30:36.024 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:30:36.036 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:36.284 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 39 16 61666 61650 98.7521 52.1875 0.00305909 0.0100552 2023-09-23T15:30:36.544 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:36 smithi017 ceph-mon[215646]: pgmap v1237: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:30:36.544 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:36 smithi017 ceph-mon[222878]: pgmap v1237: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:30:36.556 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:36.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:36.565 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:36.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:36.723 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:36.803 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:36.856 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:36 smithi027 ceph-mon[195233]: pgmap v1237: 119 pgs: 119 active+clean; 3.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2023-09-23T15:30:36.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:36.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:36.957 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:36.990 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:37.039 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:37.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:37.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:37.143 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:37.212 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:37.273 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:37.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:37.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:37.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:30:37.284274+0000 min lat: 0.00117915 max lat: 0.408537 avg lat: 0.0101947 2023-09-23T15:30:37.283 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:30:37.284 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 40 16 62733 62717 97.9506 66.6875 0.00275747 0.0101947 2023-09-23T15:30:37.297 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:30:37.297 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:30:37.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:37.465 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:37.653 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1237,"stamp":"2023-09-23T15:30:34.917208+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3913455539,"num_objects":59671,"num_object_clones":0,"num_object_copies":179013,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59671,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":60551,"num_write_kb":3829941,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":4466766,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65668,"ondisk_log_size":65668,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15554628,"kb_used_data":11500100,"kb_used_omap":57,"kb_used_meta":4054406,"kb_avail":734406588,"statfs":{"total":767960285184,"available":752032346112,"internally_reserved":0,"allocated":11776102400,"data_stored":11762617220,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4151711769},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":622755840,"num_objects":9504,"num_object_clones":0,"num_object_copies":28512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9505,"num_write_kb":608320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.244009"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":769,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683326+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:30:26.683326+0000","last_peered":"2023-09-23T15:30:26.683326+0000","last_clean":"2023-09-23T15:30:26.683326+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:30:26.683326+0000","last_undegraded":"2023-09-23T15:30:26.683326+0000","last_fullsized":"2023-09-23T15:30:26.683326+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680914+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:30:26.680914+0000","last_peered":"2023-09-23T15:30:26.680914+0000","last_clean":"2023-09-23T15:30:26.680914+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:30:26.680914+0000","last_undegraded":"2023-09-23T15:30:26.680914+0000","last_fullsized":"2023-09-23T15:30:26.680914+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":327,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.693473+0000","last_change":"2023-09-23T15:30:26.692661+0000","last_active":"2023-09-23T15:30:26.693473+0000","last_peered":"2023-09-23T15:30:26.693473+0000","last_clean":"2023-09-23T15:30:26.693473+0000","last_became_active":"2023-09-23T15:30:26.692331+0000","last_became_peered":"2023-09-23T15:30:26.692331+0000","last_unstale":"2023-09-23T15:30:26.693473+0000","last_undegraded":"2023-09-23T15:30:26.693473+0000","last_fullsized":"2023-09-23T15:30:26.693473+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:03.180374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":503,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683684+0000","last_change":"2023-09-23T15:28:55.922068+0000","last_active":"2023-09-23T15:30:26.683684+0000","last_peered":"2023-09-23T15:30:26.683684+0000","last_clean":"2023-09-23T15:30:26.683684+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:30:26.683684+0000","last_undegraded":"2023-09-23T15:30:26.683684+0000","last_fullsized":"2023-09-23T15:30:26.683684+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:55.921999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:55.921999+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:38.044680+0000","scrub_duration":0.0074994720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680201+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:30:26.680201+0000","last_peered":"2023-09-23T15:30:26.680201+0000","last_clean":"2023-09-23T15:30:26.680201+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:30:26.680201+0000","last_undegraded":"2023-09-23T15:30:26.680201+0000","last_fullsized":"2023-09-23T15:30:26.680201+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":417,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680286+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:30:26.680286+0000","last_peered":"2023-09-23T15:30:26.680286+0000","last_clean":"2023-09-23T15:30:26.680286+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:30:26.680286+0000","last_undegraded":"2023-09-23T15:30:26.680286+0000","last_fullsized":"2023-09-23T15:30:26.680286+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":557,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681054+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:30:26.681054+0000","last_peered":"2023-09-23T15:30:26.681054+0000","last_clean":"2023-09-23T15:30:26.681054+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:30:26.681054+0000","last_undegraded":"2023-09-23T15:30:26.681054+0000","last_fullsized":"2023-09-23T15:30:26.681054+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":358,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.715891+0000","last_change":"2023-09-23T15:30:26.692470+0000","last_active":"2023-09-23T15:30:28.715891+0000","last_peered":"2023-09-23T15:30:28.715891+0000","last_clean":"2023-09-23T15:30:28.715891+0000","last_became_active":"2023-09-23T15:30:26.692148+0000","last_became_peered":"2023-09-23T15:30:26.692148+0000","last_unstale":"2023-09-23T15:30:28.715891+0000","last_undegraded":"2023-09-23T15:30:28.715891+0000","last_fullsized":"2023-09-23T15:30:28.715891+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:32:36.258289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":372,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689387+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:30:26.689387+0000","last_peered":"2023-09-23T15:30:26.689387+0000","last_clean":"2023-09-23T15:30:26.689387+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:30:26.689387+0000","last_undegraded":"2023-09-23T15:30:26.689387+0000","last_fullsized":"2023-09-23T15:30:26.689387+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":690,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687681+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:30:26.687681+0000","last_peered":"2023-09-23T15:30:26.687681+0000","last_clean":"2023-09-23T15:30:26.687681+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:30:26.687681+0000","last_undegraded":"2023-09-23T15:30:26.687681+0000","last_fullsized":"2023-09-23T15:30:26.687681+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":371,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680158+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:30:26.680158+0000","last_peered":"2023-09-23T15:30:26.680158+0000","last_clean":"2023-09-23T15:30:26.680158+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:30:26.680158+0000","last_undegraded":"2023-09-23T15:30:26.680158+0000","last_fullsized":"2023-09-23T15:30:26.680158+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":355,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.708005+0000","last_change":"2023-09-23T15:30:26.706619+0000","last_active":"2023-09-23T15:30:26.708005+0000","last_peered":"2023-09-23T15:30:26.708005+0000","last_clean":"2023-09-23T15:30:26.708005+0000","last_became_active":"2023-09-23T15:30:26.706343+0000","last_became_peered":"2023-09-23T15:30:26.706343+0000","last_unstale":"2023-09-23T15:30:26.708005+0000","last_undegraded":"2023-09-23T15:30:26.708005+0000","last_fullsized":"2023-09-23T15:30:26.708005+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:47.811881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":831,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682438+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:30:26.682438+0000","last_peered":"2023-09-23T15:30:26.682438+0000","last_clean":"2023-09-23T15:30:26.682438+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:30:26.682438+0000","last_undegraded":"2023-09-23T15:30:26.682438+0000","last_fullsized":"2023-09-23T15:30:26.682438+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":438,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.684432+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:30:26.684432+0000","last_peered":"2023-09-23T15:30:26.684432+0000","last_clean":"2023-09-23T15:30:26.684432+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:30:26.684432+0000","last_undegraded":"2023-09-23T15:30:26.684432+0000","last_fullsized":"2023-09-23T15:30:26.684432+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":328,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.705488+0000","last_change":"2023-09-23T15:30:26.703714+0000","last_active":"2023-09-23T15:30:26.705488+0000","last_peered":"2023-09-23T15:30:26.705488+0000","last_clean":"2023-09-23T15:30:26.705488+0000","last_became_active":"2023-09-23T15:30:26.703402+0000","last_became_peered":"2023-09-23T15:30:26.703402+0000","last_unstale":"2023-09-23T15:30:26.705488+0000","last_undegraded":"2023-09-23T15:30:26.705488+0000","last_fullsized":"2023-09-23T15:30:26.705488+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:48.014628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":524,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683889+0000","last_change":"2023-09-23T15:28:59.984242+0000","last_active":"2023-09-23T15:30:26.683889+0000","last_peered":"2023-09-23T15:30:26.683889+0000","last_clean":"2023-09-23T15:30:26.683889+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:30:26.683889+0000","last_undegraded":"2023-09-23T15:30:26.683889+0000","last_fullsized":"2023-09-23T15:30:26.683889+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:59.984131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:59.984131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:34.209362+0000","scrub_duration":0.015840911999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":167,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687955+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:30:26.687955+0000","last_peered":"2023-09-23T15:30:26.687955+0000","last_clean":"2023-09-23T15:30:26.687955+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:30:26.687955+0000","last_undegraded":"2023-09-23T15:30:26.687955+0000","last_fullsized":"2023-09-23T15:30:26.687955+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":315,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689950+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:30:26.689950+0000","last_peered":"2023-09-23T15:30:26.689950+0000","last_clean":"2023-09-23T15:30:26.689950+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:30:26.689950+0000","last_undegraded":"2023-09-23T15:30:26.689950+0000","last_fullsized":"2023-09-23T15:30:26.689950+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":572,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689611+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:30:26.689611+0000","last_peered":"2023-09-23T15:30:26.689611+0000","last_clean":"2023-09-23T15:30:26.689611+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:30:26.689611+0000","last_undegraded":"2023-09-23T15:30:26.689611+0000","last_fullsized":"2023-09-23T15:30:26.689611+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":88,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680655+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:30:26.680655+0000","last_peered":"2023-09-23T15:30:26.680655+0000","last_clean":"2023-09-23T15:30:26.680655+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:30:26.680655+0000","last_undegraded":"2023-09-23T15:30:26.680655+0000","last_fullsized":"2023-09-23T15:30:26.680655+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682609+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:30:26.682609+0000","last_peered":"2023-09-23T15:30:26.682609+0000","last_clean":"2023-09-23T15:30:26.682609+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:30:26.682609+0000","last_undegraded":"2023-09-23T15:30:26.682609+0000","last_fullsized":"2023-09-23T15:30:26.682609+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":733,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680591+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:30:26.680591+0000","last_peered":"2023-09-23T15:30:26.680591+0000","last_clean":"2023-09-23T15:30:26.680591+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:30:26.680591+0000","last_undegraded":"2023-09-23T15:30:26.680591+0000","last_fullsized":"2023-09-23T15:30:26.680591+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":171,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687396+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:30:26.687396+0000","last_peered":"2023-09-23T15:30:26.687396+0000","last_clean":"2023-09-23T15:30:26.687396+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:30:26.687396+0000","last_undegraded":"2023-09-23T15:30:26.687396+0000","last_fullsized":"2023-09-23T15:30:26.687396+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":377,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.706003+0000","last_change":"2023-09-23T15:30:26.705220+0000","last_active":"2023-09-23T15:30:26.706003+0000","last_peered":"2023-09-23T15:30:26.706003+0000","last_clean":"2023-09-23T15:30:26.706003+0000","last_became_active":"2023-09-23T15:30:26.704922+0000","last_became_peered":"2023-09-23T15:30:26.704922+0000","last_unstale":"2023-09-23T15:30:26.706003+0000","last_undegraded":"2023-09-23T15:30:26.706003+0000","last_fullsized":"2023-09-23T15:30:26.706003+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:25.436067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":796,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683458+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:30:26.683458+0000","last_peered":"2023-09-23T15:30:26.683458+0000","last_clean":"2023-09-23T15:30:26.683458+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:30:26.683458+0000","last_undegraded":"2023-09-23T15:30:26.683458+0000","last_fullsized":"2023-09-23T15:30:26.683458+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":416,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680880+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:30:26.680880+0000","last_peered":"2023-09-23T15:30:26.680880+0000","last_clean":"2023-09-23T15:30:26.680880+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:30:26.680880+0000","last_undegraded":"2023-09-23T15:30:26.680880+0000","last_fullsized":"2023-09-23T15:30:26.680880+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":423,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.684023+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:30:26.684023+0000","last_peered":"2023-09-23T15:30:26.684023+0000","last_clean":"2023-09-23T15:30:26.684023+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:30:26.684023+0000","last_undegraded":"2023-09-23T15:30:26.684023+0000","last_fullsized":"2023-09-23T15:30:26.684023+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":748,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690293+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:30:26.690293+0000","last_peered":"2023-09-23T15:30:26.690293+0000","last_clean":"2023-09-23T15:30:26.690293+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:30:26.690293+0000","last_undegraded":"2023-09-23T15:30:26.690293+0000","last_fullsized":"2023-09-23T15:30:26.690293+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687430+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:30:26.687430+0000","last_peered":"2023-09-23T15:30:26.687430+0000","last_clean":"2023-09-23T15:30:26.687430+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:30:26.687430+0000","last_undegraded":"2023-09-23T15:30:26.687430+0000","last_fullsized":"2023-09-23T15:30:26.687430+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":404,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681867+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:30:26.681867+0000","last_peered":"2023-09-23T15:30:26.681867+0000","last_clean":"2023-09-23T15:30:26.681867+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:30:26.681867+0000","last_undegraded":"2023-09-23T15:30:26.681867+0000","last_fullsized":"2023-09-23T15:30:26.681867+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":737,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683830+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:30:26.683830+0000","last_peered":"2023-09-23T15:30:26.683830+0000","last_clean":"2023-09-23T15:30:26.683830+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:30:26.683830+0000","last_undegraded":"2023-09-23T15:30:26.683830+0000","last_fullsized":"2023-09-23T15:30:26.683830+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":358,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.686070+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:30:26.686070+0000","last_peered":"2023-09-23T15:30:26.686070+0000","last_clean":"2023-09-23T15:30:26.686070+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:30:26.686070+0000","last_undegraded":"2023-09-23T15:30:26.686070+0000","last_fullsized":"2023-09-23T15:30:26.686070+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":378,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.706653+0000","last_change":"2023-09-23T15:30:26.706427+0000","last_active":"2023-09-23T15:30:26.706653+0000","last_peered":"2023-09-23T15:30:26.706653+0000","last_clean":"2023-09-23T15:30:26.706653+0000","last_became_active":"2023-09-23T15:30:26.706086+0000","last_became_peered":"2023-09-23T15:30:26.706086+0000","last_unstale":"2023-09-23T15:30:26.706653+0000","last_undegraded":"2023-09-23T15:30:26.706653+0000","last_fullsized":"2023-09-23T15:30:26.706653+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:42.578372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"422'3746","reported_seq":3762,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.782903+0000","last_change":"2023-09-23T15:29:48.688598+0000","last_active":"2023-09-23T15:30:33.782903+0000","last_peered":"2023-09-23T15:30:33.782903+0000","last_clean":"2023-09-23T15:30:33.782903+0000","last_became_active":"2023-09-23T15:29:48.688227+0000","last_became_peered":"2023-09-23T15:29:48.688227+0000","last_unstale":"2023-09-23T15:30:33.782903+0000","last_undegraded":"2023-09-23T15:30:33.782903+0000","last_fullsized":"2023-09-23T15:30:33.782903+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3746,"log_dups_size":0,"ondisk_log_size":3746,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:04.960560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245497856,"num_objects":3746,"num_object_clones":0,"num_object_copies":11238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3746,"num_write_kb":239744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":718,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.688027+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:30:26.688027+0000","last_peered":"2023-09-23T15:30:26.688027+0000","last_clean":"2023-09-23T15:30:26.688027+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:30:26.688027+0000","last_undegraded":"2023-09-23T15:30:26.688027+0000","last_fullsized":"2023-09-23T15:30:26.688027+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":161,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.688682+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:30:26.688682+0000","last_peered":"2023-09-23T15:30:26.688682+0000","last_clean":"2023-09-23T15:30:26.688682+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:30:26.688682+0000","last_undegraded":"2023-09-23T15:30:26.688682+0000","last_fullsized":"2023-09-23T15:30:26.688682+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":430,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683358+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:30:26.683358+0000","last_peered":"2023-09-23T15:30:26.683358+0000","last_clean":"2023-09-23T15:30:26.683358+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:30:26.683358+0000","last_undegraded":"2023-09-23T15:30:26.683358+0000","last_fullsized":"2023-09-23T15:30:26.683358+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"422'3660","reported_seq":3676,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.570306+0000","last_change":"2023-09-23T15:29:48.696982+0000","last_active":"2023-09-23T15:30:34.570306+0000","last_peered":"2023-09-23T15:30:34.570306+0000","last_clean":"2023-09-23T15:30:34.570306+0000","last_became_active":"2023-09-23T15:29:48.696666+0000","last_became_peered":"2023-09-23T15:29:48.696666+0000","last_unstale":"2023-09-23T15:30:34.570306+0000","last_undegraded":"2023-09-23T15:30:34.570306+0000","last_fullsized":"2023-09-23T15:30:34.570306+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3660,"log_dups_size":0,"ondisk_log_size":3660,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:34.479018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239861760,"num_objects":3660,"num_object_clones":0,"num_object_copies":10980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3660,"num_write_kb":234240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":577,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.716154+0000","last_change":"2023-09-23T15:30:26.693915+0000","last_active":"2023-09-23T15:30:28.716154+0000","last_peered":"2023-09-23T15:30:28.716154+0000","last_clean":"2023-09-23T15:30:28.716154+0000","last_became_active":"2023-09-23T15:30:26.693580+0000","last_became_peered":"2023-09-23T15:30:26.693580+0000","last_unstale":"2023-09-23T15:30:28.716154+0000","last_undegraded":"2023-09-23T15:30:28.716154+0000","last_fullsized":"2023-09-23T15:30:28.716154+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.882635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.882635+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0050498640000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":161,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.688175+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:30:26.688175+0000","last_peered":"2023-09-23T15:30:26.688175+0000","last_clean":"2023-09-23T15:30:26.688175+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:30:26.688175+0000","last_undegraded":"2023-09-23T15:30:26.688175+0000","last_fullsized":"2023-09-23T15:30:26.688175+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680306+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:30:26.680306+0000","last_peered":"2023-09-23T15:30:26.680306+0000","last_clean":"2023-09-23T15:30:26.680306+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:30:26.680306+0000","last_undegraded":"2023-09-23T15:30:26.680306+0000","last_fullsized":"2023-09-23T15:30:26.680306+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"422'3620","reported_seq":3637,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.814671+0000","last_change":"2023-09-23T15:29:48.697177+0000","last_active":"2023-09-23T15:30:33.814671+0000","last_peered":"2023-09-23T15:30:33.814671+0000","last_clean":"2023-09-23T15:30:33.814671+0000","last_became_active":"2023-09-23T15:29:48.696842+0000","last_became_peered":"2023-09-23T15:29:48.696842+0000","last_unstale":"2023-09-23T15:30:33.814671+0000","last_undegraded":"2023-09-23T15:30:33.814671+0000","last_fullsized":"2023-09-23T15:30:33.814671+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3620,"log_dups_size":0,"ondisk_log_size":3620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:18.816926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237240320,"num_objects":3620,"num_object_clones":0,"num_object_copies":10860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3620,"num_write_kb":231680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":669,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689235+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:30:26.689235+0000","last_peered":"2023-09-23T15:30:26.689235+0000","last_clean":"2023-09-23T15:30:26.689235+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:30:26.689235+0000","last_undegraded":"2023-09-23T15:30:26.689235+0000","last_fullsized":"2023-09-23T15:30:26.689235+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":149,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687762+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:30:26.687762+0000","last_peered":"2023-09-23T15:30:26.687762+0000","last_clean":"2023-09-23T15:30:26.687762+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:30:26.687762+0000","last_undegraded":"2023-09-23T15:30:26.687762+0000","last_fullsized":"2023-09-23T15:30:26.687762+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":402,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.685513+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:30:26.685513+0000","last_peered":"2023-09-23T15:30:26.685513+0000","last_clean":"2023-09-23T15:30:26.685513+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:30:26.685513+0000","last_undegraded":"2023-09-23T15:30:26.685513+0000","last_fullsized":"2023-09-23T15:30:26.685513+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"422'3795","reported_seq":3812,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.375015+0000","last_change":"2023-09-23T15:29:48.698992+0000","last_active":"2023-09-23T15:30:34.375015+0000","last_peered":"2023-09-23T15:30:34.375015+0000","last_clean":"2023-09-23T15:30:34.375015+0000","last_became_active":"2023-09-23T15:29:48.698673+0000","last_became_peered":"2023-09-23T15:29:48.698673+0000","last_unstale":"2023-09-23T15:30:34.375015+0000","last_undegraded":"2023-09-23T15:30:34.375015+0000","last_fullsized":"2023-09-23T15:30:34.375015+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3795,"log_dups_size":0,"ondisk_log_size":3795,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:12.184760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248709120,"num_objects":3795,"num_object_clones":0,"num_object_copies":11385,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3795,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3795,"num_write_kb":242880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":639,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681058+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:30:26.681058+0000","last_peered":"2023-09-23T15:30:26.681058+0000","last_clean":"2023-09-23T15:30:26.681058+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:30:26.681058+0000","last_undegraded":"2023-09-23T15:30:26.681058+0000","last_fullsized":"2023-09-23T15:30:26.681058+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":397,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.684102+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:30:26.684102+0000","last_peered":"2023-09-23T15:30:26.684102+0000","last_clean":"2023-09-23T15:30:26.684102+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:30:26.684102+0000","last_undegraded":"2023-09-23T15:30:26.684102+0000","last_fullsized":"2023-09-23T15:30:26.684102+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":423,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680905+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:30:26.680905+0000","last_peered":"2023-09-23T15:30:26.680905+0000","last_clean":"2023-09-23T15:30:26.680905+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:30:26.680905+0000","last_undegraded":"2023-09-23T15:30:26.680905+0000","last_fullsized":"2023-09-23T15:30:26.680905+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"422'3501","reported_seq":3519,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:30.675108+0000","last_change":"2023-09-23T15:29:48.679906+0000","last_active":"2023-09-23T15:30:30.675108+0000","last_peered":"2023-09-23T15:30:30.675108+0000","last_clean":"2023-09-23T15:30:30.675108+0000","last_became_active":"2023-09-23T15:29:48.679584+0000","last_became_peered":"2023-09-23T15:29:48.679584+0000","last_unstale":"2023-09-23T15:30:30.675108+0000","last_undegraded":"2023-09-23T15:30:30.675108+0000","last_fullsized":"2023-09-23T15:30:30.675108+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3501,"log_dups_size":0,"ondisk_log_size":3501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:53.805715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229441536,"num_objects":3501,"num_object_clones":0,"num_object_copies":10503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3501,"num_write_kb":224064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":674,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687285+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:30:26.687285+0000","last_peered":"2023-09-23T15:30:26.687285+0000","last_clean":"2023-09-23T15:30:26.687285+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:30:26.687285+0000","last_undegraded":"2023-09-23T15:30:26.687285+0000","last_fullsized":"2023-09-23T15:30:26.687285+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":74,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.719449+0000","last_change":"2023-09-23T15:30:26.690336+0000","last_active":"2023-09-23T15:30:28.719449+0000","last_peered":"2023-09-23T15:30:28.719449+0000","last_clean":"2023-09-23T15:30:28.719449+0000","last_became_active":"2023-09-23T15:30:26.690141+0000","last_became_peered":"2023-09-23T15:30:26.690141+0000","last_unstale":"2023-09-23T15:30:28.719449+0000","last_undegraded":"2023-09-23T15:30:28.719449+0000","last_fullsized":"2023-09-23T15:30:28.719449+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:44.578933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680566+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:30:26.680566+0000","last_peered":"2023-09-23T15:30:26.680566+0000","last_clean":"2023-09-23T15:30:26.680566+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:30:26.680566+0000","last_undegraded":"2023-09-23T15:30:26.680566+0000","last_fullsized":"2023-09-23T15:30:26.680566+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"422'3764","reported_seq":3781,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.375846+0000","last_change":"2023-09-23T15:29:48.698981+0000","last_active":"2023-09-23T15:30:34.375846+0000","last_peered":"2023-09-23T15:30:34.375846+0000","last_clean":"2023-09-23T15:30:34.375846+0000","last_became_active":"2023-09-23T15:29:48.698707+0000","last_became_peered":"2023-09-23T15:29:48.698707+0000","last_unstale":"2023-09-23T15:30:34.375846+0000","last_undegraded":"2023-09-23T15:30:34.375846+0000","last_fullsized":"2023-09-23T15:30:34.375846+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3764,"log_dups_size":0,"ondisk_log_size":3764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:28.109594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246677504,"num_objects":3764,"num_object_clones":0,"num_object_copies":11292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3764,"num_write_kb":240896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":745,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683500+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:30:26.683500+0000","last_peered":"2023-09-23T15:30:26.683500+0000","last_clean":"2023-09-23T15:30:26.683500+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:30:26.683500+0000","last_undegraded":"2023-09-23T15:30:26.683500+0000","last_fullsized":"2023-09-23T15:30:26.683500+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":389,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.685604+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:30:26.685604+0000","last_peered":"2023-09-23T15:30:26.685604+0000","last_clean":"2023-09-23T15:30:26.685604+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:30:26.685604+0000","last_undegraded":"2023-09-23T15:30:26.685604+0000","last_fullsized":"2023-09-23T15:30:26.685604+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681009+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:30:26.681009+0000","last_peered":"2023-09-23T15:30:26.681009+0000","last_clean":"2023-09-23T15:30:26.681009+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:30:26.681009+0000","last_undegraded":"2023-09-23T15:30:26.681009+0000","last_fullsized":"2023-09-23T15:30:26.681009+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"422'3721","reported_seq":3738,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.534613+0000","last_change":"2023-09-23T15:29:48.683338+0000","last_active":"2023-09-23T15:30:34.534613+0000","last_peered":"2023-09-23T15:30:34.534613+0000","last_clean":"2023-09-23T15:30:34.534613+0000","last_became_active":"2023-09-23T15:29:48.682970+0000","last_became_peered":"2023-09-23T15:29:48.682970+0000","last_unstale":"2023-09-23T15:30:34.534613+0000","last_undegraded":"2023-09-23T15:30:34.534613+0000","last_fullsized":"2023-09-23T15:30:34.534613+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3721,"log_dups_size":0,"ondisk_log_size":3721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:20.273166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243859456,"num_objects":3721,"num_object_clones":0,"num_object_copies":11163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3721,"num_write_kb":238144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":702,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690144+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:30:26.690144+0000","last_peered":"2023-09-23T15:30:26.690144+0000","last_clean":"2023-09-23T15:30:26.690144+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:30:26.690144+0000","last_undegraded":"2023-09-23T15:30:26.690144+0000","last_fullsized":"2023-09-23T15:30:26.690144+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":78,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680346+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:30:26.680346+0000","last_peered":"2023-09-23T15:30:26.680346+0000","last_clean":"2023-09-23T15:30:26.680346+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:30:26.680346+0000","last_undegraded":"2023-09-23T15:30:26.680346+0000","last_fullsized":"2023-09-23T15:30:26.680346+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681148+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:30:26.681148+0000","last_peered":"2023-09-23T15:30:26.681148+0000","last_clean":"2023-09-23T15:30:26.681148+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:30:26.681148+0000","last_undegraded":"2023-09-23T15:30:26.681148+0000","last_fullsized":"2023-09-23T15:30:26.681148+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"422'3710","reported_seq":3727,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.815665+0000","last_change":"2023-09-23T15:29:48.680338+0000","last_active":"2023-09-23T15:30:33.815665+0000","last_peered":"2023-09-23T15:30:33.815665+0000","last_clean":"2023-09-23T15:30:33.815665+0000","last_became_active":"2023-09-23T15:29:48.679979+0000","last_became_peered":"2023-09-23T15:29:48.679979+0000","last_unstale":"2023-09-23T15:30:33.815665+0000","last_undegraded":"2023-09-23T15:30:33.815665+0000","last_fullsized":"2023-09-23T15:30:33.815665+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3710,"log_dups_size":0,"ondisk_log_size":3710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:54.650904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243138560,"num_objects":3710,"num_object_clones":0,"num_object_copies":11130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3710,"num_write_kb":237440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":757,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683736+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:30:26.683736+0000","last_peered":"2023-09-23T15:30:26.683736+0000","last_clean":"2023-09-23T15:30:26.683736+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:30:26.683736+0000","last_undegraded":"2023-09-23T15:30:26.683736+0000","last_fullsized":"2023-09-23T15:30:26.683736+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":388,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.705986+0000","last_change":"2023-09-23T15:30:26.705171+0000","last_active":"2023-09-23T15:30:26.705986+0000","last_peered":"2023-09-23T15:30:26.705986+0000","last_clean":"2023-09-23T15:30:26.705986+0000","last_became_active":"2023-09-23T15:30:26.704853+0000","last_became_peered":"2023-09-23T15:30:26.704853+0000","last_unstale":"2023-09-23T15:30:26.705986+0000","last_undegraded":"2023-09-23T15:30:26.705986+0000","last_fullsized":"2023-09-23T15:30:26.705986+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:11.875343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":356,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690236+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:30:26.690236+0000","last_peered":"2023-09-23T15:30:26.690236+0000","last_clean":"2023-09-23T15:30:26.690236+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:30:26.690236+0000","last_undegraded":"2023-09-23T15:30:26.690236+0000","last_fullsized":"2023-09-23T15:30:26.690236+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"422'3781","reported_seq":3798,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.407242+0000","last_change":"2023-09-23T15:29:48.682296+0000","last_active":"2023-09-23T15:30:34.407242+0000","last_peered":"2023-09-23T15:30:34.407242+0000","last_clean":"2023-09-23T15:30:34.407242+0000","last_became_active":"2023-09-23T15:29:48.681890+0000","last_became_peered":"2023-09-23T15:29:48.681890+0000","last_unstale":"2023-09-23T15:30:34.407242+0000","last_undegraded":"2023-09-23T15:30:34.407242+0000","last_fullsized":"2023-09-23T15:30:34.407242+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3781,"log_dups_size":0,"ondisk_log_size":3781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:34.635613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247791616,"num_objects":3781,"num_object_clones":0,"num_object_copies":11343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3781,"num_write_kb":241984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":662,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689316+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:30:26.689316+0000","last_peered":"2023-09-23T15:30:26.689316+0000","last_clean":"2023-09-23T15:30:26.689316+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:30:26.689316+0000","last_undegraded":"2023-09-23T15:30:26.689316+0000","last_fullsized":"2023-09-23T15:30:26.689316+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":639,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690367+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:30:26.690367+0000","last_peered":"2023-09-23T15:30:26.690367+0000","last_clean":"2023-09-23T15:30:26.690367+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:30:26.690367+0000","last_undegraded":"2023-09-23T15:30:26.690367+0000","last_fullsized":"2023-09-23T15:30:26.690367+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":397,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683915+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:30:26.683915+0000","last_peered":"2023-09-23T15:30:26.683915+0000","last_clean":"2023-09-23T15:30:26.683915+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:30:26.683915+0000","last_undegraded":"2023-09-23T15:30:26.683915+0000","last_fullsized":"2023-09-23T15:30:26.683915+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":385,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682273+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:30:26.682273+0000","last_peered":"2023-09-23T15:30:26.682273+0000","last_clean":"2023-09-23T15:30:26.682273+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:30:26.682273+0000","last_undegraded":"2023-09-23T15:30:26.682273+0000","last_fullsized":"2023-09-23T15:30:26.682273+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"422'3821","reported_seq":3838,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.536658+0000","last_change":"2023-09-23T15:29:48.686875+0000","last_active":"2023-09-23T15:30:34.536658+0000","last_peered":"2023-09-23T15:30:34.536658+0000","last_clean":"2023-09-23T15:30:34.536658+0000","last_became_active":"2023-09-23T15:29:48.686520+0000","last_became_peered":"2023-09-23T15:29:48.686520+0000","last_unstale":"2023-09-23T15:30:34.536658+0000","last_undegraded":"2023-09-23T15:30:34.536658+0000","last_fullsized":"2023-09-23T15:30:34.536658+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3821,"log_dups_size":0,"ondisk_log_size":3821,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:52.851173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3821,"num_write_kb":244544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":718,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683511+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:30:26.683511+0000","last_peered":"2023-09-23T15:30:26.683511+0000","last_clean":"2023-09-23T15:30:26.683511+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:30:26.683511+0000","last_undegraded":"2023-09-23T15:30:26.683511+0000","last_fullsized":"2023-09-23T15:30:26.683511+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":956,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.684295+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:30:26.684295+0000","last_peered":"2023-09-23T15:30:26.684295+0000","last_clean":"2023-09-23T15:30:26.684295+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:30:26.684295+0000","last_undegraded":"2023-09-23T15:30:26.684295+0000","last_fullsized":"2023-09-23T15:30:26.684295+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682396+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:30:26.682396+0000","last_peered":"2023-09-23T15:30:26.682396+0000","last_clean":"2023-09-23T15:30:26.682396+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:30:26.682396+0000","last_undegraded":"2023-09-23T15:30:26.682396+0000","last_fullsized":"2023-09-23T15:30:26.682396+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":321,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690020+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:30:26.690020+0000","last_peered":"2023-09-23T15:30:26.690020+0000","last_clean":"2023-09-23T15:30:26.690020+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:30:26.690020+0000","last_undegraded":"2023-09-23T15:30:26.690020+0000","last_fullsized":"2023-09-23T15:30:26.690020+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"422'3822","reported_seq":3838,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.794825+0000","last_change":"2023-09-23T15:29:48.686854+0000","last_active":"2023-09-23T15:30:33.794825+0000","last_peered":"2023-09-23T15:30:33.794825+0000","last_clean":"2023-09-23T15:30:33.794825+0000","last_became_active":"2023-09-23T15:29:48.686539+0000","last_became_peered":"2023-09-23T15:29:48.686539+0000","last_unstale":"2023-09-23T15:30:33.794825+0000","last_undegraded":"2023-09-23T15:30:33.794825+0000","last_fullsized":"2023-09-23T15:30:33.794825+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3822,"log_dups_size":0,"ondisk_log_size":3822,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:33.671531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250478592,"num_objects":3822,"num_object_clones":0,"num_object_copies":11466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3822,"num_write_kb":244608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":785,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687285+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:30:26.687285+0000","last_peered":"2023-09-23T15:30:26.687285+0000","last_clean":"2023-09-23T15:30:26.687285+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:30:26.687285+0000","last_undegraded":"2023-09-23T15:30:26.687285+0000","last_fullsized":"2023-09-23T15:30:26.687285+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1040,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682776+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:30:26.682776+0000","last_peered":"2023-09-23T15:30:26.682776+0000","last_clean":"2023-09-23T15:30:26.682776+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:30:26.682776+0000","last_undegraded":"2023-09-23T15:30:26.682776+0000","last_fullsized":"2023-09-23T15:30:26.682776+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":316,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681558+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:30:26.681558+0000","last_peered":"2023-09-23T15:30:26.681558+0000","last_clean":"2023-09-23T15:30:26.681558+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:30:26.681558+0000","last_undegraded":"2023-09-23T15:30:26.681558+0000","last_fullsized":"2023-09-23T15:30:26.681558+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":325,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690162+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:30:26.690162+0000","last_peered":"2023-09-23T15:30:26.690162+0000","last_clean":"2023-09-23T15:30:26.690162+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:30:26.690162+0000","last_undegraded":"2023-09-23T15:30:26.690162+0000","last_fullsized":"2023-09-23T15:30:26.690162+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"422'3765","reported_seq":3780,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.563501+0000","last_change":"2023-09-23T15:29:48.684663+0000","last_active":"2023-09-23T15:30:34.563501+0000","last_peered":"2023-09-23T15:30:34.563501+0000","last_clean":"2023-09-23T15:30:34.563501+0000","last_became_active":"2023-09-23T15:29:48.684293+0000","last_became_peered":"2023-09-23T15:29:48.684293+0000","last_unstale":"2023-09-23T15:30:34.563501+0000","last_undegraded":"2023-09-23T15:30:34.563501+0000","last_fullsized":"2023-09-23T15:30:34.563501+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3765,"log_dups_size":0,"ondisk_log_size":3765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:18.182468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246743040,"num_objects":3765,"num_object_clones":0,"num_object_copies":11295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3765,"num_write_kb":240960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":767,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683892+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:30:26.683892+0000","last_peered":"2023-09-23T15:30:26.683892+0000","last_clean":"2023-09-23T15:30:26.683892+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:30:26.683892+0000","last_undegraded":"2023-09-23T15:30:26.683892+0000","last_fullsized":"2023-09-23T15:30:26.683892+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":728,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.695050+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:30:26.695050+0000","last_peered":"2023-09-23T15:30:26.695050+0000","last_clean":"2023-09-23T15:30:26.695050+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:30:26.695050+0000","last_undegraded":"2023-09-23T15:30:26.695050+0000","last_fullsized":"2023-09-23T15:30:26.695050+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":291,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.694668+0000","last_change":"2023-09-23T15:30:26.694100+0000","last_active":"2023-09-23T15:30:26.694668+0000","last_peered":"2023-09-23T15:30:26.694668+0000","last_clean":"2023-09-23T15:30:26.694668+0000","last_became_active":"2023-09-23T15:30:26.693767+0000","last_became_peered":"2023-09-23T15:30:26.693767+0000","last_unstale":"2023-09-23T15:30:26.694668+0000","last_undegraded":"2023-09-23T15:30:26.694668+0000","last_fullsized":"2023-09-23T15:30:26.694668+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.703270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":414,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.685958+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:30:26.685958+0000","last_peered":"2023-09-23T15:30:26.685958+0000","last_clean":"2023-09-23T15:30:26.685958+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:30:26.685958+0000","last_undegraded":"2023-09-23T15:30:26.685958+0000","last_fullsized":"2023-09-23T15:30:26.685958+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"422'3621","reported_seq":3638,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.824050+0000","last_change":"2023-09-23T15:29:48.679894+0000","last_active":"2023-09-23T15:30:33.824050+0000","last_peered":"2023-09-23T15:30:33.824050+0000","last_clean":"2023-09-23T15:30:33.824050+0000","last_became_active":"2023-09-23T15:29:48.679648+0000","last_became_peered":"2023-09-23T15:29:48.679648+0000","last_unstale":"2023-09-23T15:30:33.824050+0000","last_undegraded":"2023-09-23T15:30:33.824050+0000","last_fullsized":"2023-09-23T15:30:33.824050+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3621,"log_dups_size":0,"ondisk_log_size":3621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:23.079554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3621,"num_write_kb":231744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":765,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690056+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:30:26.690056+0000","last_peered":"2023-09-23T15:30:26.690056+0000","last_clean":"2023-09-23T15:30:26.690056+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:30:26.690056+0000","last_undegraded":"2023-09-23T15:30:26.690056+0000","last_fullsized":"2023-09-23T15:30:26.690056+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":135,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.694903+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:30:26.694903+0000","last_peered":"2023-09-23T15:30:26.694903+0000","last_clean":"2023-09-23T15:30:26.694903+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:30:26.694903+0000","last_undegraded":"2023-09-23T15:30:26.694903+0000","last_fullsized":"2023-09-23T15:30:26.694903+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":409,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680503+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:30:26.680503+0000","last_peered":"2023-09-23T15:30:26.680503+0000","last_clean":"2023-09-23T15:30:26.680503+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:30:26.680503+0000","last_undegraded":"2023-09-23T15:30:26.680503+0000","last_fullsized":"2023-09-23T15:30:26.680503+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":402,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680072+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:30:26.680072+0000","last_peered":"2023-09-23T15:30:26.680072+0000","last_clean":"2023-09-23T15:30:26.680072+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:30:26.680072+0000","last_undegraded":"2023-09-23T15:30:26.680072+0000","last_fullsized":"2023-09-23T15:30:26.680072+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"422'3838","reported_seq":3855,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.542188+0000","last_change":"2023-09-23T15:29:48.683852+0000","last_active":"2023-09-23T15:30:34.542188+0000","last_peered":"2023-09-23T15:30:34.542188+0000","last_clean":"2023-09-23T15:30:34.542188+0000","last_became_active":"2023-09-23T15:29:48.683560+0000","last_became_peered":"2023-09-23T15:29:48.683560+0000","last_unstale":"2023-09-23T15:30:34.542188+0000","last_undegraded":"2023-09-23T15:30:34.542188+0000","last_fullsized":"2023-09-23T15:30:34.542188+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3838,"log_dups_size":0,"ondisk_log_size":3838,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:38.866091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251527168,"num_objects":3838,"num_object_clones":0,"num_object_copies":11514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3838,"num_write_kb":245632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":748,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.685632+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:30:26.685632+0000","last_peered":"2023-09-23T15:30:26.685632+0000","last_clean":"2023-09-23T15:30:26.685632+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:30:26.685632+0000","last_undegraded":"2023-09-23T15:30:26.685632+0000","last_fullsized":"2023-09-23T15:30:26.685632+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1161,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.687835+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:30:26.687835+0000","last_peered":"2023-09-23T15:30:26.687835+0000","last_clean":"2023-09-23T15:30:26.687835+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:30:26.687835+0000","last_undegraded":"2023-09-23T15:30:26.687835+0000","last_fullsized":"2023-09-23T15:30:26.687835+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":301,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690214+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:30:26.690214+0000","last_peered":"2023-09-23T15:30:26.690214+0000","last_clean":"2023-09-23T15:30:26.690214+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:30:26.690214+0000","last_undegraded":"2023-09-23T15:30:26.690214+0000","last_fullsized":"2023-09-23T15:30:26.690214+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":402,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681367+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:30:26.681367+0000","last_peered":"2023-09-23T15:30:26.681367+0000","last_clean":"2023-09-23T15:30:26.681367+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:30:26.681367+0000","last_undegraded":"2023-09-23T15:30:26.681367+0000","last_fullsized":"2023-09-23T15:30:26.681367+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"422'3669","reported_seq":3686,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:33.801494+0000","last_change":"2023-09-23T15:29:48.685734+0000","last_active":"2023-09-23T15:30:33.801494+0000","last_peered":"2023-09-23T15:30:33.801494+0000","last_clean":"2023-09-23T15:30:33.801494+0000","last_became_active":"2023-09-23T15:29:48.684906+0000","last_became_peered":"2023-09-23T15:29:48.684906+0000","last_unstale":"2023-09-23T15:30:33.801494+0000","last_undegraded":"2023-09-23T15:30:33.801494+0000","last_fullsized":"2023-09-23T15:30:33.801494+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3669,"log_dups_size":0,"ondisk_log_size":3669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:06.890491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240451584,"num_objects":3669,"num_object_clones":0,"num_object_copies":11007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3669,"num_write_kb":234816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":713,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681170+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:30:26.681170+0000","last_peered":"2023-09-23T15:30:26.681170+0000","last_clean":"2023-09-23T15:30:26.681170+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:30:26.681170+0000","last_undegraded":"2023-09-23T15:30:26.681170+0000","last_fullsized":"2023-09-23T15:30:26.681170+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":194,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682815+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:30:26.682815+0000","last_peered":"2023-09-23T15:30:26.682815+0000","last_clean":"2023-09-23T15:30:26.682815+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:30:26.682815+0000","last_undegraded":"2023-09-23T15:30:26.682815+0000","last_fullsized":"2023-09-23T15:30:26.682815+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1056,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689888+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:30:26.689888+0000","last_peered":"2023-09-23T15:30:26.689888+0000","last_clean":"2023-09-23T15:30:26.689888+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:30:26.689888+0000","last_undegraded":"2023-09-23T15:30:26.689888+0000","last_fullsized":"2023-09-23T15:30:26.689888+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":369,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.712251+0000","last_change":"2023-09-23T15:30:26.710395+0000","last_active":"2023-09-23T15:30:26.712251+0000","last_peered":"2023-09-23T15:30:26.712251+0000","last_clean":"2023-09-23T15:30:26.712251+0000","last_became_active":"2023-09-23T15:30:26.710076+0000","last_became_peered":"2023-09-23T15:30:26.710076+0000","last_unstale":"2023-09-23T15:30:26.712251+0000","last_undegraded":"2023-09-23T15:30:26.712251+0000","last_fullsized":"2023-09-23T15:30:26.712251+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.313559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"422'3810","reported_seq":3827,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:34.545788+0000","last_change":"2023-09-23T15:29:48.697095+0000","last_active":"2023-09-23T15:30:34.545788+0000","last_peered":"2023-09-23T15:30:34.545788+0000","last_clean":"2023-09-23T15:30:34.545788+0000","last_became_active":"2023-09-23T15:29:48.696636+0000","last_became_peered":"2023-09-23T15:29:48.696636+0000","last_unstale":"2023-09-23T15:30:34.545788+0000","last_undegraded":"2023-09-23T15:30:34.545788+0000","last_fullsized":"2023-09-23T15:30:34.545788+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":3810,"log_dups_size":0,"ondisk_log_size":3810,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:52.284501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249692160,"num_objects":3810,"num_object_clones":0,"num_object_copies":11430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3810,"num_write_kb":243840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":562,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.697035+0000","last_change":"2023-09-23T15:30:26.695170+0000","last_active":"2023-09-23T15:30:26.697035+0000","last_peered":"2023-09-23T15:30:26.697035+0000","last_clean":"2023-09-23T15:30:26.697035+0000","last_became_active":"2023-09-23T15:30:26.694799+0000","last_became_peered":"2023-09-23T15:30:26.694799+0000","last_unstale":"2023-09-23T15:30:26.697035+0000","last_undegraded":"2023-09-23T15:30:26.697035+0000","last_fullsized":"2023-09-23T15:30:26.697035+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:29.288370+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":875,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689606+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:30:26.689606+0000","last_peered":"2023-09-23T15:30:26.689606+0000","last_clean":"2023-09-23T15:30:26.689606+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:30:26.689606+0000","last_undegraded":"2023-09-23T15:30:26.689606+0000","last_fullsized":"2023-09-23T15:30:26.689606+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":461,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680158+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:30:26.680158+0000","last_peered":"2023-09-23T15:30:26.680158+0000","last_clean":"2023-09-23T15:30:26.680158+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:30:26.680158+0000","last_undegraded":"2023-09-23T15:30:26.680158+0000","last_fullsized":"2023-09-23T15:30:26.680158+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680389+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:30:26.680389+0000","last_peered":"2023-09-23T15:30:26.680389+0000","last_clean":"2023-09-23T15:30:26.680389+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:30:26.680389+0000","last_undegraded":"2023-09-23T15:30:26.680389+0000","last_fullsized":"2023-09-23T15:30:26.680389+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":631,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.715915+0000","last_change":"2023-09-23T15:30:26.740237+0000","last_active":"2023-09-23T15:30:28.715915+0000","last_peered":"2023-09-23T15:30:28.715915+0000","last_clean":"2023-09-23T15:30:28.715915+0000","last_became_active":"2023-09-23T15:30:26.690718+0000","last_became_peered":"2023-09-23T15:30:26.690718+0000","last_unstale":"2023-09-23T15:30:28.715915+0000","last_undegraded":"2023-09-23T15:30:28.715915+0000","last_fullsized":"2023-09-23T15:30:28.715915+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:00.439552+0000","scrub_duration":0.013441308000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":441,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681316+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:30:26.681316+0000","last_peered":"2023-09-23T15:30:26.681316+0000","last_clean":"2023-09-23T15:30:26.681316+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:30:26.681316+0000","last_undegraded":"2023-09-23T15:30:26.681316+0000","last_fullsized":"2023-09-23T15:30:26.681316+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":426,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682602+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:30:26.682602+0000","last_peered":"2023-09-23T15:30:26.682602+0000","last_clean":"2023-09-23T15:30:26.682602+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:30:26.682602+0000","last_undegraded":"2023-09-23T15:30:26.682602+0000","last_fullsized":"2023-09-23T15:30:26.682602+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":599,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.373260+0000","last_change":"2023-09-23T15:30:28.373196+0000","last_active":"2023-09-23T15:30:28.373260+0000","last_peered":"2023-09-23T15:30:28.373260+0000","last_clean":"2023-09-23T15:30:28.373260+0000","last_became_active":"2023-09-23T15:30:26.699434+0000","last_became_peered":"2023-09-23T15:30:26.699434+0000","last_unstale":"2023-09-23T15:30:28.373260+0000","last_undegraded":"2023-09-23T15:30:28.373260+0000","last_fullsized":"2023-09-23T15:30:28.373260+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:26.970670+0000","scrub_duration":0.0018471900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":462,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.684239+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:30:26.684239+0000","last_peered":"2023-09-23T15:30:26.684239+0000","last_clean":"2023-09-23T15:30:26.684239+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:30:26.684239+0000","last_undegraded":"2023-09-23T15:30:26.684239+0000","last_fullsized":"2023-09-23T15:30:26.684239+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.682529+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:30:26.682529+0000","last_peered":"2023-09-23T15:30:26.682529+0000","last_clean":"2023-09-23T15:30:26.682529+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:30:26.682529+0000","last_undegraded":"2023-09-23T15:30:26.682529+0000","last_fullsized":"2023-09-23T15:30:26.682529+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":442,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.683821+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:30:26.683821+0000","last_peered":"2023-09-23T15:30:26.683821+0000","last_clean":"2023-09-23T15:30:26.683821+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:30:26.683821+0000","last_undegraded":"2023-09-23T15:30:26.683821+0000","last_fullsized":"2023-09-23T15:30:26.683821+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.681140+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:30:26.681140+0000","last_peered":"2023-09-23T15:30:26.681140+0000","last_clean":"2023-09-23T15:30:26.681140+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:30:26.681140+0000","last_undegraded":"2023-09-23T15:30:26.681140+0000","last_fullsized":"2023-09-23T15:30:26.681140+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":609,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.689495+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:30:26.689495+0000","last_peered":"2023-09-23T15:30:26.689495+0000","last_clean":"2023-09-23T15:30:26.689495+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:30:26.689495+0000","last_undegraded":"2023-09-23T15:30:26.689495+0000","last_fullsized":"2023-09-23T15:30:26.689495+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":54,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.699939+0000","last_change":"2023-09-23T15:30:26.699597+0000","last_active":"2023-09-23T15:30:26.699939+0000","last_peered":"2023-09-23T15:30:26.699939+0000","last_clean":"2023-09-23T15:30:26.699939+0000","last_became_active":"2023-09-23T15:30:26.699233+0000","last_became_peered":"2023-09-23T15:30:26.699233+0000","last_unstale":"2023-09-23T15:30:26.699939+0000","last_undegraded":"2023-09-23T15:30:26.699939+0000","last_fullsized":"2023-09-23T15:30:26.699939+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.804745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":346,"reported_epoch":422,"state":"active+clean","last_fresh":"2023-09-23T15:30:28.718981+0000","last_change":"2023-09-23T15:30:26.685257+0000","last_active":"2023-09-23T15:30:28.718981+0000","last_peered":"2023-09-23T15:30:28.718981+0000","last_clean":"2023-09-23T15:30:28.718981+0000","last_became_active":"2023-09-23T15:30:26.684954+0000","last_became_peered":"2023-09-23T15:30:26.684954+0000","last_unstale":"2023-09-23T15:30:28.718981+0000","last_undegraded":"2023-09-23T15:30:28.718981+0000","last_fullsized":"2023-09-23T15:30:28.718981+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:49.244950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":602,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.690869+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:30:26.690869+0000","last_peered":"2023-09-23T15:30:26.690869+0000","last_clean":"2023-09-23T15:30:26.690869+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:30:26.690869+0000","last_undegraded":"2023-09-23T15:30:26.690869+0000","last_fullsized":"2023-09-23T15:30:26.690869+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":376,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680249+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:30:26.680249+0000","last_peered":"2023-09-23T15:30:26.680249+0000","last_clean":"2023-09-23T15:30:26.680249+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:30:26.680249+0000","last_undegraded":"2023-09-23T15:30:26.680249+0000","last_fullsized":"2023-09-23T15:30:26.680249+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":64,"reported_epoch":421,"state":"active+clean","last_fresh":"2023-09-23T15:30:26.680147+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:30:26.680147+0000","last_peered":"2023-09-23T15:30:26.680147+0000","last_clean":"2023-09-23T15:30:26.680147+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:30:26.680147+0000","last_undegraded":"2023-09-23T15:30:26.680147+0000","last_fullsized":"2023-09-23T15:30:26.680147+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3908829184,"num_objects":59644,"num_object_clones":0,"num_object_copies":178932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":59644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":59644,"num_write_kb":3817216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11718295552,"data_stored":11718295552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":59644,"ondisk_log_size":59644,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":426528,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768740,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1075016,"kb_used_data":963072,"kb_used_omap":13,"kb_used_meta":111922,"kb_avail":92670136,"statfs":{"total":95995035648,"available":94894219264,"internally_reserved":0,"allocated":986185728,"data_stored":984818085,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":114608222},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":5.5919999999999996,"5min":5.7519999999999998,"15min":5.7779999999999996},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":80,"5min":80,"15min":80},"last":0.255},{"interface":"front","average":{"1min":5.4880000000000004,"5min":5.8239999999999998,"15min":5.8799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":80.117000000000004,"5min":80.117000000000004,"15min":80.117000000000004},"last":0.39400000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":6.2830000000000004,"5min":18.283000000000001,"15min":20.283000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":78.632000000000005,"5min":157.096,"15min":157.096},"last":0.28100000000000003},{"interface":"front","average":{"1min":5.3289999999999997,"5min":18.210999999999999,"15min":20.358000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":76.489000000000004,"5min":157.011,"15min":157.011},"last":0.42999999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2759999999999998,"5min":1.0649999999999999,"15min":0.86299999999999999},"min":{"1min":0.33500000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":8.8460000000000001,"5min":8.8460000000000001,"15min":8.8460000000000001},"last":0.441},{"interface":"front","average":{"1min":1.1439999999999999,"5min":0.996,"15min":0.97099999999999997},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":3.6930000000000001,"5min":4.109,"15min":4.109},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":5.6980000000000004,"5min":4.3959999999999999,"15min":4.1790000000000003},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":76.019000000000005,"5min":76.019000000000005,"15min":76.019000000000005},"last":0.48199999999999998},{"interface":"front","average":{"1min":6.7709999999999999,"5min":4.9390000000000001,"15min":4.633},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":76.317999999999998,"5min":76.317999999999998,"15min":76.317999999999998},"last":0.34999999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":0.65800000000000003,"15min":0.59799999999999998},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.3710000000000004,"5min":4.3710000000000004,"15min":4.3710000000000004},"last":0.98899999999999999},{"interface":"front","average":{"1min":1.3740000000000001,"5min":0.67000000000000004,"15min":0.55300000000000005},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":8.8089999999999993,"5min":8.8089999999999993,"15min":8.8089999999999993},"last":0.89700000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":0.79100000000000004,"15min":0.80200000000000005},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":2.1840000000000002,"5min":3.4279999999999999,"15min":3.4279999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.1339999999999999,"5min":0.80800000000000005,"15min":0.754},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.3369999999999997,"5min":4.3369999999999997,"15min":4.3369999999999997},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.75600000000000001,"15min":0.73699999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":4.5119999999999996,"5min":4.5119999999999996,"15min":4.5119999999999996},"last":0.628},{"interface":"front","average":{"1min":1.6839999999999999,"5min":0.85499999999999998,"15min":0.71699999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.2810000000000006,"5min":9.2810000000000006,"15min":9.2810000000000006},"last":0.56000000000000005}]}]},{"osd":6,"up_from":304,"seq":1305670058083,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1835388,"kb_used_data":1683444,"kb_used_omap":13,"kb_used_meta":151922,"kb_avail":91909764,"statfs":{"total":95995035648,"available":94115598336,"internally_reserved":0,"allocated":1723846656,"data_stored":1722490630,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":155568483},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0230000000000001,"5min":1.879,"15min":2.593},"min":{"1min":0.246,"5min":0.224,"15min":0.224},"max":{"1min":10.827,"5min":46.740000000000002,"15min":184.53200000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.3240000000000001,"5min":1.865,"15min":2.9169999999999998},"min":{"1min":0.26700000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.2460000000000004,"5min":46.549999999999997,"15min":86.227999999999994},"last":4.2549999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.5669999999999999,"5min":1.6459999999999999,"15min":2.8090000000000002},"min":{"1min":0.28299999999999997,"5min":0.221,"15min":0.221},"max":{"1min":9.9710000000000001,"5min":9.9710000000000001,"15min":86.534000000000006},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.0629999999999999,"5min":1.351,"15min":2.7349999999999999},"min":{"1min":0.34200000000000003,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":3.8100000000000001,"5min":6.8710000000000004,"15min":86.144000000000005},"last":0.45000000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.732,"5min":1.7949999999999999,"15min":1.694},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.1980000000000004,"5min":9.2379999999999995,"15min":9.8469999999999995},"last":4.2229999999999999},{"interface":"front","average":{"1min":1.224,"5min":1.603,"15min":1.7729999999999999},"min":{"1min":0.39100000000000001,"5min":0.35199999999999998,"15min":0.307},"max":{"1min":4.5709999999999997,"5min":9.3650000000000002,"15min":9.3650000000000002},"last":0.89700000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6259999999999999,"5min":3.1800000000000002,"15min":2.7229999999999999},"min":{"1min":0.33400000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":7.9569999999999999,"5min":142.67599999999999,"15min":142.67599999999999},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.7410000000000001,"5min":3.6509999999999998,"15min":2.923},"min":{"1min":0.41299999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":18.149000000000001,"5min":142.541,"15min":142.541},"last":4.1740000000000004}]},{"osd":4,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68799999999999994,"5min":0.78000000000000003,"15min":0.96699999999999997},"min":{"1min":0.28899999999999998,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.8460000000000001,"5min":6.6479999999999997,"15min":8.1219999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.272,"5min":1.095,"15min":1.641},"min":{"1min":0.318,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.569,"5min":5.569,"15min":13.627000000000001},"last":2.645}]},{"osd":5,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.7839999999999998,"15min":2.258},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":4.2030000000000003,"5min":130.91300000000001,"15min":130.91300000000001},"last":3.6600000000000001},{"interface":"front","average":{"1min":1.26,"5min":2.6160000000000001,"15min":1.9750000000000001},"min":{"1min":0.318,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.6470000000000002,"5min":130.92599999999999,"15min":130.92599999999999},"last":0.52200000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0369999999999999,"5min":1.196,"15min":1.222},"min":{"1min":0.35199999999999998,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.7009999999999996,"5min":9.3209999999999997,"15min":9.3209999999999997},"last":2.9830000000000001},{"interface":"front","average":{"1min":1.3180000000000001,"5min":1.4430000000000001,"15min":1.464},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.492,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.67200000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015383,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303580,"kb_used_data":677652,"kb_used_omap":6,"kb_used_meta":625913,"kb_avail":92441572,"statfs":{"total":95995035648,"available":94660169728,"internally_reserved":0,"allocated":693915648,"data_stored":691794134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":640934987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:30:24 2023","interfaces":[{"interface":"back","average":{"1min":5.8140000000000001,"5min":3.2410000000000001,"15min":3.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":101.27800000000001,"5min":101.27800000000001,"15min":101.27800000000001},"last":0.432},{"interface":"front","average":{"1min":6.4039999999999999,"5min":3.1200000000000001,"15min":3.673},"min":{"1min":0.25800000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":101.051,"5min":101.051,"15min":101.051},"last":2.9580000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:30:24 2023","interfaces":[{"interface":"back","average":{"1min":10.122,"5min":4.2080000000000002,"15min":4.0540000000000003},"min":{"1min":0.28599999999999998,"5min":0.221,"15min":0.221},"max":{"1min":186.304,"5min":186.304,"15min":186.304},"last":1.921},{"interface":"front","average":{"1min":9.7560000000000002,"5min":4.0700000000000003,"15min":3.3969999999999998},"min":{"1min":0.32300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":186.541,"5min":186.541,"15min":186.541},"last":8.2249999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.93999999999999995,"5min":1.931,"15min":4.5919999999999996},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.3120000000000003,"5min":15.191000000000001,"15min":211.405},"last":8.2669999999999995},{"interface":"front","average":{"1min":1.3260000000000001,"5min":2.0310000000000001,"15min":4.782},"min":{"1min":0.28599999999999998,"5min":0.26400000000000001,"15min":0.246},"max":{"1min":7.8940000000000001,"5min":14.609999999999999,"15min":210.988},"last":4.3019999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:30:24 2023","interfaces":[{"interface":"back","average":{"1min":1.1399999999999999,"5min":2.9159999999999999,"15min":3.4569999999999999},"min":{"1min":0.35899999999999999,"5min":0.34399999999999997,"15min":0.29799999999999999},"max":{"1min":4.0439999999999996,"5min":72.602000000000004,"15min":80.570999999999998},"last":4.8419999999999996},{"interface":"front","average":{"1min":0.83699999999999997,"5min":2.1819999999999999,"15min":2.9769999999999999},"min":{"1min":0.32500000000000001,"5min":0.312,"15min":0.308},"max":{"1min":2.516,"5min":73.454999999999998,"15min":80.75},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:30:24 2023","interfaces":[{"interface":"back","average":{"1min":0.97299999999999998,"5min":3.5169999999999999,"15min":3.4710000000000001},"min":{"1min":0.40999999999999998,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":3.0289999999999999,"5min":173.18799999999999,"15min":173.18799999999999},"last":6.4539999999999997},{"interface":"front","average":{"1min":1.2090000000000001,"5min":3.9550000000000001,"15min":3.7789999999999999},"min":{"1min":0.40100000000000002,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.2169999999999996,"5min":172.822,"15min":172.822},"last":4.532}]},{"osd":6,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":2.7250000000000001,"15min":3.044},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.0490000000000004,"5min":54.198,"15min":54.198},"last":4.6669999999999998},{"interface":"front","average":{"1min":1.619,"5min":1.8939999999999999,"15min":2.1800000000000002},"min":{"1min":0.42699999999999999,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":8.5800000000000001,"5min":15.605,"15min":15.605},"last":3.052}]},{"osd":7,"last update":"Sat Sep 23 15:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.9410000000000001,"5min":2.0990000000000002,"15min":2.125},"min":{"1min":0.46800000000000003,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":8.6120000000000001,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":3.0219999999999998},{"interface":"front","average":{"1min":2.3279999999999998,"5min":4.524,"15min":4.8899999999999997},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":7.7709999999999999,"5min":54.223999999999997,"15min":54.223999999999997},"last":4.7450000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893430,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3157244,"kb_used_data":2428020,"kb_used_omap":8,"kb_used_meta":729207,"kb_avail":90587908,"statfs":{"total":95995035648,"available":92762017792,"internally_reserved":0,"allocated":2486292480,"data_stored":2484316279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":746708564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:30:15 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":5.8559999999999999,"15min":4.5099999999999998},"min":{"1min":0.23000000000000001,"5min":0.216,"15min":0.18099999999999999},"max":{"1min":15.205,"5min":141.52199999999999,"15min":141.52199999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.95399999999999996,"5min":6.0190000000000001,"15min":4.2629999999999999},"min":{"1min":0.28699999999999998,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":3.6520000000000001,"5min":141.62799999999999,"15min":141.62799999999999},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":2.4870000000000001,"5min":2.0710000000000002,"15min":3.2650000000000001},"min":{"1min":0.23699999999999999,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":10.75,"5min":14.994,"15min":93.873999999999995},"last":1.552},{"interface":"front","average":{"1min":1.484,"5min":1.484,"15min":3.2589999999999999},"min":{"1min":0.27300000000000002,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":6.5300000000000002,"5min":10.874000000000001,"15min":94.652000000000001},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 15:30:02 2023","interfaces":[{"interface":"back","average":{"1min":3.988,"5min":5.484,"15min":5.8620000000000001},"min":{"1min":0.313,"5min":0.26500000000000001,"15min":0.25900000000000001},"max":{"1min":41.640000000000001,"5min":162.154,"15min":197.78899999999999},"last":4.4500000000000002},{"interface":"front","average":{"1min":4.234,"5min":5.016,"15min":6.5},"min":{"1min":0.35499999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":41.319000000000003,"5min":162.215,"15min":200.58799999999999},"last":2.4100000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.8640000000000001,"15min":1.895},"min":{"1min":0.36399999999999999,"5min":0.33200000000000002,"15min":0.28000000000000003},"max":{"1min":10.832000000000001,"5min":11.887,"15min":15.358000000000001},"last":3.472},{"interface":"front","average":{"1min":2.4359999999999999,"5min":2.258,"15min":2.0070000000000001},"min":{"1min":0.371,"5min":0.314,"15min":0.29399999999999998},"max":{"1min":9.7769999999999992,"5min":13.656000000000001,"15min":13.656000000000001},"last":4.5629999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":2.1539999999999999,"15min":2.1539999999999999},"min":{"1min":0.33600000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":8.5039999999999996,"5min":44.57,"15min":64.349999999999994},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.0659999999999998,"5min":2.5920000000000001,"15min":2.2149999999999999},"min":{"1min":0.42999999999999999,"5min":0.34399999999999997,"15min":0.312},"max":{"1min":10.797000000000001,"5min":44.588000000000001,"15min":44.588000000000001},"last":0.42999999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:29:39 2023","interfaces":[{"interface":"back","average":{"1min":1.954,"5min":2.3180000000000001,"15min":2.6720000000000002},"min":{"1min":0.44400000000000001,"5min":0.379,"15min":0.379},"max":{"1min":11.615,"5min":32.097999999999999,"15min":32.097999999999999},"last":1.3979999999999999},{"interface":"front","average":{"1min":2.1909999999999998,"5min":2.6720000000000002,"15min":3.1379999999999999},"min":{"1min":0.45000000000000001,"5min":0.371,"15min":0.371},"max":{"1min":9.7409999999999997,"5min":32.188000000000002,"15min":32.188000000000002},"last":4.5149999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":2.0590000000000002,"15min":2.2160000000000002},"min":{"1min":0.45100000000000001,"5min":0.318,"15min":0.318},"max":{"1min":4.0899999999999999,"5min":10.537000000000001,"15min":10.537000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.119,"5min":3.5459999999999998,"15min":3.9500000000000002},"min":{"1min":0.44800000000000001,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":3.948,"5min":12.651,"15min":12.651},"last":4.6289999999999996}]}]},{"osd":2,"up_from":105,"seq":450971566380,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2131564,"kb_used_data":1437540,"kb_used_omap":11,"kb_used_meta":694004,"kb_avail":91613588,"statfs":{"total":95995035648,"available":93812314112,"internally_reserved":0,"allocated":1472040960,"data_stored":1470077144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":710660880},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":4.9589999999999996,"15min":5.2699999999999996},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.17199999999999999},"max":{"1min":6.6420000000000003,"5min":235.517,"15min":235.517},"last":1.1639999999999999},{"interface":"front","average":{"1min":1.1379999999999999,"5min":4.7510000000000003,"15min":5.226},"min":{"1min":0.27300000000000002,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.9390000000000001,"5min":235.33799999999999,"15min":235.33799999999999},"last":1.1830000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:30:23 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":2.532,"15min":3.4630000000000001},"min":{"1min":0.248,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":2.593,"5min":61.506,"15min":71.819000000000003},"last":2.3170000000000002},{"interface":"front","average":{"1min":1.397,"5min":2.29,"15min":3.1629999999999998},"min":{"1min":0.29399999999999998,"5min":0.23899999999999999,"15min":0.20899999999999999},"max":{"1min":5.452,"5min":61.524999999999999,"15min":71.805000000000007},"last":0.28100000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:29:59 2023","interfaces":[{"interface":"back","average":{"1min":1.097,"5min":3.3919999999999999,"15min":5.25},"min":{"1min":0.27900000000000003,"5min":0.255,"15min":0.23499999999999999},"max":{"1min":5.2119999999999997,"5min":75.701999999999998,"15min":86.332999999999998},"last":1.7929999999999999},{"interface":"front","average":{"1min":0.89300000000000002,"5min":3.198,"15min":5.2990000000000004},"min":{"1min":0.30399999999999999,"5min":0.27200000000000002,"15min":0.252},"max":{"1min":3.9670000000000001,"5min":75.388999999999996,"15min":86.158000000000001},"last":2.5750000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.05,"5min":1.4019999999999999,"15min":1.667},"min":{"1min":0.316,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":3.4830000000000001,"5min":40.119999999999997,"15min":40.119999999999997},"last":2.0739999999999998},{"interface":"front","average":{"1min":0.999,"5min":1.6399999999999999,"15min":1.837},"min":{"1min":0.35599999999999998,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":2.7519999999999998,"5min":40.198,"15min":40.198},"last":2.0169999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:30:29 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":3.355,"15min":2.5230000000000001},"min":{"1min":0.372,"5min":0.34799999999999998,"15min":0.28899999999999998},"max":{"1min":2.9750000000000001,"5min":107.021,"15min":107.021},"last":2.2839999999999998},{"interface":"front","average":{"1min":0.91400000000000003,"5min":3.012,"15min":2.3439999999999999},"min":{"1min":0.41799999999999998,"5min":0.318,"15min":0.317},"max":{"1min":2.323,"5min":107.136,"15min":107.136},"last":0.38200000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:29:35 2023","interfaces":[{"interface":"back","average":{"1min":1.9330000000000001,"5min":1.7889999999999999,"15min":2.1339999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.35399999999999998},"max":{"1min":8.359,"5min":9.2289999999999992,"15min":9.4600000000000009},"last":2.9860000000000002},{"interface":"front","average":{"1min":2.0840000000000001,"5min":1.5960000000000001,"15min":1.827},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":8.5069999999999997,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":0.34899999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:29:45 2023","interfaces":[{"interface":"back","average":{"1min":8.2170000000000005,"5min":3.6349999999999998,"15min":2.871},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":112.70699999999999,"5min":112.70699999999999,"15min":112.70699999999999},"last":1.7470000000000001},{"interface":"front","average":{"1min":7.5679999999999996,"5min":4.1559999999999997,"15min":3.5880000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":109.402,"5min":109.402,"15min":109.402},"last":3.0299999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402852,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2009544,"kb_used_data":1204032,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":91735608,"statfs":{"total":95995035648,"available":93937262592,"internally_reserved":0,"allocated":1232928768,"data_stored":1230940393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":824832169},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:30:02 2023","interfaces":[{"interface":"back","average":{"1min":5.774,"5min":3.9780000000000002,"15min":3.734},"min":{"1min":0.23499999999999999,"5min":0.221,"15min":0.183},"max":{"1min":74.671000000000006,"5min":88.489000000000004,"15min":108.318},"last":0.245},{"interface":"front","average":{"1min":6.5709999999999997,"5min":4.4619999999999997,"15min":4.1840000000000002},"min":{"1min":0.28599999999999998,"5min":0.189,"15min":0.184},"max":{"1min":74.269999999999996,"5min":88.938000000000002,"15min":108.889},"last":0.34300000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0580000000000001,"5min":4.0659999999999998,"15min":4.5209999999999999},"min":{"1min":0.22700000000000001,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":3.8820000000000001,"5min":121.26900000000001,"15min":126.875},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":4.3920000000000003,"15min":4.9089999999999998},"min":{"1min":0.29299999999999998,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":7.1699999999999999,"5min":121.012,"15min":128.249},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:30:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.625,"15min":2.5899999999999999},"min":{"1min":0.32200000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":9.5830000000000002,"5min":9.5830000000000002,"15min":61.732999999999997},"last":79.808999999999997},{"interface":"front","average":{"1min":2.3879999999999999,"5min":2.0790000000000002,"15min":3.5339999999999998},"min":{"1min":0.31,"5min":0.246,"15min":0.246},"max":{"1min":12.548,"5min":17.091000000000001,"15min":119.449},"last":76.131}]},{"osd":4,"last update":"Sat Sep 23 15:30:02 2023","interfaces":[{"interface":"back","average":{"1min":2.1360000000000001,"5min":1.706,"15min":2.1749999999999998},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":9.4359999999999999,"5min":17.027999999999999,"15min":54.051000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.603,"5min":1.5009999999999999,"15min":1.9990000000000001},"min":{"1min":0.33500000000000002,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":7.6630000000000003,"5min":7.6630000000000003,"15min":52.228999999999999},"last":0.90800000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:30:02 2023","interfaces":[{"interface":"back","average":{"1min":2.0209999999999999,"5min":2.7959999999999998,"15min":2.948},"min":{"1min":0.371,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":9.3759999999999994,"5min":82.793000000000006,"15min":148.619},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.635,"5min":2.5289999999999999,"15min":2.7240000000000002},"min":{"1min":0.30299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":7.758,"5min":82.875,"15min":148.56999999999999},"last":0.72999999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:30:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":3.347,"15min":2.6349999999999998},"min":{"1min":0.37,"5min":0.33800000000000002,"15min":0.32700000000000001},"max":{"1min":3.9300000000000002,"5min":84.665000000000006,"15min":84.665000000000006},"last":0.85199999999999998},{"interface":"front","average":{"1min":1.4410000000000001,"5min":3.669,"15min":3.4409999999999998},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":4.468,"5min":84.897999999999996,"15min":84.897999999999996},"last":0.93899999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:29:47 2023","interfaces":[{"interface":"back","average":{"1min":1.7190000000000001,"5min":10.141999999999999,"15min":11.545999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":6.7679999999999998,"5min":216.56899999999999,"15min":216.56899999999999},"last":0.442},{"interface":"front","average":{"1min":2.9289999999999998,"5min":11.323,"15min":12.722},"min":{"1min":0.49199999999999999,"5min":0.432,"15min":0.432},"max":{"1min":17.300000000000001,"5min":216.548,"15min":216.548},"last":0.53800000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986619,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2336960,"kb_used_data":1904568,"kb_used_omap":0,"kb_used_meta":432384,"kb_avail":91408192,"statfs":{"total":95995035648,"available":93601988608,"internally_reserved":0,"allocated":1950277632,"data_stored":1948917567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":442761216},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:30:26 2023","interfaces":[{"interface":"back","average":{"1min":3.3250000000000002,"5min":3.5099999999999998,"15min":3.9319999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.219},"max":{"1min":36.865000000000002,"5min":159.624,"15min":162.75999999999999},"last":0.25800000000000001},{"interface":"front","average":{"1min":3.1150000000000002,"5min":3.5529999999999999,"15min":3.9399999999999999},"min":{"1min":0.28999999999999998,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":37.081000000000003,"5min":159.71000000000001,"15min":162.34800000000001},"last":0.68600000000000005}]},{"osd":1,"last update":"Sat Sep 23 15:30:14 2023","interfaces":[{"interface":"back","average":{"1min":1.206,"5min":4.9009999999999998,"15min":3.552},"min":{"1min":0.28399999999999997,"5min":0.27900000000000003,"15min":0.245},"max":{"1min":4.7290000000000001,"5min":144.76900000000001,"15min":144.76900000000001},"last":0.79400000000000004},{"interface":"front","average":{"1min":0.72399999999999998,"5min":5.1689999999999996,"15min":3.839},"min":{"1min":0.317,"5min":0.25,"15min":0.25},"max":{"1min":1.9610000000000001,"5min":144.58799999999999,"15min":144.58799999999999},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:30:26 2023","interfaces":[{"interface":"back","average":{"1min":1.2110000000000001,"5min":1.1839999999999999,"15min":2.6389999999999998},"min":{"1min":0.32900000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":2.9529999999999998,"5min":6.0019999999999998,"15min":78.751000000000005},"last":1.8089999999999999},{"interface":"front","average":{"1min":1.0669999999999999,"5min":1.226,"15min":2.7149999999999999},"min":{"1min":0.36199999999999999,"5min":0.315,"15min":0.28999999999999998},"max":{"1min":2.835,"5min":7.4699999999999998,"15min":77.930999999999997},"last":0.876}]},{"osd":3,"last update":"Sat Sep 23 15:30:05 2023","interfaces":[{"interface":"back","average":{"1min":2.766,"5min":2.573,"15min":4.1749999999999998},"min":{"1min":0.41599999999999998,"5min":0.371,"15min":0.30399999999999999},"max":{"1min":35.692,"5min":96.587000000000003,"15min":156.46000000000001},"last":2.1019999999999999},{"interface":"front","average":{"1min":2.8239999999999998,"5min":2.6120000000000001,"15min":4.2430000000000003},"min":{"1min":0.39600000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":35.920999999999999,"5min":96.784999999999997,"15min":156.74299999999999},"last":1.615}]},{"osd":5,"last update":"Sat Sep 23 15:30:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":1.1899999999999999,"15min":1.3009999999999999},"min":{"1min":0.27600000000000002,"5min":0.26500000000000001,"15min":0.221},"max":{"1min":3.548,"5min":40.685000000000002,"15min":87.912999999999997},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.51500000000000001,"5min":1.2250000000000001,"15min":1.4039999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.246},"max":{"1min":0.72099999999999997,"5min":38.439,"15min":88.602999999999994},"last":0.42499999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:29:38 2023","interfaces":[{"interface":"back","average":{"1min":1.3919999999999999,"5min":0.88200000000000001,"15min":1.258},"min":{"1min":0.32900000000000001,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":7.508,"5min":7.508,"15min":7.6470000000000002},"last":0.81200000000000006},{"interface":"front","average":{"1min":0.85699999999999998,"5min":0.85199999999999998,"15min":1.7330000000000001},"min":{"1min":0.318,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":3.806,"5min":5.1589999999999998,"15min":7.5380000000000003},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:29:41 2023","interfaces":[{"interface":"back","average":{"1min":0.875,"5min":1.2949999999999999,"15min":1.365},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":3.8839999999999999,"5min":5.7249999999999996,"15min":5.7249999999999996},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.80700000000000005,"5min":1.0069999999999999,"15min":1.04},"min":{"1min":0.33700000000000002,"5min":0.314,"15min":0.314},"max":{"1min":4.5700000000000003,"5min":4.8300000000000001,"15min":4.8300000000000001},"last":0.40699999999999997}]}]},{"osd":5,"up_from":39,"seq":167503724982,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1705332,"kb_used_data":1201772,"kb_used_omap":0,"kb_used_meta":503552,"kb_avail":92039820,"statfs":{"total":95995035648,"available":94248775680,"internally_reserved":0,"allocated":1230614528,"data_stored":1229262988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":515637248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:29:30 2023","interfaces":[{"interface":"back","average":{"1min":3.109,"5min":6.0570000000000004,"15min":4.6319999999999997},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.217},"max":{"1min":18.937000000000001,"5min":196.77000000000001,"15min":196.77000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":2.9820000000000002,"5min":5.9109999999999996,"15min":4.7389999999999999},"min":{"1min":0.25800000000000001,"5min":0.24299999999999999,"15min":0.23300000000000001},"max":{"1min":18.562000000000001,"5min":196.91399999999999,"15min":196.91399999999999},"last":0.41999999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:30:20 2023","interfaces":[{"interface":"back","average":{"1min":1.155,"5min":4.2969999999999997,"15min":3.9670000000000001},"min":{"1min":0.26700000000000002,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":4.9729999999999999,"5min":193.66499999999999,"15min":193.66499999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":1.004,"5min":4.359,"15min":3.7669999999999999},"min":{"1min":0.30099999999999999,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.0460000000000003,"5min":193.52099999999999,"15min":193.52099999999999},"last":0.46400000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:29:30 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.4450000000000001,"15min":2.6920000000000002},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.25},"max":{"1min":6.0570000000000004,"5min":6.2069999999999999,"15min":193.00999999999999},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.98,"5min":1.7789999999999999,"15min":2.9199999999999999},"min":{"1min":0.33800000000000002,"5min":0.316,"15min":0.27900000000000003},"max":{"1min":8.8010000000000002,"5min":9.125,"15min":193.15700000000001},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:30:04 2023","interfaces":[{"interface":"back","average":{"1min":1.5469999999999999,"5min":4.2320000000000002,"15min":6.3849999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.29199999999999998},"max":{"1min":6.2880000000000003,"5min":150.75899999999999,"15min":266.70299999999997},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.377,"5min":4.0700000000000003,"15min":6.5190000000000001},"min":{"1min":0.32700000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":10.762,"5min":150.62299999999999,"15min":266.80399999999997},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:30:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":1.0069999999999999,"15min":1.659},"min":{"1min":0.32100000000000001,"5min":0.249,"15min":0.249},"max":{"1min":3.331,"5min":9.7189999999999994,"15min":138.68600000000001},"last":0.249},{"interface":"front","average":{"1min":0.93300000000000005,"5min":0.92900000000000005,"15min":1.6539999999999999},"min":{"1min":0.34000000000000002,"5min":0.33200000000000002,"15min":0.27800000000000002},"max":{"1min":2.984,"5min":4.7140000000000004,"15min":138.49600000000001},"last":0.28499999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:29:40 2023","interfaces":[{"interface":"back","average":{"1min":0.96199999999999997,"5min":1.2250000000000001,"15min":2.02},"min":{"1min":0.34000000000000002,"5min":0.33400000000000002,"15min":0.308},"max":{"1min":2.992,"5min":8.6989999999999998,"15min":10.577999999999999},"last":35.667000000000002},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.1559999999999999,"15min":2.0659999999999998},"min":{"1min":0.36199999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.944,"5min":6.242,"15min":13.257999999999999},"last":36.140999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:29:42 2023","interfaces":[{"interface":"back","average":{"1min":1.4650000000000001,"5min":1.319,"15min":1.2949999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":9.7400000000000002,"5min":9.7400000000000002,"15min":9.7400000000000002},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.0029999999999999,"5min":1.5009999999999999,"15min":1.5840000000000001},"min":{"1min":0.36799999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.081,"5min":6.0060000000000002,"15min":6.0060000000000002},"last":0.32100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2440101888,"data_stored":2440101888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":688652288,"data_stored":688652288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1477246976,"data_stored":1477246976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1231028224,"data_stored":1231028224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1945894912,"data_stored":1945894912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1225785344,"data_stored":1225785344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1725431808,"data_stored":1725431808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":984154112,"data_stored":984154112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:30:37.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 41 2023-09-23T15:30:37.678 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:37.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:37.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:37.708 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:37 smithi017 ceph-mon[215646]: from='client.46574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:37.708 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:37 smithi017 ceph-mon[222878]: from='client.46574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:37.783 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:37.812 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:37.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:37 smithi027 ceph-mon[195233]: from='client.46574 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:30:37.860 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:37.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:37.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:37.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:38.055 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:38.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:30:38.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 41 16 63632 63616 96.9203 56.1875 0.0164033 0.0103123 2023-09-23T15:30:38.321 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:38.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:38.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:38.335 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:38] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-23T15:30:38.397 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:38.410 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:38.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:38.450 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:38.484 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:38.506 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:38.524 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:38.565 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:38.578 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:38 smithi027 ceph-mon[195233]: pgmap v1238: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-09-23T15:30:38.578 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3284939118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.578 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:38 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.578 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1677559069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:38.580 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:30:38.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[215646]: pgmap v1238: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-09-23T15:30:38.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3284939118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1677559069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:38.623 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[222878]: pgmap v1238: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 980 op/s 2023-09-23T15:30:38.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3284939118' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:38.624 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1677559069' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:38.624 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:38.645 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:38.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:38.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:38.666 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:38.712 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.733 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:30:38.757 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:38.809 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:38.913 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:38.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:38.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:39.004 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:39.025 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:39.090 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:39.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 42 16 64793 64777 96.3406 72.5625 0.00635276 0.0103746 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.569 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:39 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:39.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:39.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: osdmap e423: 8 total, 8 up, 8 in 2023-09-23T15:30:39.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: failed to encode map e423 with expected crc 2023-09-23T15:30:39.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:39 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:40.283 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:40.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:40.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:40.288 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 43 16 65646 65630 95.3403 53.3125 0.00321815 0.0104477 2023-09-23T15:30:40.387 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:40.438 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:40.727 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:40 smithi027 ceph-mon[195233]: pgmap v1240: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T15:30:40.736 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:40.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:40.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:40.801 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:40 smithi017 ceph-mon[222878]: pgmap v1240: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T15:30:40.804 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:40 smithi017 ceph-mon[215646]: pgmap v1240: 119 pgs: 119 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s 2023-09-23T15:30:40.847 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:40.913 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:41.142 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:41.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:41.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:41.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:41.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 44 16 66764 66748 94.7616 69.875 0.00930741 0.0105479 2023-09-23T15:30:41.325 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:41.430 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:41.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:41.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:41.447 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:30:41.447 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:30:41.447 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:30:41.447 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:30:41.447 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:30:41.536 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:41.590 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:41.702 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:41.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:41.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:41.812 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:41.889 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:41.945 INFO:teuthology.orchestra.run.smithi017.stderr:Error EINVAL: specified pgp_num 41 > pg_num 31 2023-09-23T15:30:41.968 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:41.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:41.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:42.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:42.101 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:42.148 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:42.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:42.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:42.252 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:42.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 45 16 67658 67642 93.8977 55.875 0.00772901 0.0106432 2023-09-23T15:30:42.292 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:42.361 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:42.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:42.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:42.467 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:42.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:30:42.525 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:42.613 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:30:42.621 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:42.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:42.727 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:42.793 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:42.832 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:30:42.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:42 smithi027 ceph-mon[195233]: pgmap v1241: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:42 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1656370805' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:42.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:42.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[215646]: pgmap v1241: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:42.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1656370805' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:42.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:42.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[222878]: pgmap v1241: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:42.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1656370805' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:42.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T15:30:43.105 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:43.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:43.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:43.134 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T15:30:43.218 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:43.243 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:30:43.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 46 16 68724 68708 93.305 66.625 0.0151126 0.0107119 2023-09-23T15:30:43.295 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:43.399 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:43.400 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:43.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:43.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:43.511 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:43.572 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:43.615 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:43.667 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:43.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:43.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:43.777 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:43.828 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:44.027 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:44.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:44.132 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:44.173 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:44.195 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:44 smithi027 ceph-mon[195233]: pgmap v1242: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:44.195 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:44.248 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:44.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:44.292 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 47 16 69778 69762 92.7215 65.875 0.0125852 0.0107803 2023-09-23T15:30:44.359 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:44.426 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:44.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:44.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[215646]: pgmap v1242: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:44.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:44.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:44.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[222878]: pgmap v1242: 119 pgs: 119 active+clean; 4.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:30:44.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:44.633 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:44.877 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:44.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:44.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:44.982 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:45.056 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:45.214 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:45.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:45.219 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:45.289 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 48 16 70706 70690 91.9983 58 0.00959179 0.0108634 2023-09-23T15:30:45.320 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:45.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:45.400 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50575 8.4 8:2aa5975c:::benchmark_data_smithi017_485823_object50574:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.20929241299996, 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1322413290000002, 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.401 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50575 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189079+0000", 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189078+0000", 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189126+0000", 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7305000000000002e-05 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189145+0000", 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9389e-05 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189150+0000", 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0950000000000003e-06 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189213+0000", 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2564000000000006e-05 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.189224+0000", 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.097e-05 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136177+0000", 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94695325500000005 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136190+0000", 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3512e-05 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296297+0000", 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.160106919 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296465+0000", 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016758800000000001 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.296632+0000", 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016706199999999999 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305668+0000", 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090357079999999999 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310724+0000", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050558169999999998 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319181+0000", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0084576249999999999 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.319233+0000", 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2042000000000001e-05 2023-09-23T15:30:45.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321321+0000", 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020875210000000002 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50584 8.4 8:2701dc0f:::benchmark_data_smithi017_485823_object50583:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.199543393, 2023-09-23T15:30:45.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1246680649999998, 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50584 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198828+0000", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198827+0000", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198852+0000", 2023-09-23T15:30:45.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4725000000000001e-05 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198864+0000", 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2480000000000001e-05 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198869+0000", 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8330000000000001e-06 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198914+0000", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5084999999999999e-05 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.198921+0000", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6320000000000002e-06 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136226+0000", 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93730514799999998 2023-09-23T15:30:45.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136234+0000", 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0450000000000001e-06 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298417+0000", 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.162183065 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298513+0000", 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5981000000000001e-05 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.298613+0000", 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100184 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305680+0000", 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070664450000000002 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310775+0000", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050952819999999996 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321371+0000", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010595751 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.321413+0000", 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2296999999999998e-05 2023-09-23T15:30:45.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.323496+0000", 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020832870000000001 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50598 8.4 8:23276db8:::benchmark_data_smithi017_485823_object50597:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.18751739100003, 2023-09-23T15:30:45.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1164891140000002, 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50598 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210854+0000", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210890+0000", 2023-09-23T15:30:45.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6159999999999999e-05 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210905+0000", 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5471000000000001e-05 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210912+0000", 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7299999999999999e-06 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210973+0000", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1427999999999995e-05 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.210982+0000", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4239999999999993e-06 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136258+0000", 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92527595200000001 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136265+0000", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.413e-06 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300042+0000", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.163777327 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300173+0000", 2023-09-23T15:30:45.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013054000000000001 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.300273+0000", 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9741999999999995e-05 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.308013+0000", 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077400969999999996 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310837+0000", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028244099999999999 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.325081+0000", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014243281 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.325147+0000", 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6896999999999993e-05 2023-09-23T15:30:45.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327344+0000", 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021961089999999999 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50603 8.4 8:2629f6f1:::benchmark_data_smithi017_485823_object50602:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.18379521099996, 2023-09-23T15:30:45.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1148396169999999, 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50603 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:30:45.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214577+0000", 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214576+0000", 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214595+0000", 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9415e-05 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214607+0000", 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1593000000000001e-05 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214612+0000", 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2560000000000002e-06 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214654+0000", 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2367000000000003e-05 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.214660+0000", 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.755e-06 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136288+0000", 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92162821399999995 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136296+0000", 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9079999999999995e-06 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301680+0000", 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.165384051 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301775+0000", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4308000000000006e-05 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.301887+0000", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112654 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.308024+0000", 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061371790000000004 2023-09-23T15:30:45.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310884+0000", 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028592689999999998 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327407+0000", 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.016523044000000001 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.327471+0000", 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4053999999999994e-05 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.329416+0000", 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001945348 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50611 8.4 8:263c3c34:::benchmark_data_smithi017_485823_object50610:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:30:45.433 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.17360292900003, 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1099267500000001, 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50611 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:30:45.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224769+0000", 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224767+0000", 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224833+0000", 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6395000000000004e-05 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224864+0000", 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1495999999999998e-05 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224888+0000", 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4000999999999999e-05 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224936+0000", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7963999999999997e-05 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.224942+0000", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8150000000000002e-06 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136321+0000", 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91137871999999998 2023-09-23T15:30:45.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136329+0000", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6939999999999995e-06 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303349+0000", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.16701991 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303507+0000", 2023-09-23T15:30:45.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015815600000000001 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.303691+0000", 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018426799999999999 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310737+0000", 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070460219999999999 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.311156+0000", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041903600000000002 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.332349+0000", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021193132 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.332434+0000", 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4668999999999999e-05 2023-09-23T15:30:45.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334696+0000", 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022617779999999999 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:50612 8.4 8:2bb28cd9:::benchmark_data_smithi017_485823_object50611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 529.17328213999997, 2023-09-23T15:30:45.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1116500949999999, 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 50612 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225090+0000", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225089+0000", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225146+0000", 2023-09-23T15:30:45.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7031999999999999e-05 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225156+0000", 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0304e-05 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225160+0000", 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0199999999999996e-06 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225201+0000", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0935e-05 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:56.225209+0000", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9500000000000001e-06 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136352+0000", 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91114233099999997 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:57.136359+0000", 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5009999999999998e-06 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305733+0000", 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1693735030000001 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.305920+0000", 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018741100000000001 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.306064+0000", 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014344600000000001 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.310849+0000", 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047855119999999996 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.312187+0000", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001338452 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334753+0000", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022565686000000001 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.334800+0000", 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.655e-05 2023-09-23T15:30:45.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:21:58.336740+0000", 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001940048 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148874 8.4 8:22ed0ae9:::benchmark_data_smithi017_485823_object58737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.57050005500002, 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99896163400000004, 2023-09-23T15:30:45.449 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148874 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827871+0000", 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827874+0000", 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.311e-06 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827879+0000", 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4600000000000002e-06 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827883+0000", 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3730000000000001e-06 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831017+0000", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031349569999999998 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831029+0000", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1875000000000001e-05 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782403+0000", 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95137329500000001 2023-09-23T15:30:45.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782500+0000", 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7763999999999994e-05 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782578+0000", 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7972999999999994e-05 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794469+0000", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011890258000000001 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826626+0000", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032157673999999997 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826744+0000", 2023-09-23T15:30:45.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117747 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826793+0000", 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9308000000000003e-05 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826833+0000", 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0000000000000003e-05 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.456 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148906 8.4 8:29f45dea:::benchmark_data_smithi017_485823_object58769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.52981909200003, 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95839901500000002, 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.457 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148906 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868552+0000", 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868555+0000", 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868558+0000", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7699999999999999e-06 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868561+0000", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868612+0000", 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0853000000000003e-05 2023-09-23T15:30:45.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868618+0000", 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4529999999999999e-06 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785580+0000", 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91696206300000005 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785720+0000", 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139677 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785862+0000", 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014233400000000001 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799990+0000", 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014128043999999999 2023-09-23T15:30:45.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826695+0000", 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026704894 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826890+0000", 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019431599999999999 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826928+0000", 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8148000000000001e-05 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826952+0000", 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4049e-05 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148912 8.4 8:2ad9ecb5:::benchmark_data_smithi017_485823_object58775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.464 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.525865528, 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.954622631, 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148912 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872505+0000", 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:45.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872512+0000", 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8390000000000001e-06 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872524+0000", 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2469e-05 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872530+0000", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.139e-06 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872582+0000", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1983e-05 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872592+0000", 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3710000000000006e-06 2023-09-23T15:30:45.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788471+0000", 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91587959200000002 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788569+0000", 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8047000000000005e-05 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788661+0000", 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1267000000000006e-05 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799997+0000", 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011336841 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827002+0000", 2023-09-23T15:30:45.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027004980000000001 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827082+0000", 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9484e-05 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827107+0000", 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5216999999999999e-05 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827129+0000", 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1801e-05 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148934 8.4 8:2bdf4584:::benchmark_data_smithi017_485823_object58797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.49415725599999, 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92312028400000001, 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.472 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148934 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904213+0000", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904224+0000", 2023-09-23T15:30:45.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0525e-05 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904236+0000", 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2649999999999999e-05 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904243+0000", 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.241e-06 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904339+0000", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6711000000000003e-05 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904349+0000", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7340000000000003e-06 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791162+0000", 2023-09-23T15:30:45.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88681287099999995 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791285+0000", 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012337700000000001 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791419+0000", 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134029 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804693+0000", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013273785 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827031+0000", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022337704999999999 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827243+0000", 2023-09-23T15:30:45.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021219399999999999 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827294+0000", 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.113e-05 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827335+0000", 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0677999999999999e-05 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.479 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148942 8.4 8:2f2d7443:::benchmark_data_smithi017_485823_object58805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.48787573599998, 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91690406000000002, 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148942 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.480 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.481 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910495+0000", 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910500+0000", 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.57e-06 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910514+0000", 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.365e-05 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910522+0000", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7740000000000004e-06 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910590+0000", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8472000000000004e-05 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910602+0000", 2023-09-23T15:30:45.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1343e-05 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794512+0000", 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88391032800000002 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794644+0000", 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013237700000000001 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794779+0000", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134709 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804705+0000", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099256569999999992 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827184+0000", 2023-09-23T15:30:45.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022478946999999999 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827355+0000", 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171628 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827381+0000", 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5789999999999999e-05 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.486 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827400+0000", 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8995e-05 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148943 8.4 8:26834e88:::benchmark_data_smithi017_485823_object58806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.48783231800002, 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.916991004, 2023-09-23T15:30:45.487 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148943 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.488 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910542+0000", 2023-09-23T15:30:45.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7889999999999999e-06 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910554+0000", 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2216e-05 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910560+0000", 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1990000000000003e-06 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.490 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910623+0000", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3254000000000001e-05 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910628+0000", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797471+0000", 2023-09-23T15:30:45.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88684291900000001 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797586+0000", 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114815 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797686+0000", 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100275 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810164+0000", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012477492 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827205+0000", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017040726999999999 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827444+0000", 2023-09-23T15:30:45.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023902400000000001 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827492+0000", 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8707e-05 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827530+0000", 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8102000000000002e-05 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.494 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148947 8.4 8:25191517:::benchmark_data_smithi017_485823_object58810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.48349567499997, 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91295790200000004, 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.495 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148947 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914879+0000", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.683e-06 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914884+0000", 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8189999999999997e-06 2023-09-23T15:30:45.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914888+0000", 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.121e-06 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914950+0000", 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1359999999999995e-05 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.498 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914961+0000", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.076e-05 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800046+0000", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88508585500000003 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800194+0000", 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014714700000000001 2023-09-23T15:30:45.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800323+0000", 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129197 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810173+0000", 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098501830000000002 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827554+0000", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017381285 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827782+0000", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022772600000000001 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827812+0000", 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0433000000000002e-05 2023-09-23T15:30:45.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827834+0000", 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1628e-05 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148958 8.4 8:2e39e1d6:::benchmark_data_smithi017_485823_object58821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:30:45.502 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 378.469887019, 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89949832200000002, 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148958 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:30:45.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928483+0000", 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928493+0000", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0638999999999999e-05 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928514+0000", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0859000000000001e-05 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928523+0000", 2023-09-23T15:30:45.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5860000000000005e-06 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928590+0000", 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7299999999999996e-05 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928600+0000", 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7189999999999996e-06 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802452+0000", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87385171800000006 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802535+0000", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3803999999999999e-05 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802599+0000", 2023-09-23T15:30:45.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3928000000000002e-05 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815177+0000", 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012578119 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827593+0000", 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012415981 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827890+0000", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029696900000000001 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827945+0000", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4586999999999998e-05 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827983+0000", 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8109999999999999e-05 2023-09-23T15:30:45.509 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83112 9.a 9:5853ad9e:::benchmark_data_smithi017_543597_object83111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.506346167, 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0903713559999999, 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.510 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83112 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:30:45.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892024+0000", 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892085+0000", 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1613999999999997e-05 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892103+0000", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7893e-05 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892110+0000", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892224+0000", 2023-09-23T15:30:45.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114508 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892371+0000", 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014686500000000001 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892552+0000", 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018046499999999999 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.514 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898624+0000", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060722090000000003 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912313+0000", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013688707 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912322+0000", 2023-09-23T15:30:45.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4839999999999993e-06 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.970323+0000", 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0580004119999999 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971026+0000", 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070397199999999997 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982334+0000", 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011307914000000001 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982397+0000", 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2570999999999994e-05 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.517 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83117 9.a 9:51c7f339:::benchmark_data_smithi017_543597_object83116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.502847395, 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0870892780000001, 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83117 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.518 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895523+0000", 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895570+0000", 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6748e-05 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895583+0000", 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3390000000000001e-05 2023-09-23T15:30:45.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895590+0000", 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895803+0000", 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021333300000000001 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.521 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895915+0000", 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111697 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896025+0000", 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109511 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.899384+0000", 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033596300000000002 2023-09-23T15:30:45.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912365+0000", 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012981029999999999 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912370+0000", 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5549999999999996e-06 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971071+0000", 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.058701627 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.973148+0000", 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020767469999999999 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982568+0000", 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094202609999999992 2023-09-23T15:30:45.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982614+0000", 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5259000000000002e-05 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83120 9.7 9:e8f3216b:::benchmark_data_smithi017_543597_object83119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 229.500272005, 2023-09-23T15:30:45.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.062896818, 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83120 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:30:45.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898098+0000", 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898169+0000", 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0426000000000006e-05 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898180+0000", 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1114999999999999e-05 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898186+0000", 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0159999999999997e-06 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898238+0000", 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1777000000000002e-05 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898377+0000", 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138825 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898557+0000", 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000180058 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.901069+0000", 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025124309999999999 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.530 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912790+0000", 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011721352000000001 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912795+0000", 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5430000000000004e-06 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948511+0000", 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0357156709999999 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.949214+0000", 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070288299999999996 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960946+0000", 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011732340000000001 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960997+0000", 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0702999999999999e-05 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88042 9.a 9:5af6f1a8:::benchmark_data_smithi017_543597_object4251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 185.004839971, 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9167718300000001, 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:30:45.533 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88042 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393530+0000", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393536+0000", 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7370000000000003e-06 2023-09-23T15:30:45.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393552+0000", 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5713999999999999e-05 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393560+0000", 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0800000000000006e-06 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393617+0000", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6884e-05 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393770+0000", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015349600000000001 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393905+0000", 2023-09-23T15:30:45.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013507699999999999 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400071+0000", 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061654379999999996 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400083+0000", 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2109e-05 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.538 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389036+0000", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98895255699999995 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389049+0000", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3633e-05 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303003+0000", 2023-09-23T15:30:45.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.913953927 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303184+0000", 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000181175 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310231+0000", 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007047045 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310304+0000", 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2308e-05 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88055 9.a 9:5bed7ee7:::benchmark_data_smithi017_543597_object4264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 184.95837087199999, 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8698463969999999, 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.541 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88055 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440000+0000", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440006+0000", 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5049999999999998e-06 2023-09-23T15:30:45.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440018+0000", 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2359e-05 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440025+0000", 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2849999999999997e-06 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440084+0000", 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.855e-05 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440092+0000", 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.585e-06 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389071+0000", 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94897856800000002 2023-09-23T15:30:45.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389079+0000", 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6590000000000007e-06 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303210+0000", 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91413175199999996 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303344+0000", 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013339999999999999 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303490+0000", 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014616299999999999 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.307392+0000", 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003901655 2023-09-23T15:30:45.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309301+0000", 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019098609999999999 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309677+0000", 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037558600000000002 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309714+0000", 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6791000000000001e-05 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309847+0000", 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133454 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88057 9.7 9:eebbf230:::benchmark_data_smithi017_543597_object4266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:30:45.549 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 184.95443306499999, 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.85708753, 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88057 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:30:45.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443938+0000", 2023-09-23T15:30:45.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443943+0000", 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5079999999999999e-06 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443955+0000", 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2754e-05 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443965+0000", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8339999999999994e-06 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444010+0000", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5330000000000001e-05 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444019+0000", 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2320000000000001e-06 2023-09-23T15:30:45.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387832+0000", 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.943813609 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387845+0000", 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2853e-05 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297101+0000", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90925576399999997 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297239+0000", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013821800000000001 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297322+0000", 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3220999999999995e-05 2023-09-23T15:30:45.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.299762+0000", 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024393729999999999 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300155+0000", 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039295999999999998 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300804+0000", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064965300000000004 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300868+0000", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3898000000000004e-05 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.301026+0000", 2023-09-23T15:30:45.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015802500000000001 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:30:45.558 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:30:45.910 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:45.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:45.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:46.013 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:46.060 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:46.275 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:46 smithi017 ceph-mon[215646]: pgmap v1243: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-09-23T15:30:46.276 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:46 smithi017 ceph-mon[222878]: pgmap v1243: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-09-23T15:30:46.291 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 49 16 71395 71379 90.9999 43.0625 0.00961651 0.0109841 2023-09-23T15:30:46.312 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:46 smithi027 ceph-mon[195233]: pgmap v1243: 119 pgs: 119 active+clean; 4.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2023-09-23T15:30:46.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:30:46.341 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:46.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:46.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:46.448 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:46.479 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:46.545 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:46.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:46.552 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:46.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:46.685 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:46.753 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:46.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:46.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:46.866 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:46.897 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:46.961 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:46.966 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:46.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:47.002 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:47.056 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:47.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:47.097 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:47.146 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:47.148 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:47.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:47.182 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:47.219 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:47.245 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:30:47.262 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:47.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:47.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:47.289 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:47.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.289 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.290 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:30:47.290 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 50 16 72172 72156 90.151 48.5625 0.0186903 0.0110858 2023-09-23T15:30:47.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3531959391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:47.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2461686751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.352 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.391 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:47.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3531959391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:47.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2461686751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3531959391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2461686751' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:30:47.480 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:48.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:30:48.136 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:30:48.290 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:48] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: pgmap v1244: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.291 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[215646]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.291 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 51 16 73090 73074 89.5082 57.375 0.0197548 0.0111656 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: pgmap v1244: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.292 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:48 smithi017 ceph-mon[222878]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.551 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:48.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:48.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:48.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: pgmap v1244: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: osdmap e424: 8 total, 8 up, 8 in 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:48 smithi027 ceph-mon[195233]: failed to encode map e424 with expected crc 2023-09-23T15:30:48.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:48.756 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:49.290 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 52 16 74179 74163 89.0956 68.0625 0.014734 0.0112192 2023-09-23T15:30:49.378 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:49.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:49.389 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:49.489 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:49.516 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:49.561 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:49.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:49.570 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:49.671 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:49.739 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:49.839 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:49.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:49.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:49.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:49.996 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:50.040 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:50.067 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:50.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:50.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:50.182 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:50.234 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:50.290 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 53 16 75092 75076 88.4913 57.0625 0.0182977 0.0112958 2023-09-23T15:30:50.309 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:50.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:50.421 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:50.461 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:50.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[222878]: 1.c scrub starts 2023-09-23T15:30:50.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[222878]: 1.c scrub ok 2023-09-23T15:30:50.529 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[222878]: pgmap v1246: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T15:30:50.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[215646]: 1.c scrub starts 2023-09-23T15:30:50.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[215646]: 1.c scrub ok 2023-09-23T15:30:50.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:50 smithi017 ceph-mon[215646]: pgmap v1246: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T15:30:50.538 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:50.546 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:50 smithi027 ceph-mon[195233]: 1.c scrub starts 2023-09-23T15:30:50.546 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:50 smithi027 ceph-mon[195233]: 1.c scrub ok 2023-09-23T15:30:50.546 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:50 smithi027 ceph-mon[195233]: pgmap v1246: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 62 MiB/s wr, 989 op/s 2023-09-23T15:30:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:50.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:50.649 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:50.686 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:30:50.686 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:30:50.729 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:50.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:30:51.228 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:30:51.290 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 54 16 76169 76153 88.0992 67.3125 0.0174586 0.0113451 2023-09-23T15:30:51.971 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:51.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:52.083 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:52.141 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:52.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:30:52.270 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:52.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:52.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:52.290 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 55 16 77181 77165 87.6474 63.25 0.00794779 0.0114036 2023-09-23T15:30:52.366 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:52.379 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:52.420 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:52.442 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:52.494 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:52.499 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:52 smithi027 ceph-mon[195233]: pgmap v1247: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:52.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:52.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:52.600 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:52.619 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:52 smithi017 ceph-mon[222878]: pgmap v1247: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:52.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:52 smithi017 ceph-mon[215646]: pgmap v1247: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:52.652 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:52.743 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:52.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:52.858 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:52.921 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:52.988 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:52.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:52.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:53.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:53.177 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:53.290 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:53.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:53.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 56 16 78078 78062 87.0709 56.0625 0.00990444 0.0114785 2023-09-23T15:30:53.396 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:53.449 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:53.539 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:53.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:53.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:30:53.576 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:30:53.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:53.682 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:53.739 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:53.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:30:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:53.743 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:53.828 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:53.843 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:53.900 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:53.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:30:53.975 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:53.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:53.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:54.079 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:30:54.146 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:54.261 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:30:54.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:54.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:30:54.298 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 57 16 78935 78919 86.4832 53.5625 0.00555982 0.0115549 2023-09-23T15:30:54.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:54.418 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: pgmap v1248: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:30:54.557 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[215646]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: pgmap v1248: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:30:54.558 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:54 smithi017 ceph-mon[222878]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T15:30:54.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: pgmap v1248: 119 pgs: 119 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 58 MiB/s wr, 934 op/s 2023-09-23T15:30:54.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.559 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: failed to encode map e425 with expected crc 2023-09-23T15:30:54.560 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2887474011' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:30:54.560 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:54 smithi027 ceph-mon[195233]: osdmap e425: 8 total, 8 up, 8 in 2023-09-23T15:30:55.263 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:55.275 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:55.275 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:55.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 58 16 79961 79945 86.0978 64.125 0.00576571 0.0116103 2023-09-23T15:30:55.377 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:55.450 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:55.528 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:30:55.802 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:55.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:55.916 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:55.959 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:56.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:56.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:56.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:30:56.168 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:30:56.239 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:56.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 59 16 80884 80868 85.6164 57.6875 0.00367279 0.0116623 2023-09-23T15:30:56.560 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:30:56.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:30:56.642 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:30:56.684 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:30:56.717 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:30:56.756 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:30:56.806 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:30:56.845 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:30:56.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:56 smithi027 ceph-mon[195233]: pgmap v1250: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:30:56.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/963672142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.858 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.859 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:30:56.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:56 smithi017 ceph-mon[215646]: pgmap v1250: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:30:56.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/963672142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:56.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:56 smithi017 ceph-mon[222878]: pgmap v1250: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2023-09-23T15:30:56.894 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/963672142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:30:56.915 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:30:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:56.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:30:57.035 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:30:57.101 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:57.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:30:57.299790+0000 min lat: 0.00117915 max lat: 0.408537 avg lat: 0.0117453 2023-09-23T15:30:57.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:30:57.299 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 60 16 81718 81702 85.0584 52.125 0.165577 0.0117453 2023-09-23T15:30:57.553 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:30:57.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:57.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:30:57.623 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:30:57.659 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:30:57.741 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:57.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1549307695' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.810 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:57 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.811 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1549307695' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.812 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:57 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1549307695' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:57 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:30:57.897 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:30:57.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:57.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:30:58.007 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:30:58.036 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:58.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:30:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:30:58] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T15:30:58.305 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 61 16 82656 82640 84.6174 58.625 0.0104494 0.0118123 2023-09-23T15:30:58.391 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:30:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:30:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:30:58.391 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:30:58.391 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:30:58.546 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:30:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:58.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:30:58.638 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:30:58.660 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:58.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: pgmap v1251: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:58.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: pgmap v1251: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:58.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:30:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:58.716 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:58.820 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:58.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:58.833 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: pgmap v1251: 119 pgs: 119 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 59 MiB/s wr, 940 op/s 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: osdmap e426: 8 total, 8 up, 8 in 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: failed to encode map e426 with expected crc 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:30:58.834 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:30:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:30:58.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:58.935 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:59.000 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:59.097 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:59.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:59.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:59.203 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:30:59.261 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:59.304 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 62 16 83736 83720 84.3414 67.5 0.0103407 0.0118515 2023-09-23T15:30:59.380 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:30:59.385 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:30:59.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:59.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:30:59.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:30:59.556 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:59.657 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:30:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:59.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:30:59.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:30:59.770 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:30:59.816 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:30:59.892 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:30:59.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:30:59.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:00.002 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:00.029 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:00.072 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:00.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:00.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:00.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:00.232 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:00.305 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 63 16 84531 84515 83.7915 49.6875 0.0113853 0.0119291 2023-09-23T15:31:00.315 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:00.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:00.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:00.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:00.486 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:00.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:31:00.589 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:00 smithi027 ceph-mon[195233]: pgmap v1253: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-09-23T15:31:00.642 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:00.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:00.667 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:00.675 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:00.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:00 smithi017 ceph-mon[215646]: pgmap v1253: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-09-23T15:31:00.701 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:00 smithi017 ceph-mon[222878]: pgmap v1253: 119 pgs: 119 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 42 MiB/s wr, 667 op/s 2023-09-23T15:31:00.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:00.768 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:00.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:01.003 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:01.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:01.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:01.115 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:01.226 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:01.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 64 16 85390 85374 83.3178 53.6875 0.00974508 0.011997 2023-09-23T15:31:01.503 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:01.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:01.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:01.612 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:01.697 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:02.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 65 16 86321 86305 82.9313 58.1875 0.0170788 0.0120535 2023-09-23T15:31:02.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:02 smithi017 ceph-mon[215646]: pgmap v1254: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:31:02.601 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:31:02.606 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:31:02.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:02 smithi017 ceph-mon[222878]: pgmap v1254: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:31:02.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:31:03.017 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:03.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:03.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:03.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:02 smithi027 ceph-mon[195233]: pgmap v1254: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 75 MiB/s wr, 1.19k op/s 2023-09-23T15:31:03.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:31:03.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:03.182 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:03.294 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:03.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:03.303 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:03.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 66 15 87352 87337 82.6522 64.5 0.00805566 0.0120951 2023-09-23T15:31:03.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:03.464 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:03.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:31:03.730 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[215646]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.731 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.732 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:03 smithi017 ceph-mon[222878]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.733 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:03.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:03.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:03.743 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:03.771 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:31:03.844 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1907542234' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: osdmap e427: 8 total, 8 up, 8 in 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.881 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.882 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.882 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:03 smithi027 ceph-mon[195233]: failed to encode map e427 with expected crc 2023-09-23T15:31:03.891 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:03.989 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:03.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:04.097 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:04.145 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:04.217 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:04.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:04.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 67 16 88291 88275 82.2937 58.625 0.017832 0.0121463 2023-09-23T15:31:04.328 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:04.405 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83106 9.e 9:7e487db8:::benchmark_data_smithi017_543597_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.479149971, 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1221425209999998, 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:31:04.406 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83106 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916522+0000", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916551+0000", 2023-09-23T15:31:04.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9121000000000001e-05 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916561+0000", 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4059999999999994e-06 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916570+0000", 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5149999999999995e-06 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918608+0000", 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020381800000000001 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918619+0000", 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1222e-05 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913525+0000", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490589500000004 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913537+0000", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1749000000000001e-05 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.028480+0000", 2023-09-23T15:31:04.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1149425470000001 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.029082+0000", 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060205899999999995 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038593+0000", 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095110260000000005 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038666+0000", 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3088000000000002e-05 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83114 9.d 9:bfb40ed3:::benchmark_data_smithi017_543597_object83113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.479097376, 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024477069, 2023-09-23T15:31:04.413 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83114 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916575+0000", 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916600+0000", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4606000000000001e-05 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916607+0000", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916613+0000", 2023-09-23T15:31:04.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2219999999999999e-06 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916703+0000", 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9252000000000005e-05 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916711+0000", 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9039999999999998e-06 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.931237+0000", 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0145257830000001 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933189+0000", 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001951745 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933360+0000", 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017091799999999999 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940451+0000", 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070909079999999999 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940940+0000", 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048871099999999996 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940977+0000", 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7722999999999998e-05 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.941053+0000", 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5729000000000006e-05 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.420 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83118 9.e 9:7615419c:::benchmark_data_smithi017_543597_object83117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.47904738899999, 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1299481519999999, 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.421 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83118 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916625+0000", 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916651+0000", 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5911e-05 2023-09-23T15:31:04.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916659+0000", 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916665+0000", 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9460000000000003e-06 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918662+0000", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019976350000000002 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918669+0000", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2910000000000002e-06 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913570+0000", 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490066300000002 2023-09-23T15:31:04.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913580+0000", 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.030629+0000", 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170488599999999 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032411+0000", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001782063 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032533+0000", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012223299999999999 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038742+0000", 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006208613 2023-09-23T15:31:04.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038809+0000", 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6854999999999996e-05 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045301+0000", 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064921479999999997 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045369+0000", 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8693000000000001e-05 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046574+0000", 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012047169999999999 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.429 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83121 9.e 9:71f6c9a6:::benchmark_data_smithi017_543597_object83120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.47891764100001, 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1305874610000002, 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83121 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.430 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.431 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916755+0000", 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916784+0000", 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9522e-05 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916793+0000", 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1000000000000004e-06 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.432 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916799+0000", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7530000000000004e-06 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918708+0000", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001908734 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918717+0000", 2023-09-23T15:31:04.433 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6629999999999999e-06 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914427+0000", 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99571021100000001 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914437+0000", 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0592e-05 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.434 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034594+0000", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12015687 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034760+0000", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001661 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034983+0000", 2023-09-23T15:31:04.435 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022256399999999999 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038750+0000", 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037670210000000002 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038859+0000", 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010865 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.436 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046631+0000", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077723879999999999 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046672+0000", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0759000000000001e-05 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047343+0000", 2023-09-23T15:31:04.437 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067126700000000002 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83124 9.3 9:c55b3f52:::benchmark_data_smithi017_543597_object83123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 248.476672116, 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0418616510000001, 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.438 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83124 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.439 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919000+0000", 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.440 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919028+0000", 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8895e-05 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919041+0000", 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2996000000000001e-05 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919051+0000", 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5729999999999997e-06 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.441 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919101+0000", 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9864999999999998e-05 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919108+0000", 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2760000000000003e-06 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.442 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.944052+0000", 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024943395 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946310+0000", 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022579900000000001 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946472+0000", 2023-09-23T15:31:04.443 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016254900000000001 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.958448+0000", 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975497999999999 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960747+0000", 2023-09-23T15:31:04.444 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002299592 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960794+0000", 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6409000000000002e-05 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960863+0000", 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9227999999999995e-05 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.445 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83137 9.e 9:7d999fd1:::benchmark_data_smithi017_543597_object83136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 247.404075947, 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056604109, 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.446 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83137 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.447 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991596+0000", 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.448 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991621+0000", 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4907000000000001e-05 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991634+0000", 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2556e-05 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.449 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991642+0000", 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1340000000000004e-06 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991689+0000", 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7781000000000002e-05 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991696+0000", 2023-09-23T15:31:04.450 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5239999999999997e-06 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038894+0000", 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.047198039 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039062+0000", 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016794100000000001 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.451 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039246+0000", 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018399900000000001 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045231+0000", 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059852339999999999 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.452 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046596+0000", 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001365125 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047421+0000", 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082434100000000003 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.453 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047483+0000", 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2686000000000003e-05 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048201+0000", 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071811100000000001 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.454 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83228 9.e 9:7f83ce07:::benchmark_data_smithi017_543597_object83227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 247.239526732, 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89863112300000003, 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83228 2023-09-23T15:31:04.455 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.456 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156145+0000", 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156194+0000", 2023-09-23T15:31:04.457 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8671000000000002e-05 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156210+0000", 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6716999999999999e-05 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156219+0000", 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:31:04.458 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156278+0000", 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8897999999999998e-05 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156292+0000", 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.459 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042118+0000", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88582658800000003 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042277+0000", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015909299999999999 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042470+0000", 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193022 2023-09-23T15:31:04.460 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048224+0000", 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057530630000000001 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048291+0000", 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7751999999999998e-05 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.461 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.053961+0000", 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056697969999999999 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054020+0000", 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9419999999999997e-05 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.462 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054778+0000", 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075720099999999999 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83232 9.e 9:72608dfb:::benchmark_data_smithi017_543597_object83231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 247.227665094, 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88775156300000002, 2023-09-23T15:31:04.463 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83232 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.464 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168006+0000", 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.465 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168035+0000", 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8824e-05 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168053+0000", 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8185e-05 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168065+0000", 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1391e-05 2023-09-23T15:31:04.466 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168131+0000", 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168140+0000", 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7129999999999994e-06 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.467 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048345+0000", 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88020536000000005 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048556+0000", 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021095900000000001 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.468 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048793+0000", 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023668699999999999 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054856+0000", 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060630129999999999 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054871+0000", 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851000000000001e-05 2023-09-23T15:31:04.469 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054915+0000", 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4753999999999999e-05 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054975+0000", 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9562999999999999e-05 2023-09-23T15:31:04.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:31:04.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078479899999999998 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83259 9.4 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:31:04.473 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 246.339968137, 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.144342432, 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83259 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055704+0000", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055749+0000", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5618000000000003e-05 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:31:04.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012000000000001e-05 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055769+0000", 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055829+0000", 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0362000000000002e-05 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056020+0000", 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190998 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056238+0000", 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021735600000000001 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061804+0000", 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055659680000000001 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065167+0000", 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033630359999999998 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197822+0000", 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132655089 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197887+0000", 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200048+0000", 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002161035 2023-09-23T15:31:04.480 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83260 9.4 9:295d7846:::benchmark_data_smithi017_543597_object83259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 246.33988860599999, 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.146144756, 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.481 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83260 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055784+0000", 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055805+0000", 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:31:04.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055812+0000", 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4960000000000001e-06 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055819+0000", 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.058865+0000", 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030458220000000001 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059023+0000", 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015745099999999999 2023-09-23T15:31:04.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059189+0000", 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166126 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065176+0000", 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005987132 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068875+0000", 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003699698 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200125+0000", 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131249117999999 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200200+0000", 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4951999999999997e-05 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.201929+0000", 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001729897 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83261 9.4 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 246.33906329600001, 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.147188163999999, 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83261 2023-09-23T15:31:04.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056609+0000", 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056631+0000", 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1569999999999998e-05 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056646+0000", 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4924e-05 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056654+0000", 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0160000000000008e-06 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061842+0000", 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051878740000000003 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062037+0000", 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019499900000000001 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062246+0000", 2023-09-23T15:31:04.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020959400000000001 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065285+0000", 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030387909999999999 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069749+0000", 2023-09-23T15:31:04.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044636470000000003 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202002+0000", 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132253035 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202066+0000", 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4215999999999996e-05 2023-09-23T15:31:04.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203798+0000", 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017322940000000001 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83265 9.4 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:31:04.496 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 246.33012002300001, 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.139692201999999, 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83265 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065572+0000", 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8663000000000001e-05 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065576+0000", 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7530000000000001e-06 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065580+0000", 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.326e-06 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065625+0000", 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5077999999999998e-05 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065774+0000", 2023-09-23T15:31:04.500 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014929099999999999 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065946+0000", 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017193699999999999 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068958+0000", 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030115939999999998 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069898+0000", 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093998699999999998 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203861+0000", 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.133963208000001 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203918+0000", 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6950999999999997e-05 2023-09-23T15:31:04.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.205245+0000", 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013277479999999999 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88039 9.d 9:b338c46d:::benchmark_data_smithi017_543597_object4248:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:31:04.503 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.98489334600001, 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90200742499999997, 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88039 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:31:04.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410779+0000", 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410784+0000", 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5480000000000001e-06 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410796+0000", 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1653e-05 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410807+0000", 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1181e-05 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410861+0000", 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3981999999999999e-05 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410880+0000", 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8553999999999999e-05 2023-09-23T15:31:04.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420827+0000", 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0099472469999999993 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420840+0000", 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2969999999999999e-05 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.302886+0000", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88204606500000005 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303061+0000", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175076 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312708+0000", 2023-09-23T15:31:04.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096470840000000002 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312787+0000", 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9296999999999997e-05 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.510 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88041 9.d 9:b6e8eb25:::benchmark_data_smithi017_543597_object4250:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.98485232900001, 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90221110000000004, 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88041 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410820+0000", 2023-09-23T15:31:04.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410824+0000", 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3720000000000001e-06 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410831+0000", 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410837+0000", 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.19e-06 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410931+0000", 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3634000000000005e-05 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:31:04.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410938+0000", 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422325+0000", 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011386744000000001 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422336+0000", 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1028e-05 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303400+0000", 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88106453799999995 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303535+0000", 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013442700000000001 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303696+0000", 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016155799999999999 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309737+0000", 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060407810000000003 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309784+0000", 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7237000000000002e-05 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312835+0000", 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030510619999999998 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312900+0000", 2023-09-23T15:31:04.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4393999999999994e-05 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313032+0000", 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013220300000000001 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.519 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88044 9.d 9:bfc8f2ba:::benchmark_data_smithi017_543597_object4253:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.98483191, 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90561986000000005, 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88044 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:31:04.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410843+0000", 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2079999999999999e-06 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410849+0000", 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6489999999999996e-06 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410854+0000", 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9810000000000003e-06 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410963+0000", 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109166 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410970+0000", 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5649999999999998e-06 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422375+0000", 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01140536 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422395+0000", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9913000000000001e-05 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306676+0000", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88428063000000001 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306827+0000", 2023-09-23T15:31:04.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015183300000000001 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306965+0000", 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137427 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313079+0000", 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061140869999999998 2023-09-23T15:31:04.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313118+0000", 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316281+0000", 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031639160000000001 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316328+0000", 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6897000000000002e-05 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316461+0000", 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013290900000000001 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88045 9.d 9:b617d99b:::benchmark_data_smithi017_543597_object4254:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.984815008, 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91186458299999995, 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.529 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88045 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410860+0000", 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5600000000000001e-06 2023-09-23T15:31:04.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410866+0000", 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6860000000000003e-06 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410871+0000", 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9980000000000003e-06 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410993+0000", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122359 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411000+0000", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0950000000000001e-06 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422421+0000", 2023-09-23T15:31:04.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011421093 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422430+0000", 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309805+0000", 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88737535099999998 2023-09-23T15:31:04.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309933+0000", 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012809900000000001 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310023+0000", 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9247999999999997e-05 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313160+0000", 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031370759999999999 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316471+0000", 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003311278 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322477+0000", 2023-09-23T15:31:04.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060065279999999997 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322536+0000", 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8792000000000002e-05 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322723+0000", 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018669199999999999 2023-09-23T15:31:04.538 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88046 9.3 9:c0e328f8:::benchmark_data_smithi017_543597_object4255:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.98479822799999, 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89982923699999995, 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.539 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88046 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:31:04.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410877+0000", 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4669999999999998e-06 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410885+0000", 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.841e-06 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410890+0000", 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411148+0000", 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025797 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411157+0000", 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5380000000000002e-06 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413770+0000", 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026130089999999999 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413782+0000", 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.289e-05 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303621+0000", 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88983906499999998 2023-09-23T15:31:04.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303734+0000", 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011293099999999999 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303859+0000", 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012465600000000001 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310319+0000", 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006459752 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310340+0000", 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1526999999999999e-05 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310470+0000", 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012976200000000001 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310519+0000", 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8906999999999998e-05 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310704+0000", 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018528199999999999 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.547 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88047 9.3 9:c39718c3:::benchmark_data_smithi017_543597_object4256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 202.984770728, 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90486523799999996, 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.548 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88047 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410902+0000", 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410905+0000", 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6520000000000002e-06 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410911+0000", 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410917+0000", 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5759999999999996e-06 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411181+0000", 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026399799999999999 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411187+0000", 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6200000000000001e-06 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413808+0000", 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026202399999999998 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413816+0000", 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2530000000000004e-06 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307084+0000", 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89326843099999997 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307210+0000", 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125623 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307340+0000", 2023-09-23T15:31:04.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013041499999999999 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.311327+0000", 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039864119999999999 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313609+0000", 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002282315 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315515+0000", 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019060640000000001 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315579+0000", 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3367999999999994e-05 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315768+0000", 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018929 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18484 10.2 10:46869aa5:::benchmark_data_smithi017_604517_object18483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 60.325440219000001, 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40252255100000001, 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.557 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18484 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070231+0000", 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070257+0000", 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6016e-05 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070274+0000", 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6682999999999999e-05 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070283+0000", 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0580000000000004e-06 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.560 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070333+0000", 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0442999999999998e-05 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070465+0000", 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013140100000000001 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070591+0000", 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126329 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072125+0000", 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001533855 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077175+0000", 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050495369999999998 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472642+0000", 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39546752299999999 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472704+0000", 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2088999999999995e-05 2023-09-23T15:31:04.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472756+0000", 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1493999999999999e-05 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.564 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18486 10.2 10:43afa650:::benchmark_data_smithi017_604517_object18485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 60.325013331999997, 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40280343299999999, 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18486 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.565 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070659+0000", 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070685+0000", 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5811000000000001e-05 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:04.567 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070697+0000", 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1596999999999999e-05 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070705+0000", 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1359999999999997e-06 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071435+0000", 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072993699999999995 2023-09-23T15:31:04.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071545+0000", 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011006999999999999 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071635+0000", 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0004000000000001e-05 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072295+0000", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00066030600000000004 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077270+0000", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049749349999999998 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473366+0000", 2023-09-23T15:31:04.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39609567099999998 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473429+0000", 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3528999999999994e-05 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473463+0000", 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4245999999999997e-05 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.571 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.572 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.572 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:04.572 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:04.572 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:31:04.572 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:04.573 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:04.573 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:04.573 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:04.618 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:04.657 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:04.687 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:04.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:04.735 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.736 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.737 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.737 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:31:04.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:04 smithi017 ceph-mon[215646]: pgmap v1256: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 636 op/s 2023-09-23T15:31:04.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:04 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4133602359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:04 smithi017 ceph-mon[222878]: pgmap v1256: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 636 op/s 2023-09-23T15:31:04.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:04 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4133602359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:04 smithi027 ceph-mon[195233]: pgmap v1256: 119 pgs: 119 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 40 MiB/s wr, 636 op/s 2023-09-23T15:31:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:04 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4133602359' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:05.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 68 16 89027 89011 81.7602 46 0.0213514 0.0122254 2023-09-23T15:31:05.362 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:05.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:05.469 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:05.535 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:06.181 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:31:06.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 69 16 89906 89890 81.3716 54.9375 0.01317 0.0122842 2023-09-23T15:31:06.446 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:06.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:06.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:06.508 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:06.564 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:06.615 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:06.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:06 smithi017 ceph-mon[222878]: pgmap v1257: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T15:31:06.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:06 smithi017 ceph-mon[215646]: pgmap v1257: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T15:31:06.753 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:06.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:06.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:06.828 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:06.883 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:06.934 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:06 smithi027 ceph-mon[195233]: pgmap v1257: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T15:31:06.938 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:07.074 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:07.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:07.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:07.184 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:07.262 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:07.308 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 70 16 90804 90788 81.011 56.125 0.0200336 0.0123394 2023-09-23T15:31:07.512 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:07.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:07.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:07.637 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:31:07.693 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:07.722 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:07.767 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:07.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:07.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:07.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:07.934 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:07.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:07.945 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:08.105 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:08.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:08] "GET /metrics HTTP/1.1" 200 36420 "" "Prometheus/2.43.0" 2023-09-23T15:31:08.249 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:08.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:08.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:08.311 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 71 16 91576 91560 80.5466 48.25 0.018002 0.0124109 2023-09-23T15:31:08.357 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:08.364 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:31:08.428 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:08.509 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:08.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:08.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:08.619 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: pgmap v1258: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:08.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[222878]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: pgmap v1258: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:08.678 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:08 smithi027 ceph-mon[195233]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T15:31:08.681 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:08.798 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:08.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:08.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: pgmap v1258: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: failed to encode map e428 with expected crc 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3335905070' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:08.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:08 smithi017 ceph-mon[215646]: osdmap e428: 8 total, 8 up, 8 in 2023-09-23T15:31:09.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:09.151 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:09.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:09.261 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:09.315 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 72 16 92520 92504 80.2427 59 0.0136969 0.0124569 2023-09-23T15:31:09.343 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:09.355 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:09.463 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:09.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:09.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:09.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:09.657 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:09.794 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:09.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:09.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:09.905 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:09.967 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:10.084 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:10.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:10.087 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:10.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:10.228 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:10.274 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:10.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:10.277 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:10.315 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 73 16 93340 93324 79.8456 51.25 0.00736095 0.0125017 2023-09-23T15:31:10.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:10.410 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:10.453 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:10.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:10.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:10.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:10.586 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:10.643 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:31:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:10 smithi027 ceph-mon[195233]: pgmap v1260: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-09-23T15:31:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:10 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1374869463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.902 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:10.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[215646]: pgmap v1260: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-09-23T15:31:10.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1374869463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.904 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[222878]: pgmap v1260: 119 pgs: 119 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s 2023-09-23T15:31:10.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1374869463' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:31:10.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:10.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:10.968 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:31:11.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:11.099 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:11.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 74 16 94033 94017 79.3507 43.3125 0.0097556 0.0125975 2023-09-23T15:31:11.471 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:11.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:11.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:31:11.594 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:11.681 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:11 smithi027 ceph-mon[195233]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.682 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:31:11.682 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:11 smithi027 ceph-mon[195233]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T15:31:11.682 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:11 smithi027 ceph-mon[195233]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.698 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:11.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[215646]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:31:11.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[215646]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T15:31:11.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[215646]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.763 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[222878]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:31:11.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[222878]: osdmap e429: 8 total, 8 up, 8 in 2023-09-23T15:31:11.764 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:11 smithi017 ceph-mon[222878]: failed to encode map e429 with expected crc 2023-09-23T15:31:11.904 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:11.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:11.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:12.011 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:12.020 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:12.035 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:12.077 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:12.078 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:12.086 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:31:12.086 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:31:12.116 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:12.158 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:12.199 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:12.243 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:12.282 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:12.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 75 16 94862 94846 78.9837 51.8125 0.0097848 0.0126554 2023-09-23T15:31:12.319 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:12.320 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:12.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.324 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.325 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:31:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:12.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:12.434 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:12.523 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:12 smithi017 ceph-mon[215646]: pgmap v1262: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T15:31:12.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2985519248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:12 smithi017 ceph-mon[222878]: pgmap v1262: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T15:31:12.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2985519248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:12 smithi027 ceph-mon[195233]: pgmap v1262: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2023-09-23T15:31:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2985519248' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:13.142 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:13.167 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:31:13.317 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 76 16 95775 95759 78.6953 57.0625 0.0152816 0.012702 2023-09-23T15:31:13.437 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:13.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:13.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:13.542 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:13.569 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:13.733 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:13.997 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:14.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:14.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:14.105 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:14.198 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:14.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 77 16 96617 96601 78.3565 52.625 0.010994 0.0127572 2023-09-23T15:31:14.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:14 smithi027 ceph-mon[195233]: pgmap v1263: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s 2023-09-23T15:31:14.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:14.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:14.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[222878]: pgmap v1263: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s 2023-09-23T15:31:14.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:14.362 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[215646]: pgmap v1263: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 35 MiB/s wr, 565 op/s 2023-09-23T15:31:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:14.363 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:14.408 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:14.415 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:14.516 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:14.686 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:15.261 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:15.269 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:15.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 78 16 97345 97329 77.9355 45.5 0.0293179 0.012826 2023-09-23T15:31:15.370 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:15.450 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:15.590 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:15.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:15.597 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:15.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:15.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:31:15.759 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:15.862 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:31:15.862 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:31:15.862 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:31:15.862 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:31:15.862 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:31:15.994 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:15.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:15.999 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:16.100 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:16.164 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:16.235 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:16.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:16.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:16.247 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:31:16.266 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[222878]: pgmap v1264: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T15:31:16.267 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/193305340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.267 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[215646]: pgmap v1264: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/193305340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.267 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 79 16 98109 98093 77.5536 47.75 0.00462026 0.0128777 2023-09-23T15:31:16.341 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:16.370 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:16.528 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:16.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:16 smithi027 ceph-mon[195233]: pgmap v1264: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2023-09-23T15:31:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/193305340' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:16.633 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:16.661 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:16.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:31:16.834 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:16.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:16.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:16.941 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:17.025 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:17.126 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:17.145 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:17.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:17.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:17.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:17.274 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:31:17.305 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:17.305 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.305 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:17.306 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T15:31:17.306 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.306 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:31:17.317 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:17.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:2023-09-23T15:31:17.318716+0000 min lat: 0.00117915 max lat: 0.408537 avg lat: 0.0129567 2023-09-23T15:31:17.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T15:31:17.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 80 16 98831 98815 77.1484 45.125 0.00703476 0.0129567 2023-09-23T15:31:17.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:17.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.428 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: osdmap e430: 8 total, 8 up, 8 in 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: failed to encode map e430 with expected crc 2023-09-23T15:31:17.429 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:31:17.462 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:17.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:17.472 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:17.521 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:17.573 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:17.628 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:18.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:18] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-09-23T15:31:18.255 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:18.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:18.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:18.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 81 16 99622 99606 76.8063 49.4375 0.0126535 0.0130147 2023-09-23T15:31:18.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:18.427 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: pgmap v1266: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T15:31:18.527 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[215646]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: pgmap v1266: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T15:31:18.528 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:18 smithi017 ceph-mon[222878]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.543 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:31:18.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: pgmap v1266: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s 2023-09-23T15:31:18.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/488402233' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:31:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: osdmap e431: 8 total, 8 up, 8 in 2023-09-23T15:31:18.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:18 smithi027 ceph-mon[195233]: failed to encode map e431 with expected crc 2023-09-23T15:31:18.783 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:18.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:18.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:18.898 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:18.977 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:19.107 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:31:19.166 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:19.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:19.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:19.286 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:19.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 82 16 100253 100237 76.3508 39.4375 0.0290496 0.0130929 2023-09-23T15:31:19.385 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:19.415 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:19.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:19.610 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:19.621 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:19.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:19.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:19.656 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:19.696 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:19.732 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:19.747 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:19.770 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:31:19.770 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:31:19.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:31:19.770 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:74153 8.9 8.9a176d39 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:25.320622+0000", 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 534.44555586800004, 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97471215600000005, 2023-09-23T15:31:19.771 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 74153 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320622+0000", 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320622+0000", 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320620+0000", 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320787+0000", 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016673299999999999 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320806+0000", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8957000000000002e-05 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320814+0000", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9370000000000004e-06 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.323290+0000", 2023-09-23T15:31:19.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024764600000000002 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.323299+0000", 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4320000000000005e-06 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295322+0000", 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97202277500000001 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295334+0000", 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1421e-05 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:74156 8.9 8.e95ccc99 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:25.320828+0000", 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 534.44534891299998, 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97453406099999995, 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.777 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 74156 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320828+0000", 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320828+0000", 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320828+0000", 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320877+0000", 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8949999999999997e-05 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320887+0000", 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0222e-05 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.320892+0000", 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9069999999999998e-06 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.323317+0000", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024249639999999999 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.323321+0000", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9009999999999996e-06 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295355+0000", 2023-09-23T15:31:19.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97203376100000005 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295363+0000", 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8280000000000003e-06 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:74170 8.9 8.5e1d0499 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e304)", 2023-09-23T15:31:19.782 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:22:25.336012+0000", 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 534.43016548499997, 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95937836600000004, 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 74170 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336012+0000", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336012+0000", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336010+0000", 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336052+0000", 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1125999999999999e-05 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336066+0000", 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4290999999999999e-05 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336074+0000", 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4200000000000007e-06 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336146+0000", 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1672000000000001e-05 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:25.336157+0000", 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0855e-05 2023-09-23T15:31:19.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295384+0000", 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95922692300000001 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:22:26.295390+0000", 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5719999999999999e-06 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.787 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83113 9.1 9:84c8439c:::benchmark_data_smithi017_543597_object83112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 263.873361929, 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067041744, 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83113 2023-09-23T15:31:19.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892814+0000", 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892915+0000", 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101517 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892934+0000", 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8862999999999999e-05 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892943+0000", 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892994+0000", 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1109000000000001e-05 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893124+0000", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013067299999999999 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893274+0000", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149534 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896402+0000", 2023-09-23T15:31:19.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031281579999999998 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906247+0000", 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098445379999999999 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906253+0000", 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7000000000000002e-06 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.947718+0000", 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414645709999999 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948393+0000", 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00067537000000000001 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959773+0000", 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01137928 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959857+0000", 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4637999999999998e-05 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18474 10.d 10:b2d26cad:::benchmark_data_smithi017_604517_object18473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:19.795 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 75.697713800000002, 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40456172899999998, 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18474 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068463+0000", 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068810+0000", 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034735199999999999 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068821+0000", 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0264e-05 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068827+0000", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.443e-06 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069014+0000", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018699400000000001 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069120+0000", 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010654000000000001 2023-09-23T15:31:19.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069251+0000", 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013087999999999999 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072104+0000", 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028525260000000002 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077080+0000", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049762950000000004 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472922+0000", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39584164300000002 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472980+0000", 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8023000000000003e-05 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473025+0000", 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5479000000000003e-05 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18476 10.d 10:b8c42a96:::benchmark_data_smithi017_604517_object18475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 75.697337227999995, 2023-09-23T15:31:19.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40577544799999998, 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18476 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068872+0000", 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2694000000000002e-05 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068881+0000", 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.371e-06 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068885+0000", 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3229999999999999e-06 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070067+0000", 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011816400000000001 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070195+0000", 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012812699999999999 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070349+0000", 2023-09-23T15:31:19.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015427600000000001 2023-09-23T15:31:19.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072111+0000", 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017617259999999999 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077110+0000", 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004998794 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474512+0000", 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39740209999999998 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474569+0000", 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7841999999999999e-05 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474616+0000", 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6134000000000002e-05 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18477 10.a 10:55f810cd:::benchmark_data_smithi017_604517_object18476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 75.697287533999997, 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.403843171, 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18477 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.812 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068889+0000", 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068920+0000", 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0133000000000001e-05 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068927+0000", 2023-09-23T15:31:19.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5780000000000001e-06 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068932+0000", 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7840000000000003e-06 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072819+0000", 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003887129 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072970+0000", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015059700000000001 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.073147+0000", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017778 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.075739+0000", 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025920600000000002 2023-09-23T15:31:19.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077221+0000", 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014813630000000001 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472627+0000", 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39540625000000001 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472672+0000", 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5337999999999999e-05 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472733+0000", 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0619999999999999e-05 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18488 10.a 10:56bb30e6:::benchmark_data_smithi017_604517_object18487:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.072687+0000", 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 75.693490453999999, 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.400841591, 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18488 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072687+0000", 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072687+0000", 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072686+0000", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072737+0000", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0914000000000001e-05 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072748+0000", 2023-09-23T15:31:19.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.152e-05 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072755+0000", 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3989999999999998e-06 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.074544+0000", 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017881660000000001 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.074694+0000", 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015071 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.074863+0000", 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016869399999999999 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.075880+0000", 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0010174069999999999 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077446+0000", 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001565604 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473434+0000", 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.395988322 2023-09-23T15:31:19.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473489+0000", 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4868999999999999e-05 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473529+0000", 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9276999999999997e-05 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18491 10.d 10:b7c495b1:::benchmark_data_smithi017_604517_object18490:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.078079+0000", 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 75.688097986000002, 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39667910000000001, 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18491 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.828 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078079+0000", 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078079+0000", 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078079+0000", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078142+0000", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3625000000000002e-05 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078154+0000", 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1192000000000001e-05 2023-09-23T15:31:19.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078158+0000", 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1110000000000001e-06 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078199+0000", 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1387000000000002e-05 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078311+0000", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111847 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.078395+0000", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4103000000000004e-05 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.079331+0000", 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00093612299999999997 2023-09-23T15:31:19.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.079718+0000", 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00038737999999999999 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474673+0000", 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.394954213 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474720+0000", 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6869000000000001e-05 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474759+0000", 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8951e-05 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.834 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:65609 10.b 10:db5763ca:::benchmark_data_smithi017_604517_object65608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:40.064987+0000", 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.701190355999998, 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28653292299999999, 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 65609 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.064987+0000", 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.064987+0000", 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.064987+0000", 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.065017+0000", 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0090000000000002e-05 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.065025+0000", 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4190000000000002e-06 2023-09-23T15:31:19.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.065030+0000", 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.429e-06 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.067190+0000", 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0021597349999999999 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.067264+0000", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3632000000000002e-05 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,1", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.067367+0000", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102954 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.069188+0000", 2023-09-23T15:31:19.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00182128 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351345+0000", 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28215734799999997 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351429+0000", 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3796999999999997e-05 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351478+0000", 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8888999999999999e-05 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351520+0000", 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1875999999999997e-05 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:65610 10.d 10:b34a35f7:::benchmark_data_smithi017_604517_object65609:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:40.066407+0000", 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.699770256000001, 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28508882400000002, 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 65610 2023-09-23T15:31:19.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066407+0000", 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066407+0000", 2023-09-23T15:31:19.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066395+0000", 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999876 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066459+0000", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4837000000000003e-05 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066470+0000", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0909e-05 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066487+0000", 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6648000000000001e-05 2023-09-23T15:31:19.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066532+0000", 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5132000000000001e-05 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066638+0000", 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106401 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066759+0000", 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121012 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.070751+0000", 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039913750000000001 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.072037+0000", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0012863670000000001 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351401+0000", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.279364004 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351453+0000", 2023-09-23T15:31:19.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1749999999999997e-05 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351496+0000", 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2997000000000002e-05 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:65612 10.d 10:bac3c2d7:::benchmark_data_smithi017_604517_object65611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T15:31:19.849 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:40.066501+0000", 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.699675962000001, 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.285126993, 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 65612 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066501+0000", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066501+0000", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066501+0000", 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066530+0000", 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9473999999999999e-05 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066538+0000", 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3509999999999996e-06 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.066545+0000", 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0169999999999999e-06 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.068727+0000", 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002182668 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.068801+0000", 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3369999999999997e-05 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.068892+0000", 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.0997999999999998e-05 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.071050+0000", 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002158381 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.072559+0000", 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0015086209999999999 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351546+0000", 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27898689100000001 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351590+0000", 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4227999999999997e-05 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351628+0000", 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8544999999999998e-05 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:65617 10.0 10:032a47ef:::benchmark_data_smithi017_604517_object65616:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T15:31:19.856 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:40.074405+0000", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.691772174, 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.29045954299999999, 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 65617 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074405+0000", 2023-09-23T15:31:19.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074405+0000", 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074404+0000", 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074458+0000", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4129000000000003e-05 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074468+0000", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9280000000000005e-06 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074472+0000", 2023-09-23T15:31:19.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.634e-06 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351308+0000", 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.27683602299999999 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351445+0000", 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013631 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351576+0000", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013143399999999999 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.360986+0000", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094094529999999999 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.361028+0000", 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2797999999999999e-05 2023-09-23T15:31:19.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364773+0000", 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037444039999999998 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364826+0000", 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3284e-05 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364865+0000", 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8767999999999999e-05 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.863 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:65620 10.0 10:02dc4502:::benchmark_data_smithi017_604517_object65619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e423)", 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:40.074486+0000", 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 39.691691871000003, 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.290505454, 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.864 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 65620 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074486+0000", 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074486+0000", 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074485+0000", 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.074516+0000", 2023-09-23T15:31:19.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.057e-05 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351283+0000", 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.276766927 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.351296+0000", 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3134e-05 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.355033+0000", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037369769999999998 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.355153+0000", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012046599999999999 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.355300+0000", 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014701499999999999 2023-09-23T15:31:19.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.361034+0000", 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057339959999999999 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364734+0000", 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0037000029999999999 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364906+0000", 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017172699999999999 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364954+0000", 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8656999999999998e-05 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:40.364991+0000", 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6489999999999998e-05 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99720 10.0 10:080bb5d7:::benchmark_data_smithi017_604517_object99719:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.412100+0000", 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.354077041, 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.329910869, 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.872 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99720 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412100+0000", 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412100+0000", 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412099+0000", 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412177+0000", 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7717000000000001e-05 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412187+0000", 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.764e-06 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412192+0000", 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1850000000000003e-06 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412281+0000", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9340999999999997e-05 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412360+0000", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8604999999999997e-05 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412439+0000", 2023-09-23T15:31:19.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9258000000000006e-05 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.418478+0000", 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060392290000000001 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.452339+0000", 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.033861038000000003 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.877 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.741904+0000", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28956490800000001 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.741963+0000", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8765999999999999e-05 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742011+0000", 2023-09-23T15:31:19.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.816e-05 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99721 10.0 10:0b47a113:::benchmark_data_smithi017_604517_object99720:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.412210+0000", 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.353967892, 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.32995765199999999, 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.879 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99721 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412210+0000", 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412210+0000", 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412209+0000", 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412241+0000", 2023-09-23T15:31:19.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1767000000000003e-05 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412248+0000", 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.379e-06 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.412252+0000", 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.523e-06 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.415425+0000", 2023-09-23T15:31:19.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003172954 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.415558+0000", 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000132924 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.415670+0000", 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011281399999999999 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.886 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.418486+0000", 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002815433 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.452365+0000", 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.033879483000000002 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742075+0000", 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.289709834 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742120+0000", 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5374000000000003e-05 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742167+0000", 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6678999999999998e-05 2023-09-23T15:31:19.888 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99733 10.0 10:0e2c4160:::benchmark_data_smithi017_604517_object99732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.454453+0000", 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.311724608, 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28783792400000002, 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.889 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99733 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454453+0000", 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454453+0000", 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454451+0000", 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454493+0000", 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1878000000000001e-05 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454506+0000", 2023-09-23T15:31:19.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3022e-05 2023-09-23T15:31:19.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454513+0000", 2023-09-23T15:31:19.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.832e-06 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454647+0000", 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013423999999999999 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454805+0000", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015778999999999999 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.454934+0000", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012915 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.461734+0000", 2023-09-23T15:31:19.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067994170000000003 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.741860+0000", 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.280126404 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742212+0000", 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035185300000000001 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742254+0000", 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1814999999999999e-05 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742291+0000", 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6844000000000002e-05 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99735 10.0 10:06177202:::benchmark_data_smithi017_604517_object99734:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.456646+0000", 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.3095311789999999, 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28594494799999998, 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99735 2023-09-23T15:31:19.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456646+0000", 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456646+0000", 2023-09-23T15:31:19.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456645+0000", 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456682+0000", 2023-09-23T15:31:19.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7410999999999998e-05 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456695+0000", 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2223e-05 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.456701+0000", 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6139999999999997e-06 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.458128+0000", 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014264900000000001 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.458261+0000", 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013347399999999999 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,4", 2023-09-23T15:31:19.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.458401+0000", 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013951100000000001 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.461953+0000", 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003552047 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742353+0000", 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28040041599999999 2023-09-23T15:31:19.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742462+0000", 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010929000000000001 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742531+0000", 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8511999999999997e-05 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.742591+0000", 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0191e-05 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99742 10.d 10:bab8fbe7:::benchmark_data_smithi017_604517_object99741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.464013+0000", 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.3021641509999999, 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28499790699999999, 2023-09-23T15:31:19.905 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99742 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464013+0000", 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464013+0000", 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464012+0000", 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464284+0000", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027150500000000001 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464293+0000", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5810000000000003e-06 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464299+0000", 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6590000000000004e-06 2023-09-23T15:31:19.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464358+0000", 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9502000000000001e-05 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464471+0000", 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011289399999999999 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.464601+0000", 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129913 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.748885+0000", 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28428338800000003 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.748892+0000", 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8549999999999996e-06 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.748922+0000", 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0843999999999998e-05 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.748971+0000", 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8296e-05 2023-09-23T15:31:19.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.749011+0000", 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0497999999999997e-05 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:99749 10.d 10:b9b29d0d:::benchmark_data_smithi017_604517_object99748:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e431)", 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:31:18.474022+0000", 2023-09-23T15:31:19.912 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 1.2921555069999999, 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28640416299999999, 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 99749 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.474022+0000", 2023-09-23T15:31:19.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.474022+0000", 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.474020+0000", 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:19.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.474079+0000", 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8449000000000002e-05 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.741700+0000", 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.26762187599999998 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:19.915 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.741736+0000", 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5666e-05 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.745155+0000", 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034187509999999998 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.745281+0000", 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000125866 2023-09-23T15:31:19.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.745422+0000", 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014122700000000001 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.756154+0000", 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010732031 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.756174+0000", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0557e-05 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.760317+0000", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0041425259999999997 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.760379+0000", 2023-09-23T15:31:19.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1686999999999994e-05 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:31:18.760426+0000", 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7364000000000003e-05 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:19.919 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:31:19.920 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:19.921 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:19.921 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:19.921 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:19.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.921 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:31:19.922 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:31:20.131 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:20.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:20.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:20.237 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:20.270 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:20.318 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 83 16 100906 100890 75.9228 40.8125 0.00908956 0.0131659 2023-09-23T15:31:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:20 smithi017 ceph-mon[215646]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s 2023-09-23T15:31:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:20 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2509998999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:20 smithi017 ceph-mon[222878]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s 2023-09-23T15:31:20.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:20 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2509998999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:20.584 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:20.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:20 smithi027 ceph-mon[195233]: pgmap v1268: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 36 MiB/s wr, 574 op/s 2023-09-23T15:31:20.588 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:20 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2509998999' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:20.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:20.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:20.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:31:20.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:20.746 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:21.247 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:21.325 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 84 16 101677 101661 75.5872 48.1875 0.010058 0.0132251 2023-09-23T15:31:21.373 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:21.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:21.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:21.485 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:21.499 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:21.567 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:21.692 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:21.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:21.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:21.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:21.904 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:22.018 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:22.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:22.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:22.122 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:22.198 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:22.324 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 85 16 102458 102442 75.2723 48.8125 0.00440292 0.0132772 2023-09-23T15:31:22.353 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:22.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:22.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:22 smithi017 ceph-mon[215646]: pgmap v1269: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T15:31:22.443 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:22 smithi017 ceph-mon[222878]: pgmap v1269: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T15:31:22.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:22.499 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:22.547 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:22.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:22.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:22.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:22 smithi027 ceph-mon[195233]: pgmap v1269: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2023-09-23T15:31:22.650 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:22.719 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:23.071 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:23.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:23.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:23.178 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:23.262 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:23.265 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:23 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:31:23.265 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:23 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:31:23.289 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:31:23.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 86 16 103158 103142 74.9046 43.75 0.00885446 0.013345 2023-09-23T15:31:23.399 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:23.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:23.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:23.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:31:23.508 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:31:23.509 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:31:23.509 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:23.540 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:23 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:31:23.574 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:23.777 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:23.784 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:23.885 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:23.912 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:23.973 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:23.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:23.978 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:24.078 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:24.105 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:24.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 87 16 103764 103748 74.4791 37.875 0.00830342 0.0134227 2023-09-23T15:31:24.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:31:24.390 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:24.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:24.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:24.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:31:24.498 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: pgmap v1270: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1067612024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.531 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:24 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.587 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:24.592 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:24.604 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: pgmap v1270: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.633 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1067612024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: pgmap v1270: 119 pgs: 119 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 30 MiB/s wr, 476 op/s 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3265179908' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: osdmap e432: 8 total, 8 up, 8 in 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.634 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: failed to encode map e432 with expected crc 2023-09-23T15:31:24.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1067612024' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:24 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:31:24.686 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:24.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:24.700 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:24.801 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:24.898 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:25.064 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:25.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:25.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:25.165 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:25.176 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:25.264 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:25.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 88 16 104597 104581 74.2245 52.0625 0.00816712 0.0134677 2023-09-23T15:31:25.371 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:25.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:25.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:25.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:25 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:25 smithi027 ceph-mon[195233]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:25 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:25 smithi027 ceph-mon[195233]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:25 smithi027 ceph-mon[195233]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.530 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[215646]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[215646]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[215646]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[222878]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[222878]: osdmap e433: 8 total, 8 up, 8 in 2023-09-23T15:31:25.531 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:25 smithi017 ceph-mon[222878]: failed to encode map e433 with expected crc 2023-09-23T15:31:25.537 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:25.661 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:25.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:25.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:25.768 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:25.821 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:25.903 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:25.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:25.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:26.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:26.072 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:26.148 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:26.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:26.251 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:26.279 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:26.326 INFO:tasks.radosbench.radosbench.0.smithi017.stdout: 89 16 105306 105290 73.8886 44.3125 0.20077 0.0135163 2023-09-23T15:31:26.495 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:26.523 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:26.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:26.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:26 smithi017 ceph-mon[222878]: failed to encode map e433 with expected crc 2023-09-23T15:31:26.543 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:26 smithi017 ceph-mon[222878]: pgmap v1273: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-09-23T15:31:26.552 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:26 smithi017 ceph-mon[215646]: failed to encode map e433 with expected crc 2023-09-23T15:31:26.553 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:26 smithi017 ceph-mon[215646]: pgmap v1273: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-09-23T15:31:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:26 smithi027 ceph-mon[195233]: failed to encode map e433 with expected crc 2023-09-23T15:31:26.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:26 smithi027 ceph-mon[195233]: pgmap v1273: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2023-09-23T15:31:26.644 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:26.734 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:26.844 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:26.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:26.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:26.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:26.996 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:27.137 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:27.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:27.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:27.244 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:27.285 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total time run: 90.0114 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Total writes made: 106108 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Write size: 65536 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Object size: 65536 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Bandwidth (MB/sec): 73.6768 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Bandwidth: 34.8399 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max bandwidth (MB/sec): 208.812 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min bandwidth (MB/sec): 37.875 2023-09-23T15:31:27.327 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average IOPS: 1178 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev IOPS: 557.438 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max IOPS: 3341 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min IOPS: 606 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Average Latency(s): 0.0135689 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Stddev Latency(s): 0.0194544 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Max latency(s): 0.408537 2023-09-23T15:31:27.328 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Min latency(s): 0.00117915 2023-09-23T15:31:27.334 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T15:31:27.347 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:27.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:27.355 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:27.370 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:31:27.370 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:31:27.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:27.373 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:27.380 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:27.418 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:27.456 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:27.459 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:27.502 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:27.530 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:27.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:27.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:31:27.615 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:31:27.655 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:27.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:27.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:27.760 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:27.787 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:27.831 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:27.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:27.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:28.001 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:28.283 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:31:28.283 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:28] "GET /metrics HTTP/1.1" 200 36423 "" "Prometheus/2.43.0" 2023-09-23T15:31:28.592 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:28.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[215646]: pgmap v1274: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T15:31:28.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/971572944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:28.594 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:28.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:28.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:28.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[222878]: pgmap v1274: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T15:31:28.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/971572944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:28.597 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:28 smithi027 ceph-mon[195233]: pgmap v1274: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 52 MiB/s wr, 838 op/s 2023-09-23T15:31:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/971572944' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:28.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:28.697 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:28.769 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:28.885 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:28.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:28.915 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:31:28.931 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:29.004 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:29.087 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:29.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:29.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:29.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:29.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:29.498 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:29.616 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:29.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:29.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:29.752 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:29.797 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:29.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:29.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:29.901 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:29.929 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:29.973 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:29.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:29.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:30.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:30.137 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:30.320 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:31:30.528 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:30.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:30.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:30 smithi027 ceph-mon[195233]: pgmap v1275: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s 2023-09-23T15:31:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:30 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/703288630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.640 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:30.671 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:30.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[215646]: pgmap v1275: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s 2023-09-23T15:31:30.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/703288630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[222878]: pgmap v1275: 119 pgs: 119 active+clean; 6.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 28 MiB/s wr, 448 op/s 2023-09-23T15:31:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/703288630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:31:30.729 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:31:30.737 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:30.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:30.840 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:30.907 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:31.082 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:31.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:31.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.155 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88039 9.d 9.b6231ccd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.74231191999999, 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515619899999996, 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88039 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.156 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409929+0000", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409936+0000", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409948+0000", 2023-09-23T15:31:31.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1756999999999999e-05 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409957+0000", 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3381999999999998e-05 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410021+0000", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.101e-05 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405075+0000", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505363800000002 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405087+0000", 2023-09-23T15:31:31.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1863e-05 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88041 9.d 9.a4d7176d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.742271321, 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99514826899999997, 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.161 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88041 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409973+0000", 2023-09-23T15:31:31.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6620000000000001e-06 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409980+0000", 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7689999999999999e-06 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409985+0000", 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6720000000000003e-06 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410051+0000", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410058+0000", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9380000000000003e-06 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405110+0000", 2023-09-23T15:31:31.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505194299999999 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405119+0000", 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7710000000000001e-06 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88044 9.d 9.5d4f13fd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.166 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.742234814, 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515036099999998, 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88044 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410007+0000", 2023-09-23T15:31:31.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5220000000000002e-06 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410018+0000", 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6599999999999995e-06 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410023+0000", 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3789999999999997e-06 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410129+0000", 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010582300000000001 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410141+0000", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2234e-05 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405148+0000", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.995007164 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405158+0000", 2023-09-23T15:31:31.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9410000000000002e-06 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88045 9.d 9.d99be86d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.742211795, 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515891099999998, 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.172 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88045 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410030+0000", 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410033+0000", 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:31:31.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410039+0000", 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0480000000000004e-06 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410043+0000", 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4689999999999999e-06 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.175 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410178+0000", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134657 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410185+0000", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7519999999999999e-06 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405180+0000", 2023-09-23T15:31:31.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.994995097 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405190+0000", 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7569999999999999e-06 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88046 9.3 9.1f14c703 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:31:31.177 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.74219439300001, 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99103617399999999, 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88046 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:31.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410050+0000", 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8989999999999999e-06 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410055+0000", 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4700000000000001e-06 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410061+0000", 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7860000000000002e-06 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410100+0000", 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8804999999999999e-05 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410109+0000", 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8459999999999997e-06 2023-09-23T15:31:31.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401069+0000", 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99096060100000005 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401084+0000", 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5098e-05 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.182 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88047 9.3 9.c318e9c3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 230.74216635900001, 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99202290000000004, 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.183 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88047 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410078+0000", 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4839999999999998e-06 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410085+0000", 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7649999999999996e-06 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410090+0000", 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.899e-06 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410149+0000", 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9074999999999999e-05 2023-09-23T15:31:31.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410159+0000", 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0394000000000001e-05 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402086+0000", 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99192703699999996 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402099+0000", 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2700999999999999e-05 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118583 9.d 9:b22e1d7c:::benchmark_data_smithi017_543597_object34792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.787974851, 2023-09-23T15:31:31.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.944761042, 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118583 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:31:31.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364266+0000", 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364271+0000", 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0320000000000003e-06 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364281+0000", 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8560000000000002e-06 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364286+0000", 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3299999999999998e-06 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364340+0000", 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3464999999999997e-05 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364349+0000", 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1099999999999992e-06 2023-09-23T15:31:31.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298613+0000", 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93426368199999998 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298716+0000", 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010302999999999999 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298806+0000", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.056e-05 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305553+0000", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067469649999999997 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305618+0000", 2023-09-23T15:31:31.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5029000000000004e-05 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308931+0000", 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033124449999999998 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308986+0000", 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4809999999999999e-05 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309029+0000", 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3118000000000003e-05 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118587 9.d 9:b46673d5:::benchmark_data_smithi017_543597_object34796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.78140920600001, 2023-09-23T15:31:31.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93829474099999999, 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118587 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370832+0000", 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370837+0000", 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5280000000000003e-06 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370850+0000", 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2576e-05 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370856+0000", 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5419999999999996e-06 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370909+0000", 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3155999999999997e-05 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370915+0000", 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5300000000000002e-06 2023-09-23T15:31:31.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301114+0000", 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.930198525 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301253+0000", 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013897099999999999 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301389+0000", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135814 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305605+0000", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042168529999999996 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308882+0000", 2023-09-23T15:31:31.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032768129999999999 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309075+0000", 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000192296 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309105+0000", 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0505e-05 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309128+0000", 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2966999999999999e-05 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118593 9.d 9:b3ef2821:::benchmark_data_smithi017_543597_object34802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.77271069400001, 2023-09-23T15:31:31.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.934446412, 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118593 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:31:31.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379530+0000", 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379535+0000", 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4610000000000001e-06 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379545+0000", 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379550+0000", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2059999999999998e-06 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379683+0000", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013354299999999999 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379695+0000", 2023-09-23T15:31:31.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1833e-05 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303454+0000", 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92375848400000005 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303581+0000", 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127126 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303717+0000", 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136534 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305643+0000", 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019254490000000001 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308885+0000", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032424519999999998 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313893+0000", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050071999999999998 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313940+0000", 2023-09-23T15:31:31.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6974000000000001e-05 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313978+0000", 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8679e-05 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118617 9.d 9:b0bafb86:::benchmark_data_smithi017_543597_object34826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.212 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.75294656899999, 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91924088900000001, 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118617 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399295+0000", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399305+0000", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0543999999999999e-05 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399318+0000", 2023-09-23T15:31:31.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.325e-05 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399326+0000", 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9670000000000001e-06 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399399+0000", 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2344999999999999e-05 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399412+0000", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3570999999999999e-05 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305695+0000", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90628282500000001 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305769+0000", 2023-09-23T15:31:31.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3960000000000003e-05 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305840+0000", 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0525999999999995e-05 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309148+0000", 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033085340000000001 2023-09-23T15:31:31.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309185+0000", 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6806e-05 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318447+0000", 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092622610000000008 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318496+0000", 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8952999999999997e-05 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318537+0000", 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0771e-05 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.220 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118636 9.d 9:be5e9593:::benchmark_data_smithi017_543597_object34845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.728179958, 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89944873700000005, 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.221 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118636 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424061+0000", 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424066+0000", 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7740000000000004e-06 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424078+0000", 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2053e-05 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424084+0000", 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7379999999999999e-06 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424159+0000", 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5166999999999994e-05 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424170+0000", 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1090000000000001e-05 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307472+0000", 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88330215099999998 2023-09-23T15:31:31.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307554+0000", 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1878000000000004e-05 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307626+0000", 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1649000000000005e-05 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309193+0000", 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001566858 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313853+0000", 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046608609999999997 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323430+0000", 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095760589999999996 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323476+0000", 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6399000000000001e-05 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323511+0000", 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5370000000000002e-05 2023-09-23T15:31:31.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118669 9.d 9:b914bc60:::benchmark_data_smithi017_543597_object34878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.68699280999999, 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.85838970000000003, 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.229 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118669 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465248+0000", 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465255+0000", 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9920000000000001e-06 2023-09-23T15:31:31.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465273+0000", 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8357999999999999e-05 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465282+0000", 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465349+0000", 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6883000000000003e-05 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465360+0000", 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1181e-05 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309219+0000", 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.84385860999999995 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309293+0000", 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3689000000000005e-05 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309370+0000", 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7571999999999995e-05 2023-09-23T15:31:31.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313995+0000", 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004624548 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314027+0000", 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2759999999999998e-05 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323570+0000", 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095423469999999996 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323608+0000", 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8599000000000001e-05 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323639+0000", 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1059999999999997e-05 2023-09-23T15:31:31.236 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118722 9.d 9:bde41141:::benchmark_data_smithi017_543597_object34931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.62080631699999, 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80511723000000002, 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.237 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118722 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531435+0000", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531441+0000", 2023-09-23T15:31:31.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531458+0000", 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7099e-05 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531466+0000", 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3810000000000001e-06 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532385+0000", 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00091978700000000004 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532394+0000", 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311491+0000", 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77909717099999998 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311622+0000", 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013045400000000001 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311748+0000", 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012667700000000001 2023-09-23T15:31:31.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314036+0000", 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022880510000000001 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318410+0000", 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043731940000000004 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336468+0000", 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018058586000000001 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336516+0000", 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7856999999999999e-05 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336553+0000", 2023-09-23T15:31:31.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7351999999999997e-05 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118729 9.d 9:b8e345b2:::benchmark_data_smithi017_543597_object34938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.61448538900001, 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.799062563, 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.245 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118729 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537756+0000", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537763+0000", 2023-09-23T15:31:31.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0600000000000002e-06 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537777+0000", 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4175999999999999e-05 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537782+0000", 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537847+0000", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5366999999999999e-05 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537854+0000", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5139999999999998e-06 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314070+0000", 2023-09-23T15:31:31.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77621656999999999 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314182+0000", 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011181200000000001 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314270+0000", 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7640999999999997e-05 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323357+0000", 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.009087108 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323363+0000", 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3090000000000001e-06 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.251 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336742+0000", 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013378594000000001 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336784+0000", 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2333000000000003e-05 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336820+0000", 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5672e-05 2023-09-23T15:31:31.252 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118744 9.d 9:b54a00db:::benchmark_data_smithi017_543597_object34953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.59924072699999, 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.78398909900000002, 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.253 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118744 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:31:31.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553000+0000", 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553006+0000", 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4979999999999997e-06 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.258 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553019+0000", 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2855e-05 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553024+0000", 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.271e-06 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553069+0000", 2023-09-23T15:31:31.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.494e-05 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553077+0000", 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6000000000000001e-06 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.315967+0000", 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76288994200000004 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316067+0000", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010011 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316183+0000", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0001165 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323368+0000", 2023-09-23T15:31:31.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071846490000000004 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323401+0000", 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2699e-05 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336915+0000", 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013514884 2023-09-23T15:31:31.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336958+0000", 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2064000000000002e-05 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336991+0000", 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3380999999999998e-05 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118748 9.d 9:b5a57405:::benchmark_data_smithi017_543597_object34957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.59739597800001, 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79383968999999999, 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118748 2023-09-23T15:31:31.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554845+0000", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554853+0000", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7870000000000002e-06 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554866+0000", 2023-09-23T15:31:31.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3534e-05 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554874+0000", 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3470000000000001e-06 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554929+0000", 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5269999999999998e-05 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554938+0000", 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318565+0000", 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76362680500000002 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318662+0000", 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6918999999999998e-05 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318774+0000", 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112495 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323373+0000", 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045984939999999998 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323543+0000", 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017058199999999999 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348594+0000", 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025050833000000002 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348644+0000", 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0266000000000003e-05 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348686+0000", 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1925999999999998e-05 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.271 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118765 9.d 9:b25beba1:::benchmark_data_smithi017_543597_object34974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.57891707799999, 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77546686300000001, 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118765 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573324+0000", 2023-09-23T15:31:31.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573331+0000", 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4259999999999997e-06 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573342+0000", 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0403e-05 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.274 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573348+0000", 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1800000000000001e-06 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573421+0000", 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3468999999999997e-05 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.275 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573431+0000", 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5540000000000003e-06 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321037+0000", 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.747606306 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321179+0000", 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014166 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.276 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321251+0000", 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2272000000000002e-05 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323671+0000", 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024195829999999999 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.277 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327602+0000", 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039315790000000002 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348718+0000", 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021115821999999999 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348757+0000", 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9100999999999997e-05 2023-09-23T15:31:31.278 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348792+0000", 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4980000000000001e-05 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118776 9.d 9:b496d689:::benchmark_data_smithi017_543597_object34985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.56056777500001, 2023-09-23T15:31:31.279 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76259006200000001, 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118776 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.280 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591673+0000", 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.281 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591678+0000", 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2259999999999996e-06 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591693+0000", 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4647000000000001e-05 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.282 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591699+0000", 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4659999999999996e-06 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591758+0000", 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8990000000000003e-05 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591766+0000", 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1250000000000004e-06 2023-09-23T15:31:31.283 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323691+0000", 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.73192517599999996 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323786+0000", 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5377000000000005e-05 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.284 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323892+0000", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105882 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.332104+0000", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082115420000000005 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.333996+0000", 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018925509999999999 2023-09-23T15:31:31.285 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354164+0000", 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020168175 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354220+0000", 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5327000000000001e-05 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.286 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354265+0000", 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5158999999999999e-05 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118785 9.d 9:b48c076d:::benchmark_data_smithi017_543597_object34994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.547770181, 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74991449799999998, 2023-09-23T15:31:31.287 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118785 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.288 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604471+0000", 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.289 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604477+0000", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.806e-06 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604488+0000", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0818999999999999e-05 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604494+0000", 2023-09-23T15:31:31.290 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0739999999999998e-06 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604547+0000", 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3557999999999998e-05 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604557+0000", 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7880000000000001e-06 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.291 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325681+0000", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72112381699999994 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325761+0000", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0216000000000006e-05 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325859+0000", 2023-09-23T15:31:31.292 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8181000000000001e-05 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336439+0000", 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010580001 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336654+0000", 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000215183 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.293 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354304+0000", 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017649116999999999 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354353+0000", 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9005000000000003e-05 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.294 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354387+0000", 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4282999999999997e-05 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118797 9.d 9:b088cc93:::benchmark_data_smithi017_543597_object35006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 187.51966448300001, 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72191496499999996, 2023-09-23T15:31:31.295 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118797 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.296 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632576+0000", 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.297 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632584+0000", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4419999999999998e-06 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632601+0000", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7013000000000001e-05 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632610+0000", 2023-09-23T15:31:31.298 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632675+0000", 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4850000000000004e-05 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632689+0000", 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4800000000000001e-05 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.299 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327685+0000", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69499560800000004 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327806+0000", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012095999999999999 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327937+0000", 2023-09-23T15:31:31.300 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013145299999999999 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336643+0000", 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0087055399999999995 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336662+0000", 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.933e-05 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.301 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354421+0000", 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017759161999999998 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354461+0000", 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9267000000000002e-05 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:31:31.302 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354493+0000", 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2258999999999997e-05 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:31:31.303 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:31:31.303 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:31.305 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:31:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:31 smithi027 ceph-mon[195233]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:31 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:31:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:31 smithi027 ceph-mon[195233]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T15:31:31.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:31 smithi027 ceph-mon[195233]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:31 smithi027 ceph-mon[195233]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[222878]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:31:31.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[222878]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T15:31:31.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[222878]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.387 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[222878]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.391 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[215646]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:31:31.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[215646]: osdmap e434: 8 total, 8 up, 8 in 2023-09-23T15:31:31.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[215646]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.392 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:31 smithi017 ceph-mon[215646]: failed to encode map e434 with expected crc 2023-09-23T15:31:31.401 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:31.546 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:31.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:31.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:31.568 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:31.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:31.680 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:31.861 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:31.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:31.977 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:32.032 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:31:32.058 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:32.138 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:32.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:32.144 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:32.245 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:32.298 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:32.337 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:32 smithi017 ceph-mon[215646]: pgmap v1277: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 B/s rd, 48 MiB/s wr, 966 op/s 2023-09-23T15:31:32.338 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:32 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:32.376 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:32.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:32.380 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:32.380 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:31:32.481 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:32.508 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:32.553 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:32.556 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:32 smithi027 ceph-mon[195233]: pgmap v1277: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 B/s rd, 48 MiB/s wr, 966 op/s 2023-09-23T15:31:32.557 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:32 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:32.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:32.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:32.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:32 smithi017 ceph-mon[222878]: pgmap v1277: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 133 B/s rd, 48 MiB/s wr, 966 op/s 2023-09-23T15:31:32.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:32 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:32.707 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:32.794 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:32.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:32.805 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:32.906 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:32.960 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:33.050 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:33.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:33.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:33.164 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:33.191 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:33.234 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:33.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:33.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.332 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[215646]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.333 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:33 smithi017 ceph-mon[222878]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:33.355 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:33.469 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:31:33.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:31:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/616375659' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: osdmap e435: 8 total, 8 up, 8 in 2023-09-23T15:31:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:33 smithi027 ceph-mon[195233]: failed to encode map e435 with expected crc 2023-09-23T15:31:33.605 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:33.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:33.722 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:33.765 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:33.779 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:33.860 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:33.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:33.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:33.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:34.013 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:34.103 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:34.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:34.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:34.212 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:34.292 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:34.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:34 smithi017 ceph-mon[215646]: pgmap v1279: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 505 op/s 2023-09-23T15:31:34.550 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:34 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1905444107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:34.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:34 smithi017 ceph-mon[222878]: pgmap v1279: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 505 op/s 2023-09-23T15:31:34.550 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:34 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1905444107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:34.685 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:34.686 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:34.830 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:34 smithi027 ceph-mon[195233]: pgmap v1279: 119 pgs: 119 active+clean; 6.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 127 B/s rd, 19 MiB/s wr, 505 op/s 2023-09-23T15:31:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:34 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1905444107' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:34.880 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:34.925 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:34.974 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:35.031 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:35.076 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:35.079 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.079 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.080 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:31:35.322 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:35.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:35.330 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:35.430 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:35.465 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:35.624 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:31:35.655 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:35 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:31:35.656 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:35 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:31:35.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:35 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:31:36.442 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:36.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:36.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:36.548 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:36.641 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:36.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:36 smithi027 ceph-mon[195233]: pgmap v1280: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 900 op/s 2023-09-23T15:31:36.907 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:36.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:36 smithi017 ceph-mon[215646]: pgmap v1280: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 900 op/s 2023-09-23T15:31:36.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:36.915 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:36.919 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:36 smithi017 ceph-mon[222878]: pgmap v1280: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 900 op/s 2023-09-23T15:31:37.015 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:37.059 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:37.149 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:37.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:37.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:37.237 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:37.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:37.314 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:37.381 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:37.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:37.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:37.427 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:31:37.427 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:31:37.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:37.543 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:37.587 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:31:37.604 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:31:37.741 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:37.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:37.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:37.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:37.913 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:37.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:31:37.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:31:37.962 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:37.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:38.070 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:38.134 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:38.294 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:31:38.335 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:38] "GET /metrics HTTP/1.1" 200 36418 "" "Prometheus/2.43.0" 2023-09-23T15:31:38.342 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:38.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:38.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:38.459 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:38.556 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: pgmap v1281: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 927 op/s 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.598 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[215646]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.691 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: pgmap v1281: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 927 op/s 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:38 smithi027 ceph-mon[195233]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: pgmap v1281: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 127 B/s rd, 18 MiB/s wr, 927 op/s 2023-09-23T15:31:38.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2472276949' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:31:38.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: osdmap e436: 8 total, 8 up, 8 in 2023-09-23T15:31:38.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: failed to encode map e436 with expected crc 2023-09-23T15:31:38.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:38 smithi017 ceph-mon[222878]: failed to encode map e436 with expected crc 2023-09-23T15:31:39.648 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:39.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:39.755 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:39.838 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:40.094 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:31:40.098 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:40.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:40.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:40.211 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:40.260 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:40.340 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:40.357 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:40.457 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:40.539 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:40.686 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:40.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:40.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:40.752 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:40.761 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:40.792 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:40.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[215646]: pgmap v1283: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2023-09-23T15:31:40.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:40.868 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:40.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[222878]: pgmap v1283: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2023-09-23T15:31:40.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:40.869 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:40 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:40.882 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:41.099 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:40 smithi027 ceph-mon[195233]: pgmap v1283: 119 pgs: 119 active+clean; 6.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 449 op/s 2023-09-23T15:31:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:40 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:41.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:40 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:41.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:41.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:41.212 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:41.253 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:41.335 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:41.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:41.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:41.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:41.492 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:41.506 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:41.585 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:41.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:41.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:41.665 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:31:41.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:41.767 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3518721187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:41.767 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:41.767 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3518721187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:41.768 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:41.772 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3518721187' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:31:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: pgmap v1284: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T15:31:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: failed to encode map e437 with expected crc 2023-09-23T15:31:42.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: failed to encode map e437 with expected crc 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[215646]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: pgmap v1284: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: failed to encode map e437 with expected crc 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: failed to encode map e437 with expected crc 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:42.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:42 smithi017 ceph-mon[222878]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: pgmap v1284: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 832 op/s 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: failed to encode map e437 with expected crc 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: failed to encode map e437 with expected crc 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:31:43.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:42 smithi027 ceph-mon[195233]: osdmap e437: 8 total, 8 up, 8 in 2023-09-23T15:31:43.906 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:43.906 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:44.151 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:44.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:44.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:44.266 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:44.321 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:44.364 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:44.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:44.371 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:44.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:44.564 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:44.917 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:44.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:44.926 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:44 smithi017 ceph-mon[215646]: pgmap v1286: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:31:44.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:44.931 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:44 smithi017 ceph-mon[222878]: pgmap v1286: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:31:45.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:44 smithi027 ceph-mon[195233]: pgmap v1286: 119 pgs: 119 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 472 op/s 2023-09-23T15:31:45.105 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:45.600 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:45.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:45.624 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:31:45.624 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:31:45.672 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:45.710 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:45.711 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:45.754 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:45.755 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:45.792 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:45.839 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:45.846 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:45.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:45.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:45.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:45 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1723132700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:45.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:45 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1723132700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:45.882 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:45.908 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:31:45.935 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:45.956 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:45.986 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.988 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.989 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.989 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:31:45.995 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:45 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1723132700' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:45.997 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:46.074 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:46.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:46.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:46.180 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:46.274 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:46.809 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:46.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:46.919 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:31:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:46 smithi017 ceph-mon[215646]: pgmap v1287: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T15:31:46.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:46 smithi017 ceph-mon[222878]: pgmap v1287: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T15:31:46.944 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:46 smithi027 ceph-mon[195233]: pgmap v1287: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2023-09-23T15:31:46.947 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:47.012 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:31:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:47.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:31:47.125 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:47.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:47.295 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:47.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:47.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:47.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:47.449 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:47.514 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:31:47.522 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:47.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:47.627 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:47.671 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:47.739 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:47.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:47.762 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:47.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:47.928 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:48.122 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:48.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:48.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:48] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T15:31:48.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:48.306 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:48.441 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:48.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:48.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:48.563 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:48.624 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:48.705 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:31:48.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[222878]: pgmap v1288: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T15:31:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1348801698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:48.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[215646]: pgmap v1288: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T15:31:48.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1348801698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:48.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:49.045 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:49.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:49.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:49.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:48 smithi027 ceph-mon[195233]: pgmap v1288: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 787 op/s 2023-09-23T15:31:49.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1348801698' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:49.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:31:49.161 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:49.210 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:49.598 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:31:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[215646]: failed to encode map e438 with expected crc 2023-09-23T15:31:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:31:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[215646]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T15:31:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[215646]: failed to encode map e438 with expected crc 2023-09-23T15:31:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[222878]: failed to encode map e438 with expected crc 2023-09-23T15:31:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:31:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[222878]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T15:31:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:49 smithi017 ceph-mon[222878]: failed to encode map e438 with expected crc 2023-09-23T15:31:50.033 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:50.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:50.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:49 smithi027 ceph-mon[195233]: failed to encode map e438 with expected crc 2023-09-23T15:31:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:49 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:31:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:49 smithi027 ceph-mon[195233]: osdmap e438: 8 total, 8 up, 8 in 2023-09-23T15:31:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:49 smithi027 ceph-mon[195233]: failed to encode map e438 with expected crc 2023-09-23T15:31:50.139 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:50.237 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:50.453 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:31:50.453 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:31:50.609 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:50.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:50.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:50.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[215646]: pgmap v1290: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2023-09-23T15:31:50.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:50.725 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:50.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:50.832 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:50.945 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:50.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:51.017 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:51.058 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:50 smithi027 ceph-mon[195233]: pgmap v1290: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2023-09-23T15:31:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:51.129 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[222878]: pgmap v1290: 119 pgs: 119 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 477 op/s 2023-09-23T15:31:51.129 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:51.130 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:51.136 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:51.409 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:51.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:51.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:51.514 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:31:51.586 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:51.687 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:31:51.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:51.692 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:31:51.793 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:51.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:31:51.844 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:31:51.845 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:51.849 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:51.929 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:51.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:51.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:52.041 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:52.092 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:52.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:31:52.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:31:52.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:52.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:31:52.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:31:52.157 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:31:52.179 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:52.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:52.285 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:52.348 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:52.911 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:31:53.026 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[222878]: pgmap v1291: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2732314179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[215646]: pgmap v1291: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2732314179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.036 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:53.038 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:52 smithi027 ceph-mon[195233]: pgmap v1291: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2023-09-23T15:31:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2732314179' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:31:53.139 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:53.181 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:53.274 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:53.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:53.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:53.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:53.483 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T15:31:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:53 smithi027 ceph-mon[195233]: pgmap v1293: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T15:31:54.152 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[222878]: pgmap v1293: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T15:31:54.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:31:54.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: failed to encode map e439 with expected crc 2023-09-23T15:31:54.153 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:31:54.154 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: osdmap e439: 8 total, 8 up, 8 in 2023-09-23T15:31:54.154 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:53 smithi017 ceph-mon[215646]: pgmap v1293: 119 pgs: 119 active+clean; 5.5 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 541 op/s 2023-09-23T15:31:54.187 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:54.200 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:54.238 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:54.301 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:31:54.396 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:54.772 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:31:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:54.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:31:54.826 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50588 8.e 8:73911130:::benchmark_data_smithi017_485823_object50587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 598.62064524499999, 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.107695917, 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.827 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50588 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203598+0000", 2023-09-23T15:31:54.828 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203597+0000", 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203626+0000", 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9794999999999998e-05 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.829 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203639+0000", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2792999999999999e-05 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203648+0000", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0340000000000002e-06 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203778+0000", 2023-09-23T15:31:54.830 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129462 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.203789+0000", 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1683e-05 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123801+0000", 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.92001208899999998 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.831 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123813+0000", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1676e-05 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301009+0000", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.177195422 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301190+0000", 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018111599999999999 2023-09-23T15:31:54.832 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.301356+0000", 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166479 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307313+0000", 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059565829999999997 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.833 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307350+0000", 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6908e-05 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309727+0000", 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0023774629999999998 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.834 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.309803+0000", 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6243999999999997e-05 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311294+0000", 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014907830000000001 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50599 8.e 8:730ce43d:::benchmark_data_smithi017_485823_object50598:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:31:54.835 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 598.61241874699999, 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1058259499999998, 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50599 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.836 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211825+0000", 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211823+0000", 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.837 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211849+0000", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6114000000000001e-05 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211866+0000", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6600999999999999e-05 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211874+0000", 2023-09-23T15:31:54.838 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3229999999999998e-06 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211940+0000", 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6039999999999998e-05 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.211958+0000", 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.7448e-05 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.839 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123836+0000", 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91187878 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123844+0000", 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9489999999999996e-06 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.304978+0000", 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1811335549999999 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.840 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305174+0000", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000196164 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.305380+0000", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000206092 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311368+0000", 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059874079999999996 2023-09-23T15:31:54.841 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311447+0000", 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9140000000000005e-05 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315621+0000", 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041739070000000001 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.842 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.315691+0000", 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0588999999999998e-05 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317651+0000", 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001959391 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50609 8.e 8:797f5f67:::benchmark_data_smithi017_485823_object50608:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T15:31:54.843 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 598.60234756, 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.097560632, 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50609 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.844 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221896+0000", 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221894+0000", 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.845 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221919+0000", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5066999999999999e-05 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221946+0000", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7331e-05 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.221959+0000", 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.269e-05 2023-09-23T15:31:54.846 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222024+0000", 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4824000000000001e-05 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:56.222038+0000", 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3788000000000001e-05 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.847 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123868+0000", 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90182994900000002 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123876+0000", 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2979999999999992e-06 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307446+0000", 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183570029 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.848 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307604+0000", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015765000000000001 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.307772+0000", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000168114 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311434+0000", 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0036626810000000001 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.849 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.311453+0000", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9004e-05 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317944+0000", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064906199999999999 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.317998+0000", 2023-09-23T15:31:54.850 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4296999999999999e-05 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.319456+0000", 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014581900000000001 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50596 8.2 8:4072d885:::benchmark_data_smithi017_485823_object50595:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:31:54.851 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 597.70329720200004, 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1658852479999999, 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50596 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.852 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120946+0000", 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120944+0000", 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.853 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.120979+0000", 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4545000000000002e-05 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121002+0000", 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2994e-05 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121014+0000", 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.219e-05 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.854 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121135+0000", 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012048399999999999 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121143+0000", 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.14e-06 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.855 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123756+0000", 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026131420000000002 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.856 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.123766+0000", 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0566e-05 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275228+0000", 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1514619580000001 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275358+0000", 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000129383 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.857 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.275499+0000", 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000141419 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279452+0000", 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039528280000000002 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.858 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279468+0000", 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5583e-05 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285296+0000", 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005828415 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.285337+0000", 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1216e-05 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.859 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.286831+0000", 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0014941799999999999 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.45809.0:50597 8.2 8:4463d1ca:::benchmark_data_smithi017_485823_object50596:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e297)", 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 597.70321009199995, 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.173769987, 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.860 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.45809", 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 50597 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.861 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121033+0000", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121059+0000", 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6764999999999999e-05 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.862 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121068+0000", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6510000000000007e-06 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121075+0000", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6290000000000004e-06 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121210+0000", 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000135697 2023-09-23T15:31:54.863 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.121216+0000", 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2360000000000004e-06 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126405+0000", 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051892259999999999 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.864 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:57.126415+0000", 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0358e-05 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.277883+0000", 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.151468277 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278066+0000", 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018207400000000001 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.865 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,5", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.278266+0000", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020090099999999999 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.279562+0000", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001295825 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.281149+0000", 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00158647 2023-09-23T15:31:54.866 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293587+0000", 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012437981000000001 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.293648+0000", 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0943000000000002e-05 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.867 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:21:58.294803+0000", 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011556100000000001 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83104 9.c 9:3b5b5686:::benchmark_data_smithi017_543597_object83103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.920724+0000", 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 297.90351935199999, 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 27.385669468, 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.868 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83104 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920724+0000", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920724+0000", 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.869 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920723+0000", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920747+0000", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3821000000000001e-05 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920755+0000", 2023-09-23T15:31:54.870 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7970000000000001e-06 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.920761+0000", 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9379999999999997e-06 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.922406+0000", 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016447009999999999 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.871 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.922414+0000", 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7520000000000002e-06 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.377950+0000", 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 11.455535918000001 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.392944+0000", 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01499369 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.872 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.471580+0000", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.078636006999999994 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.476187+0000", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046068970000000004 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:11.411633+0000", 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9354460090000001 2023-09-23T15:31:54.873 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:11.411642+0000", 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4530000000000008e-06 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:12.387543+0000", 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97590012699999995 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.874 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:12.387556+0000", 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3271000000000001e-05 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:23.699520+0000", 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 11.311964506000001 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:23.699527+0000", 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7850000000000003e-06 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.875 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.306378+0000", 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.60685086899999996 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.306394+0000", 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5532999999999998e-05 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.876 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83259 9.4 e361/348 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head v 361'5248, mlcod=344'117)", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 296.76782007600002, 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.141249403, 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.877 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056423+0000", 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056450+0000", 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7029000000000002e-05 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.878 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056466+0000", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6181e-05 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056472+0000", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.772e-06 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.161146+0000", 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.104674147000001 2023-09-23T15:31:54.879 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.161175+0000", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.94e-05 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197621+0000", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.036445402000000002 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197673+0000", 2023-09-23T15:31:54.880 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2212000000000001e-05 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83260 9.4 e361/348 9:295d7846:::benchmark_data_smithi017_543597_object83259:head v 361'5249, mlcod=344'117)", 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 296.76493082100001, 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.138397940000001, 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.881 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059313+0000", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059311+0000", 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.882 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059337+0000", 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6217e-05 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059351+0000", 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3326000000000001e-05 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.883 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059359+0000", 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6440000000000006e-06 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.165673+0000", 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106313241000001 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.165706+0000", 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2969000000000002e-05 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.884 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197678+0000", 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.031972689999999998 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197711+0000", 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2249999999999998e-05 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.885 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83261 9.4 e361/348 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head v 361'5250, mlcod=344'117)", 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 296.761826031, 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.135328450999999, 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.886 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062417+0000", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062416+0000", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062444+0000", 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7557000000000001e-05 2023-09-23T15:31:54.887 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062457+0000", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3112000000000001e-05 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062467+0000", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.9769999999999995e-06 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.169452+0000", 2023-09-23T15:31:54.888 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106984581000001 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.169473+0000", 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1597999999999999e-05 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197713+0000", 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028239744000000001 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.889 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197746+0000", 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2818999999999999e-05 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46157.0:83265 9.4 e361/348 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head v 361'5251, mlcod=344'117)", 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 296.75815918799998, 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131697632, 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.890 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066084+0000", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066083+0000", 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:54.891 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066111+0000", 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7898999999999998e-05 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066127+0000", 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5858e-05 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.892 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.066137+0000", 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0261999999999999e-05 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.172807+0000", 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.106669544000001 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.172829+0000", 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2413e-05 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.893 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197748+0000", 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.024919220999999998 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197782+0000", 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3535999999999999e-05 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88051 9.5 9:a97aa252:::benchmark_data_smithi017_543597_object4260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:31:54.894 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:40.423355+0000", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 254.400888393, 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.886441536, 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88051 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423355+0000", 2023-09-23T15:31:54.895 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423355+0000", 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423353+0000", 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.896 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423360+0000", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7660000000000001e-06 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423379+0000", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9301e-05 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423395+0000", 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5062000000000001e-05 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.897 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423832+0000", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00043749300000000001 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.423844+0000", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.223e-05 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.416886+0000", 2023-09-23T15:31:54.898 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99304132899999997 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.416899+0000", 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3658999999999999e-05 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.299939+0000", 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88303995099999999 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.899 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.300086+0000", 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014648600000000001 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.300220+0000", 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000134449 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.304246+0000", 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040262420000000002 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.900 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.305310+0000", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010637859999999999 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309553+0000", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042425869999999999 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309616+0000", 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3582000000000002e-05 2023-09-23T15:31:54.901 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309797+0000", 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018022900000000001 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:49536 10.c e419/413 10:3333962c:::benchmark_data_smithi017_604517_object49535:head v 419'3127, mlcod=419'3125)", 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:23.899436+0000", 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.924807561999998, 2023-09-23T15:31:54.902 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27245197799999998, 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899436+0000", 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899436+0000", 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.903 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899435+0000", 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899560+0000", 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012441899999999999 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.904 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899569+0000", 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9830000000000002e-06 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899574+0000", 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2340000000000002e-06 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899640+0000", 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5915999999999997e-05 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.905 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.899653+0000", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3402e-05 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171840+0000", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27218732299999998 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171888+0000", 2023-09-23T15:31:54.906 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7406e-05 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:49533 10.a e419/413 10:59e59014:::benchmark_data_smithi017_604517_object49532:head v 419'3156, mlcod=419'3154)", 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:23.901226+0000", 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.923017712999993, 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27721151700000002, 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.907 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.901226+0000", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.901226+0000", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.901225+0000", 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.908 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.901270+0000", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4783999999999997e-05 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171829+0000", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.27055931500000002 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171840+0000", 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0702e-05 2023-09-23T15:31:54.909 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171891+0000", 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1536999999999998e-05 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171917+0000", 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6029000000000001e-05 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.910 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178396+0000", 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064783829999999999 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178437+0000", 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1412000000000003e-05 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:49544 10.4 10:2b666d56:::benchmark_data_smithi017_604517_object49543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T15:31:54.911 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:23.902235+0000", 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 90.922008403999996, 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.276167094, 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 49544 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.912 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.902235+0000", 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.902235+0000", 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.902234+0000", 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.913 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:23.902255+0000", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0956999999999999e-05 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171839+0000", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.26958418099999998 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171852+0000", 2023-09-23T15:31:54.914 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2680999999999999e-05 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.171903+0000", 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1910999999999998e-05 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.172129+0000", 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022604600000000001 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.915 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.172299+0000", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000169204 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178238+0000", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059390090000000003 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178246+0000", 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.242e-06 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.916 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178296+0000", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0537000000000001e-05 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178350+0000", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3940999999999997e-05 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:24.178402+0000", 2023-09-23T15:31:54.917 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1668000000000001e-05 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:51067 10.4 10:2172467c:::benchmark_data_smithi017_604517_object51066:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:25.399038+0000", 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 89.425204980999993, 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23866004599999999, 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:54.918 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 51067 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399038+0000", 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399038+0000", 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.919 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399037+0000", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399060+0000", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.315e-05 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399072+0000", 2023-09-23T15:31:54.920 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2291e-05 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.399080+0000", 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7689999999999996e-06 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.629632+0000", 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23055223699999999 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.921 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.629816+0000", 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000183439 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.629973+0000", 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015739899999999999 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.634311+0000", 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0043382990000000003 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.922 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.635847+0000", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0015354590000000001 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.637602+0000", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001754607 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.637657+0000", 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5315999999999997e-05 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.923 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:25.637698+0000", 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.1647000000000003e-05 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:53342 10.5 e421/413 10:ace6df3b:::benchmark_data_smithi017_604517_object53341:head v 421'3372, mlcod=421'3369)", 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:27.798138+0000", 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 87.026105278000003, 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.24238184400000001, 2023-09-23T15:31:54.924 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798138+0000", 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798138+0000", 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.925 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798137+0000", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798169+0000", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1862000000000001e-05 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798180+0000", 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0984e-05 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.926 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798190+0000", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3789999999999995e-06 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803992+0000", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058027060000000004 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.804014+0000", 2023-09-23T15:31:54.927 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1387999999999999e-05 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040473+0000", 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.236459689 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040520+0000", 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6672999999999999e-05 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.928 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:53347 10.8 10:136b06d8:::benchmark_data_smithi017_604517_object53346:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:27.798247+0000", 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 87.025996367000005, 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23823661600000001, 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53347 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.929 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798247+0000", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798247+0000", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798245+0000", 2023-09-23T15:31:54.930 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798269+0000", 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.368e-05 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798281+0000", 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1579e-05 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.931 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798289+0000", 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9820000000000008e-06 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798333+0000", 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4401999999999999e-05 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798476+0000", 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014329300000000001 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.932 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 2,7", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798606+0000", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012941299999999999 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.800713+0000", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021069019999999999 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.800988+0000", 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00027486200000000001 2023-09-23T15:31:54.933 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036380+0000", 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23539225899999999 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036441+0000", 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1272000000000006e-05 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.934 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036484+0000", 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2524000000000001e-05 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:53349 10.4 10:2e802f3a:::benchmark_data_smithi017_604517_object53348:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:27.798311+0000", 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 87.025932390999998, 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23835214900000001, 2023-09-23T15:31:54.935 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 53349 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798311+0000", 2023-09-23T15:31:54.937 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798311+0000", 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798310+0000", 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.938 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798329+0000", 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9060999999999999e-05 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798335+0000", 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6699999999999999e-06 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798340+0000", 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1070000000000004e-06 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.939 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798408+0000", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7490999999999995e-05 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798546+0000", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013806499999999999 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,4", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.798671+0000", 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012552 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.940 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.800892+0000", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022204059999999999 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.804006+0000", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031147700000000002 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036583+0000", 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.232576487 2023-09-23T15:31:54.941 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036628+0000", 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4728999999999998e-05 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036663+0000", 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5459000000000001e-05 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.942 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:53354 10.d e421/413 10:bd4a137f:::benchmark_data_smithi017_604517_object53353:head v 421'3399, mlcod=421'3397)", 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:27.803687+0000", 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 87.020556869999993, 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.236858386, 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803687+0000", 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.943 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803687+0000", 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803686+0000", 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.944 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803715+0000", 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8926000000000001e-05 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803735+0000", 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0426000000000001e-05 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803743+0000", 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.362e-06 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.945 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.805703+0000", 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001960294 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.805723+0000", 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0086e-05 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.946 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040523+0000", 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.234799754 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040545+0000", 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2223000000000001e-05 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:53346 10.5 e421/413 10:a1c9481b:::benchmark_data_smithi017_604517_object53345:head v 421'3373, mlcod=421'3369)", 2023-09-23T15:31:54.947 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:27.803805+0000", 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 87.020438055, 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.23675874599999999, 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803805+0000", 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:31:54.948 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803805+0000", 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803805+0000", 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803831+0000", 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6495999999999998e-05 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.949 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803843+0000", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1416999999999999e-05 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:27.803850+0000", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2420000000000003e-06 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036018+0000", 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.232168136 2023-09-23T15:31:54.950 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.036041+0000", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.3132000000000001e-05 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040547+0000", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045061679999999996 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:28.040564+0000", 2023-09-23T15:31:54.951 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6656000000000001e-05 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:31:54.952 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:31:54.953 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:31:54.953 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:54.969 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1593722894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:54.969 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1593722894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:54.970 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:31:55.017 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:31:55.043 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:55.058 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:31:55.101 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:31:55.152 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:31:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.168 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.169 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.169 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.169 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:31:55.350 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1593722894' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:31:56.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:55 smithi017 ceph-mon[215646]: pgmap v1294: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2023-09-23T15:31:56.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:55 smithi017 ceph-mon[222878]: pgmap v1294: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2023-09-23T15:31:56.254 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:31:56.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:55 smithi027 ceph-mon[195233]: pgmap v1294: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 997 op/s 2023-09-23T15:31:56.454 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:56.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:56.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:56.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:31:56.562 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:56.660 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:57.097 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:57.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:57.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:57.202 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:57.251 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:57.320 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:57.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:57.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:31:57.505 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:57.607 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:57.652 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:31:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:57.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:31:57.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:31:57.762 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:57.825 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:57.992 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:58.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:58.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:31:58.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:58.265 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:31:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:31:58] "GET /metrics HTTP/1.1" 200 36426 "" "Prometheus/2.43.0" 2023-09-23T15:31:58.319 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:31:58.321 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:31:58.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:58.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:31:58.345 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:31:58.428 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:31:58.486 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:58.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:58 smithi017 ceph-mon[215646]: pgmap v1295: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:31:58.570 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:58.607 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:31:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:58.617 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:31:58.718 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:58.771 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:58.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:58 smithi027 ceph-mon[195233]: pgmap v1295: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:31:58.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:58.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:58 smithi017 ceph-mon[222878]: pgmap v1295: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:31:58.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:31:59.258 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:31:59.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:59.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:31:59.307 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:31:59.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:31:59.390 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:59.460 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:31:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:31:59.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:31:59.568 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:31:59.637 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:31:59.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:31:59 smithi027 ceph-mon[195233]: from='client.46691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:31:59.920 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:31:59 smithi017 ceph-mon[215646]: from='client.46691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:31:59.921 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:31:59 smithi017 ceph-mon[222878]: from='client.46691 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:31:59.946 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1295,"stamp":"2023-09-23T15:31:56.974286+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5595404235,"num_objects":85335,"num_object_clones":0,"num_object_copies":256005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85335,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":127818,"num_write_kb":6803638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":4466766,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114235,"ondisk_log_size":114235,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":20751852,"kb_used_data":16496684,"kb_used_omap":57,"kb_used_meta":4254854,"kb_avail":729209364,"statfs":{"total":767960285184,"available":746710388736,"internally_reserved":0,"allocated":16892604416,"data_stored":16878440509,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4356970521},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,12,6,5,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-539623424,"num_objects":-8234,"num_object_clones":0,"num_object_copies":-24702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8234,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.289090"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":786,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706179+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:31:48.706179+0000","last_peered":"2023-09-23T15:31:48.706179+0000","last_clean":"2023-09-23T15:31:48.706179+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:31:48.706179+0000","last_undegraded":"2023-09-23T15:31:48.706179+0000","last_fullsized":"2023-09-23T15:31:48.706179+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693267+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:31:48.693267+0000","last_peered":"2023-09-23T15:31:48.693267+0000","last_clean":"2023-09-23T15:31:48.693267+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:31:48.693267+0000","last_undegraded":"2023-09-23T15:31:48.693267+0000","last_fullsized":"2023-09-23T15:31:48.693267+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":344,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.707105+0000","last_change":"2023-09-23T15:30:26.692661+0000","last_active":"2023-09-23T15:31:48.707105+0000","last_peered":"2023-09-23T15:31:48.707105+0000","last_clean":"2023-09-23T15:31:48.707105+0000","last_became_active":"2023-09-23T15:30:26.692331+0000","last_became_peered":"2023-09-23T15:30:26.692331+0000","last_unstale":"2023-09-23T15:31:48.707105+0000","last_undegraded":"2023-09-23T15:31:48.707105+0000","last_fullsized":"2023-09-23T15:31:48.707105+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:03.180374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":520,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706493+0000","last_change":"2023-09-23T15:28:55.922068+0000","last_active":"2023-09-23T15:31:48.706493+0000","last_peered":"2023-09-23T15:31:48.706493+0000","last_clean":"2023-09-23T15:31:48.706493+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:31:48.706493+0000","last_undegraded":"2023-09-23T15:31:48.706493+0000","last_fullsized":"2023-09-23T15:31:48.706493+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:55.921999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:55.921999+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:38.044680+0000","scrub_duration":0.0074994720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":364,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693598+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:31:48.693598+0000","last_peered":"2023-09-23T15:31:48.693598+0000","last_clean":"2023-09-23T15:31:48.693598+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:31:48.693598+0000","last_undegraded":"2023-09-23T15:31:48.693598+0000","last_fullsized":"2023-09-23T15:31:48.693598+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":435,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692755+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:31:48.692755+0000","last_peered":"2023-09-23T15:31:48.692755+0000","last_clean":"2023-09-23T15:31:48.692755+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:31:48.692755+0000","last_undegraded":"2023-09-23T15:31:48.692755+0000","last_fullsized":"2023-09-23T15:31:48.692755+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":574,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693406+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:31:48.693406+0000","last_peered":"2023-09-23T15:31:48.693406+0000","last_clean":"2023-09-23T15:31:48.693406+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:31:48.693406+0000","last_undegraded":"2023-09-23T15:31:48.693406+0000","last_fullsized":"2023-09-23T15:31:48.693406+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":375,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692742+0000","last_change":"2023-09-23T15:30:26.692470+0000","last_active":"2023-09-23T15:31:48.692742+0000","last_peered":"2023-09-23T15:31:48.692742+0000","last_clean":"2023-09-23T15:31:48.692742+0000","last_became_active":"2023-09-23T15:30:26.692148+0000","last_became_peered":"2023-09-23T15:30:26.692148+0000","last_unstale":"2023-09-23T15:31:48.692742+0000","last_undegraded":"2023-09-23T15:31:48.692742+0000","last_fullsized":"2023-09-23T15:31:48.692742+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:32:36.258289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":389,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685189+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:31:48.685189+0000","last_peered":"2023-09-23T15:31:48.685189+0000","last_clean":"2023-09-23T15:31:48.685189+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:31:48.685189+0000","last_undegraded":"2023-09-23T15:31:48.685189+0000","last_fullsized":"2023-09-23T15:31:48.685189+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":707,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708978+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:31:48.708978+0000","last_peered":"2023-09-23T15:31:48.708978+0000","last_clean":"2023-09-23T15:31:48.708978+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:31:48.708978+0000","last_undegraded":"2023-09-23T15:31:48.708978+0000","last_fullsized":"2023-09-23T15:31:48.708978+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":389,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693111+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:31:48.693111+0000","last_peered":"2023-09-23T15:31:48.693111+0000","last_clean":"2023-09-23T15:31:48.693111+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:31:48.693111+0000","last_undegraded":"2023-09-23T15:31:48.693111+0000","last_fullsized":"2023-09-23T15:31:48.693111+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":372,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695531+0000","last_change":"2023-09-23T15:30:26.706619+0000","last_active":"2023-09-23T15:31:48.695531+0000","last_peered":"2023-09-23T15:31:48.695531+0000","last_clean":"2023-09-23T15:31:48.695531+0000","last_became_active":"2023-09-23T15:30:26.706343+0000","last_became_peered":"2023-09-23T15:30:26.706343+0000","last_unstale":"2023-09-23T15:31:48.695531+0000","last_undegraded":"2023-09-23T15:31:48.695531+0000","last_fullsized":"2023-09-23T15:31:48.695531+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:47.811881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":849,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.711608+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:31:48.711608+0000","last_peered":"2023-09-23T15:31:48.711608+0000","last_clean":"2023-09-23T15:31:48.711608+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:31:48.711608+0000","last_undegraded":"2023-09-23T15:31:48.711608+0000","last_fullsized":"2023-09-23T15:31:48.711608+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":455,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.710598+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:31:48.710598+0000","last_peered":"2023-09-23T15:31:48.710598+0000","last_clean":"2023-09-23T15:31:48.710598+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:31:48.710598+0000","last_undegraded":"2023-09-23T15:31:48.710598+0000","last_fullsized":"2023-09-23T15:31:48.710598+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":345,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709328+0000","last_change":"2023-09-23T15:30:26.703714+0000","last_active":"2023-09-23T15:31:48.709328+0000","last_peered":"2023-09-23T15:31:48.709328+0000","last_clean":"2023-09-23T15:31:48.709328+0000","last_became_active":"2023-09-23T15:30:26.703402+0000","last_became_peered":"2023-09-23T15:30:26.703402+0000","last_unstale":"2023-09-23T15:31:48.709328+0000","last_undegraded":"2023-09-23T15:31:48.709328+0000","last_fullsized":"2023-09-23T15:31:48.709328+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:48.014628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":541,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.707292+0000","last_change":"2023-09-23T15:28:59.984242+0000","last_active":"2023-09-23T15:31:48.707292+0000","last_peered":"2023-09-23T15:31:48.707292+0000","last_clean":"2023-09-23T15:31:48.707292+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:31:48.707292+0000","last_undegraded":"2023-09-23T15:31:48.707292+0000","last_fullsized":"2023-09-23T15:31:48.707292+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:59.984131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:59.984131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:34.209362+0000","scrub_duration":0.015840911999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":184,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709880+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:31:48.709880+0000","last_peered":"2023-09-23T15:31:48.709880+0000","last_clean":"2023-09-23T15:31:48.709880+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:31:48.709880+0000","last_undegraded":"2023-09-23T15:31:48.709880+0000","last_fullsized":"2023-09-23T15:31:48.709880+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":333,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.696788+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:31:48.696788+0000","last_peered":"2023-09-23T15:31:48.696788+0000","last_clean":"2023-09-23T15:31:48.696788+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:31:48.696788+0000","last_undegraded":"2023-09-23T15:31:48.696788+0000","last_fullsized":"2023-09-23T15:31:48.696788+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":590,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694501+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:31:48.694501+0000","last_peered":"2023-09-23T15:31:48.694501+0000","last_clean":"2023-09-23T15:31:48.694501+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:31:48.694501+0000","last_undegraded":"2023-09-23T15:31:48.694501+0000","last_fullsized":"2023-09-23T15:31:48.694501+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":105,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693084+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:31:48.693084+0000","last_peered":"2023-09-23T15:31:48.693084+0000","last_clean":"2023-09-23T15:31:48.693084+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:31:48.693084+0000","last_undegraded":"2023-09-23T15:31:48.693084+0000","last_fullsized":"2023-09-23T15:31:48.693084+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693357+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:31:48.693357+0000","last_peered":"2023-09-23T15:31:48.693357+0000","last_clean":"2023-09-23T15:31:48.693357+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:31:48.693357+0000","last_undegraded":"2023-09-23T15:31:48.693357+0000","last_fullsized":"2023-09-23T15:31:48.693357+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":750,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693706+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:31:48.693706+0000","last_peered":"2023-09-23T15:31:48.693706+0000","last_clean":"2023-09-23T15:31:48.693706+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:31:48.693706+0000","last_undegraded":"2023-09-23T15:31:48.693706+0000","last_fullsized":"2023-09-23T15:31:48.693706+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":188,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.712106+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:31:48.712106+0000","last_peered":"2023-09-23T15:31:48.712106+0000","last_clean":"2023-09-23T15:31:48.712106+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:31:48.712106+0000","last_undegraded":"2023-09-23T15:31:48.712106+0000","last_fullsized":"2023-09-23T15:31:48.712106+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":394,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693839+0000","last_change":"2023-09-23T15:30:26.705220+0000","last_active":"2023-09-23T15:31:48.693839+0000","last_peered":"2023-09-23T15:31:48.693839+0000","last_clean":"2023-09-23T15:31:48.693839+0000","last_became_active":"2023-09-23T15:30:26.704922+0000","last_became_peered":"2023-09-23T15:30:26.704922+0000","last_unstale":"2023-09-23T15:31:48.693839+0000","last_undegraded":"2023-09-23T15:31:48.693839+0000","last_fullsized":"2023-09-23T15:31:48.693839+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:25.436067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":813,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706357+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:31:48.706357+0000","last_peered":"2023-09-23T15:31:48.706357+0000","last_clean":"2023-09-23T15:31:48.706357+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:31:48.706357+0000","last_undegraded":"2023-09-23T15:31:48.706357+0000","last_fullsized":"2023-09-23T15:31:48.706357+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":433,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694007+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:31:48.694007+0000","last_peered":"2023-09-23T15:31:48.694007+0000","last_clean":"2023-09-23T15:31:48.694007+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:31:48.694007+0000","last_undegraded":"2023-09-23T15:31:48.694007+0000","last_fullsized":"2023-09-23T15:31:48.694007+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":440,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.707035+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:31:48.707035+0000","last_peered":"2023-09-23T15:31:48.707035+0000","last_clean":"2023-09-23T15:31:48.707035+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:31:48.707035+0000","last_undegraded":"2023-09-23T15:31:48.707035+0000","last_fullsized":"2023-09-23T15:31:48.707035+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":766,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695071+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:31:48.695071+0000","last_peered":"2023-09-23T15:31:48.695071+0000","last_clean":"2023-09-23T15:31:48.695071+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:31:48.695071+0000","last_undegraded":"2023-09-23T15:31:48.695071+0000","last_fullsized":"2023-09-23T15:31:48.695071+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":182,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709554+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:31:48.709554+0000","last_peered":"2023-09-23T15:31:48.709554+0000","last_clean":"2023-09-23T15:31:48.709554+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:31:48.709554+0000","last_undegraded":"2023-09-23T15:31:48.709554+0000","last_fullsized":"2023-09-23T15:31:48.709554+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":422,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692820+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:31:48.692820+0000","last_peered":"2023-09-23T15:31:48.692820+0000","last_clean":"2023-09-23T15:31:48.692820+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:31:48.692820+0000","last_undegraded":"2023-09-23T15:31:48.692820+0000","last_fullsized":"2023-09-23T15:31:48.692820+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":754,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708773+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:31:48.708773+0000","last_peered":"2023-09-23T15:31:48.708773+0000","last_clean":"2023-09-23T15:31:48.708773+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:31:48.708773+0000","last_undegraded":"2023-09-23T15:31:48.708773+0000","last_fullsized":"2023-09-23T15:31:48.708773+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":375,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.684761+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:31:48.684761+0000","last_peered":"2023-09-23T15:31:48.684761+0000","last_clean":"2023-09-23T15:31:48.684761+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:31:48.684761+0000","last_undegraded":"2023-09-23T15:31:48.684761+0000","last_fullsized":"2023-09-23T15:31:48.684761+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":395,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709335+0000","last_change":"2023-09-23T15:30:26.706427+0000","last_active":"2023-09-23T15:31:48.709335+0000","last_peered":"2023-09-23T15:31:48.709335+0000","last_clean":"2023-09-23T15:31:48.709335+0000","last_became_active":"2023-09-23T15:30:26.706086+0000","last_became_peered":"2023-09-23T15:30:26.706086+0000","last_unstale":"2023-09-23T15:31:48.709335+0000","last_undegraded":"2023-09-23T15:31:48.709335+0000","last_fullsized":"2023-09-23T15:31:48.709335+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:42.578372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"439'7856","reported_seq":7889,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.041213+0000","last_change":"2023-09-23T15:29:48.688598+0000","last_active":"2023-09-23T15:31:54.041213+0000","last_peered":"2023-09-23T15:31:54.041213+0000","last_clean":"2023-09-23T15:31:54.041213+0000","last_became_active":"2023-09-23T15:29:48.688227+0000","last_became_peered":"2023-09-23T15:29:48.688227+0000","last_unstale":"2023-09-23T15:31:54.041213+0000","last_undegraded":"2023-09-23T15:31:54.041213+0000","last_fullsized":"2023-09-23T15:31:54.041213+0000","mapping_epoch":413,"log_start":"417'100","ondisk_log_start":"417'100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":7756,"log_dups_size":0,"ondisk_log_size":7756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:04.960560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349700096,"num_objects":5336,"num_object_clones":0,"num_object_copies":16008,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5336,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7856,"num_write_kb":422144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":735,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709965+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:31:48.709965+0000","last_peered":"2023-09-23T15:31:48.709965+0000","last_clean":"2023-09-23T15:31:48.709965+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:31:48.709965+0000","last_undegraded":"2023-09-23T15:31:48.709965+0000","last_fullsized":"2023-09-23T15:31:48.709965+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":178,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709417+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:31:48.709417+0000","last_peered":"2023-09-23T15:31:48.709417+0000","last_clean":"2023-09-23T15:31:48.709417+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:31:48.709417+0000","last_undegraded":"2023-09-23T15:31:48.709417+0000","last_fullsized":"2023-09-23T15:31:48.709417+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":447,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706217+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:31:48.706217+0000","last_peered":"2023-09-23T15:31:48.706217+0000","last_clean":"2023-09-23T15:31:48.706217+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:31:48.706217+0000","last_undegraded":"2023-09-23T15:31:48.706217+0000","last_fullsized":"2023-09-23T15:31:48.706217+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"439'7773","reported_seq":7807,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.588656+0000","last_change":"2023-09-23T15:29:48.696982+0000","last_active":"2023-09-23T15:31:54.588656+0000","last_peered":"2023-09-23T15:31:54.588656+0000","last_clean":"2023-09-23T15:31:54.588656+0000","last_became_active":"2023-09-23T15:29:48.696666+0000","last_became_peered":"2023-09-23T15:29:48.696666+0000","last_unstale":"2023-09-23T15:31:54.588656+0000","last_undegraded":"2023-09-23T15:31:54.588656+0000","last_fullsized":"2023-09-23T15:31:54.588656+0000","mapping_epoch":413,"log_start":"417'1300","ondisk_log_start":"417'1300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6473,"log_dups_size":0,"ondisk_log_size":6473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:34.479018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340328448,"num_objects":5193,"num_object_clones":0,"num_object_copies":15579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5193,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7773,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":606,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.696097+0000","last_change":"2023-09-23T15:30:48.890337+0000","last_active":"2023-09-23T15:31:48.696097+0000","last_peered":"2023-09-23T15:31:48.696097+0000","last_clean":"2023-09-23T15:31:48.696097+0000","last_became_active":"2023-09-23T15:30:26.693580+0000","last_became_peered":"2023-09-23T15:30:26.693580+0000","last_unstale":"2023-09-23T15:31:48.696097+0000","last_undegraded":"2023-09-23T15:31:48.696097+0000","last_fullsized":"2023-09-23T15:31:48.696097+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:25.353085+0000","scrub_duration":0.011955009000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":178,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709097+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:31:48.709097+0000","last_peered":"2023-09-23T15:31:48.709097+0000","last_clean":"2023-09-23T15:31:48.709097+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:31:48.709097+0000","last_undegraded":"2023-09-23T15:31:48.709097+0000","last_fullsized":"2023-09-23T15:31:48.709097+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692694+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:31:48.692694+0000","last_peered":"2023-09-23T15:31:48.692694+0000","last_clean":"2023-09-23T15:31:48.692694+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:31:48.692694+0000","last_undegraded":"2023-09-23T15:31:48.692694+0000","last_fullsized":"2023-09-23T15:31:48.692694+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"439'7765","reported_seq":7798,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:53.971613+0000","last_change":"2023-09-23T15:29:48.697177+0000","last_active":"2023-09-23T15:31:53.971613+0000","last_peered":"2023-09-23T15:31:53.971613+0000","last_clean":"2023-09-23T15:31:53.971613+0000","last_became_active":"2023-09-23T15:29:48.696842+0000","last_became_peered":"2023-09-23T15:29:48.696842+0000","last_unstale":"2023-09-23T15:31:53.971613+0000","last_undegraded":"2023-09-23T15:31:53.971613+0000","last_fullsized":"2023-09-23T15:31:53.971613+0000","mapping_epoch":413,"log_start":"417'1300","ondisk_log_start":"417'1300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6465,"log_dups_size":0,"ondisk_log_size":6465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:18.816926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348848128,"num_objects":5323,"num_object_clones":0,"num_object_copies":15969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7765,"num_write_kb":418816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":686,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685030+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:31:48.685030+0000","last_peered":"2023-09-23T15:31:48.685030+0000","last_clean":"2023-09-23T15:31:48.685030+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:31:48.685030+0000","last_undegraded":"2023-09-23T15:31:48.685030+0000","last_fullsized":"2023-09-23T15:31:48.685030+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":166,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709654+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:31:48.709654+0000","last_peered":"2023-09-23T15:31:48.709654+0000","last_clean":"2023-09-23T15:31:48.709654+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:31:48.709654+0000","last_undegraded":"2023-09-23T15:31:48.709654+0000","last_fullsized":"2023-09-23T15:31:48.709654+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":419,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.684811+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:31:48.684811+0000","last_peered":"2023-09-23T15:31:48.684811+0000","last_clean":"2023-09-23T15:31:48.684811+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:31:48.684811+0000","last_undegraded":"2023-09-23T15:31:48.684811+0000","last_fullsized":"2023-09-23T15:31:48.684811+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"439'8051","reported_seq":8086,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.451643+0000","last_change":"2023-09-23T15:29:48.698992+0000","last_active":"2023-09-23T15:31:54.451643+0000","last_peered":"2023-09-23T15:31:54.451643+0000","last_clean":"2023-09-23T15:31:54.451643+0000","last_became_active":"2023-09-23T15:29:48.698673+0000","last_became_peered":"2023-09-23T15:29:48.698673+0000","last_unstale":"2023-09-23T15:31:54.451643+0000","last_undegraded":"2023-09-23T15:31:54.451643+0000","last_fullsized":"2023-09-23T15:31:54.451643+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":8051,"log_dups_size":0,"ondisk_log_size":8051,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:12.184760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347144192,"num_objects":5297,"num_object_clones":0,"num_object_copies":15891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8051,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":656,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694229+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:31:48.694229+0000","last_peered":"2023-09-23T15:31:48.694229+0000","last_clean":"2023-09-23T15:31:48.694229+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:31:48.694229+0000","last_undegraded":"2023-09-23T15:31:48.694229+0000","last_fullsized":"2023-09-23T15:31:48.694229+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":414,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706627+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:31:48.706627+0000","last_peered":"2023-09-23T15:31:48.706627+0000","last_clean":"2023-09-23T15:31:48.706627+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:31:48.706627+0000","last_undegraded":"2023-09-23T15:31:48.706627+0000","last_fullsized":"2023-09-23T15:31:48.706627+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":441,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.691900+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:31:48.691900+0000","last_peered":"2023-09-23T15:31:48.691900+0000","last_clean":"2023-09-23T15:31:48.691900+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:31:48.691900+0000","last_undegraded":"2023-09-23T15:31:48.691900+0000","last_fullsized":"2023-09-23T15:31:48.691900+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"439'7999","reported_seq":8033,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:55.719553+0000","last_change":"2023-09-23T15:29:48.679906+0000","last_active":"2023-09-23T15:31:55.719553+0000","last_peered":"2023-09-23T15:31:55.719553+0000","last_clean":"2023-09-23T15:31:55.719553+0000","last_became_active":"2023-09-23T15:29:48.679584+0000","last_became_peered":"2023-09-23T15:29:48.679584+0000","last_unstale":"2023-09-23T15:31:55.719553+0000","last_undegraded":"2023-09-23T15:31:55.719553+0000","last_fullsized":"2023-09-23T15:31:55.719553+0000","mapping_epoch":413,"log_start":"417'1000","ondisk_log_start":"417'1000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6999,"log_dups_size":0,"ondisk_log_size":6999,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:53.805715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347930624,"num_objects":5309,"num_object_clones":0,"num_object_copies":15927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7999,"num_write_kb":425856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":691,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709411+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:31:48.709411+0000","last_peered":"2023-09-23T15:31:48.709411+0000","last_clean":"2023-09-23T15:31:48.709411+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:31:48.709411+0000","last_undegraded":"2023-09-23T15:31:48.709411+0000","last_fullsized":"2023-09-23T15:31:48.709411+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":91,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.691987+0000","last_change":"2023-09-23T15:30:26.690336+0000","last_active":"2023-09-23T15:31:48.691987+0000","last_peered":"2023-09-23T15:31:48.691987+0000","last_clean":"2023-09-23T15:31:48.691987+0000","last_became_active":"2023-09-23T15:30:26.690141+0000","last_became_peered":"2023-09-23T15:30:26.690141+0000","last_unstale":"2023-09-23T15:31:48.691987+0000","last_undegraded":"2023-09-23T15:31:48.691987+0000","last_fullsized":"2023-09-23T15:31:48.691987+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:44.578933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692987+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:31:48.692987+0000","last_peered":"2023-09-23T15:31:48.692987+0000","last_clean":"2023-09-23T15:31:48.692987+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:31:48.692987+0000","last_undegraded":"2023-09-23T15:31:48.692987+0000","last_fullsized":"2023-09-23T15:31:48.692987+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"439'8068","reported_seq":8102,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.451772+0000","last_change":"2023-09-23T15:29:48.698981+0000","last_active":"2023-09-23T15:31:54.451772+0000","last_peered":"2023-09-23T15:31:54.451772+0000","last_clean":"2023-09-23T15:31:54.451772+0000","last_became_active":"2023-09-23T15:29:48.698707+0000","last_became_peered":"2023-09-23T15:29:48.698707+0000","last_unstale":"2023-09-23T15:31:54.451772+0000","last_undegraded":"2023-09-23T15:31:54.451772+0000","last_fullsized":"2023-09-23T15:31:54.451772+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":8068,"log_dups_size":0,"ondisk_log_size":8068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:28.109594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357826560,"num_objects":5460,"num_object_clones":0,"num_object_copies":16380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8068,"num_write_kb":432896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":762,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706427+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:31:48.706427+0000","last_peered":"2023-09-23T15:31:48.706427+0000","last_clean":"2023-09-23T15:31:48.706427+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:31:48.706427+0000","last_undegraded":"2023-09-23T15:31:48.706427+0000","last_fullsized":"2023-09-23T15:31:48.706427+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":406,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.684928+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:31:48.684928+0000","last_peered":"2023-09-23T15:31:48.684928+0000","last_clean":"2023-09-23T15:31:48.684928+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:31:48.684928+0000","last_undegraded":"2023-09-23T15:31:48.684928+0000","last_fullsized":"2023-09-23T15:31:48.684928+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693083+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:31:48.693083+0000","last_peered":"2023-09-23T15:31:48.693083+0000","last_clean":"2023-09-23T15:31:48.693083+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:31:48.693083+0000","last_undegraded":"2023-09-23T15:31:48.693083+0000","last_fullsized":"2023-09-23T15:31:48.693083+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"439'7864","reported_seq":7898,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.582252+0000","last_change":"2023-09-23T15:29:48.683338+0000","last_active":"2023-09-23T15:31:54.582252+0000","last_peered":"2023-09-23T15:31:54.582252+0000","last_clean":"2023-09-23T15:31:54.582252+0000","last_became_active":"2023-09-23T15:29:48.682970+0000","last_became_peered":"2023-09-23T15:29:48.682970+0000","last_unstale":"2023-09-23T15:31:54.582252+0000","last_undegraded":"2023-09-23T15:31:54.582252+0000","last_fullsized":"2023-09-23T15:31:54.582252+0000","mapping_epoch":413,"log_start":"417'1400","ondisk_log_start":"417'1400","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6464,"log_dups_size":0,"ondisk_log_size":6464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:20.273166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348913664,"num_objects":5324,"num_object_clones":0,"num_object_copies":15972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7864,"num_write_kb":422016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":719,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694906+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:31:48.694906+0000","last_peered":"2023-09-23T15:31:48.694906+0000","last_clean":"2023-09-23T15:31:48.694906+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:31:48.694906+0000","last_undegraded":"2023-09-23T15:31:48.694906+0000","last_fullsized":"2023-09-23T15:31:48.694906+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":95,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692726+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:31:48.692726+0000","last_peered":"2023-09-23T15:31:48.692726+0000","last_clean":"2023-09-23T15:31:48.692726+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:31:48.692726+0000","last_undegraded":"2023-09-23T15:31:48.692726+0000","last_fullsized":"2023-09-23T15:31:48.692726+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694288+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:31:48.694288+0000","last_peered":"2023-09-23T15:31:48.694288+0000","last_clean":"2023-09-23T15:31:48.694288+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:31:48.694288+0000","last_undegraded":"2023-09-23T15:31:48.694288+0000","last_fullsized":"2023-09-23T15:31:48.694288+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"439'7783","reported_seq":7817,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.138889+0000","last_change":"2023-09-23T15:29:48.680338+0000","last_active":"2023-09-23T15:31:54.138889+0000","last_peered":"2023-09-23T15:31:54.138889+0000","last_clean":"2023-09-23T15:31:54.138889+0000","last_became_active":"2023-09-23T15:29:48.679979+0000","last_became_peered":"2023-09-23T15:29:48.679979+0000","last_unstale":"2023-09-23T15:31:54.138889+0000","last_undegraded":"2023-09-23T15:31:54.138889+0000","last_fullsized":"2023-09-23T15:31:54.138889+0000","mapping_epoch":413,"log_start":"418'2000","ondisk_log_start":"418'2000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:54.650904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344260608,"num_objects":5253,"num_object_clones":0,"num_object_copies":15759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7783,"num_write_kb":417152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":774,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708693+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:31:48.708693+0000","last_peered":"2023-09-23T15:31:48.708693+0000","last_clean":"2023-09-23T15:31:48.708693+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:31:48.708693+0000","last_undegraded":"2023-09-23T15:31:48.708693+0000","last_fullsized":"2023-09-23T15:31:48.708693+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695455+0000","last_change":"2023-09-23T15:30:26.705171+0000","last_active":"2023-09-23T15:31:48.695455+0000","last_peered":"2023-09-23T15:31:48.695455+0000","last_clean":"2023-09-23T15:31:48.695455+0000","last_became_active":"2023-09-23T15:30:26.704853+0000","last_became_peered":"2023-09-23T15:30:26.704853+0000","last_unstale":"2023-09-23T15:31:48.695455+0000","last_undegraded":"2023-09-23T15:31:48.695455+0000","last_fullsized":"2023-09-23T15:31:48.695455+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:11.875343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":374,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695029+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:31:48.695029+0000","last_peered":"2023-09-23T15:31:48.695029+0000","last_clean":"2023-09-23T15:31:48.695029+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:31:48.695029+0000","last_undegraded":"2023-09-23T15:31:48.695029+0000","last_fullsized":"2023-09-23T15:31:48.695029+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"439'7940","reported_seq":7974,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.400441+0000","last_change":"2023-09-23T15:29:48.682296+0000","last_active":"2023-09-23T15:31:54.400441+0000","last_peered":"2023-09-23T15:31:54.400441+0000","last_clean":"2023-09-23T15:31:54.400441+0000","last_became_active":"2023-09-23T15:29:48.681890+0000","last_became_peered":"2023-09-23T15:29:48.681890+0000","last_unstale":"2023-09-23T15:31:54.400441+0000","last_undegraded":"2023-09-23T15:31:54.400441+0000","last_fullsized":"2023-09-23T15:31:54.400441+0000","mapping_epoch":413,"log_start":"417'1500","ondisk_log_start":"417'1500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6440,"log_dups_size":0,"ondisk_log_size":6440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:34.635613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350748672,"num_objects":5352,"num_object_clones":0,"num_object_copies":16056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7940,"num_write_kb":425344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":679,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685180+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:31:48.685180+0000","last_peered":"2023-09-23T15:31:48.685180+0000","last_clean":"2023-09-23T15:31:48.685180+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:31:48.685180+0000","last_undegraded":"2023-09-23T15:31:48.685180+0000","last_fullsized":"2023-09-23T15:31:48.685180+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":657,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694141+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:31:48.694141+0000","last_peered":"2023-09-23T15:31:48.694141+0000","last_clean":"2023-09-23T15:31:48.694141+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:31:48.694141+0000","last_undegraded":"2023-09-23T15:31:48.694141+0000","last_fullsized":"2023-09-23T15:31:48.694141+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":414,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706963+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:31:48.706963+0000","last_peered":"2023-09-23T15:31:48.706963+0000","last_clean":"2023-09-23T15:31:48.706963+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:31:48.706963+0000","last_undegraded":"2023-09-23T15:31:48.706963+0000","last_fullsized":"2023-09-23T15:31:48.706963+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":403,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.707145+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:31:48.707145+0000","last_peered":"2023-09-23T15:31:48.707145+0000","last_clean":"2023-09-23T15:31:48.707145+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:31:48.707145+0000","last_undegraded":"2023-09-23T15:31:48.707145+0000","last_fullsized":"2023-09-23T15:31:48.707145+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"439'8057","reported_seq":8091,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.588709+0000","last_change":"2023-09-23T15:29:48.686875+0000","last_active":"2023-09-23T15:31:54.588709+0000","last_peered":"2023-09-23T15:31:54.588709+0000","last_clean":"2023-09-23T15:31:54.588709+0000","last_became_active":"2023-09-23T15:29:48.686520+0000","last_became_peered":"2023-09-23T15:29:48.686520+0000","last_unstale":"2023-09-23T15:31:54.588709+0000","last_undegraded":"2023-09-23T15:31:54.588709+0000","last_fullsized":"2023-09-23T15:31:54.588709+0000","mapping_epoch":413,"log_start":"418'1600","ondisk_log_start":"418'1600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6457,"log_dups_size":0,"ondisk_log_size":6457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:52.851173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8057,"num_write_kb":431232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":735,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706303+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:31:48.706303+0000","last_peered":"2023-09-23T15:31:48.706303+0000","last_clean":"2023-09-23T15:31:48.706303+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:31:48.706303+0000","last_undegraded":"2023-09-23T15:31:48.706303+0000","last_fullsized":"2023-09-23T15:31:48.706303+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":973,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709036+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:31:48.709036+0000","last_peered":"2023-09-23T15:31:48.709036+0000","last_clean":"2023-09-23T15:31:48.709036+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:31:48.709036+0000","last_undegraded":"2023-09-23T15:31:48.709036+0000","last_fullsized":"2023-09-23T15:31:48.709036+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693128+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:31:48.693128+0000","last_peered":"2023-09-23T15:31:48.693128+0000","last_clean":"2023-09-23T15:31:48.693128+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:31:48.693128+0000","last_undegraded":"2023-09-23T15:31:48.693128+0000","last_fullsized":"2023-09-23T15:31:48.693128+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693765+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:31:48.693765+0000","last_peered":"2023-09-23T15:31:48.693765+0000","last_clean":"2023-09-23T15:31:48.693765+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:31:48.693765+0000","last_undegraded":"2023-09-23T15:31:48.693765+0000","last_fullsized":"2023-09-23T15:31:48.693765+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"439'8042","reported_seq":8075,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.041383+0000","last_change":"2023-09-23T15:29:48.686854+0000","last_active":"2023-09-23T15:31:54.041383+0000","last_peered":"2023-09-23T15:31:54.041383+0000","last_clean":"2023-09-23T15:31:54.041383+0000","last_became_active":"2023-09-23T15:29:48.686539+0000","last_became_peered":"2023-09-23T15:29:48.686539+0000","last_unstale":"2023-09-23T15:31:54.041383+0000","last_undegraded":"2023-09-23T15:31:54.041383+0000","last_fullsized":"2023-09-23T15:31:54.041383+0000","mapping_epoch":413,"log_start":"417'300","ondisk_log_start":"417'300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":7742,"log_dups_size":0,"ondisk_log_size":7742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:33.671531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351928320,"num_objects":5370,"num_object_clones":0,"num_object_copies":16110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8042,"num_write_kb":429184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":802,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708987+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:31:48.708987+0000","last_peered":"2023-09-23T15:31:48.708987+0000","last_clean":"2023-09-23T15:31:48.708987+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:31:48.708987+0000","last_undegraded":"2023-09-23T15:31:48.708987+0000","last_fullsized":"2023-09-23T15:31:48.708987+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1058,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.710791+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:31:48.710791+0000","last_peered":"2023-09-23T15:31:48.710791+0000","last_clean":"2023-09-23T15:31:48.710791+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:31:48.710791+0000","last_undegraded":"2023-09-23T15:31:48.710791+0000","last_fullsized":"2023-09-23T15:31:48.710791+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":334,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695609+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:31:48.695609+0000","last_peered":"2023-09-23T15:31:48.695609+0000","last_clean":"2023-09-23T15:31:48.695609+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:31:48.695609+0000","last_undegraded":"2023-09-23T15:31:48.695609+0000","last_fullsized":"2023-09-23T15:31:48.695609+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":342,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694959+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:31:48.694959+0000","last_peered":"2023-09-23T15:31:48.694959+0000","last_clean":"2023-09-23T15:31:48.694959+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:31:48.694959+0000","last_undegraded":"2023-09-23T15:31:48.694959+0000","last_fullsized":"2023-09-23T15:31:48.694959+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"439'7886","reported_seq":7920,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.564302+0000","last_change":"2023-09-23T15:29:48.684663+0000","last_active":"2023-09-23T15:31:54.564302+0000","last_peered":"2023-09-23T15:31:54.564302+0000","last_clean":"2023-09-23T15:31:54.564302+0000","last_became_active":"2023-09-23T15:29:48.684293+0000","last_became_peered":"2023-09-23T15:29:48.684293+0000","last_unstale":"2023-09-23T15:31:54.564302+0000","last_undegraded":"2023-09-23T15:31:54.564302+0000","last_fullsized":"2023-09-23T15:31:54.564302+0000","mapping_epoch":413,"log_start":"417'1500","ondisk_log_start":"417'1500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6386,"log_dups_size":0,"ondisk_log_size":6386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:18.182468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345112576,"num_objects":5266,"num_object_clones":0,"num_object_copies":15798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7886,"num_write_kb":420864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":784,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706639+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:31:48.706639+0000","last_peered":"2023-09-23T15:31:48.706639+0000","last_clean":"2023-09-23T15:31:48.706639+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:31:48.706639+0000","last_undegraded":"2023-09-23T15:31:48.706639+0000","last_fullsized":"2023-09-23T15:31:48.706639+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":745,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708879+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:31:48.708879+0000","last_peered":"2023-09-23T15:31:48.708879+0000","last_clean":"2023-09-23T15:31:48.708879+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:31:48.708879+0000","last_undegraded":"2023-09-23T15:31:48.708879+0000","last_fullsized":"2023-09-23T15:31:48.708879+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":308,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708849+0000","last_change":"2023-09-23T15:30:26.694100+0000","last_active":"2023-09-23T15:31:48.708849+0000","last_peered":"2023-09-23T15:31:48.708849+0000","last_clean":"2023-09-23T15:31:48.708849+0000","last_became_active":"2023-09-23T15:30:26.693767+0000","last_became_peered":"2023-09-23T15:30:26.693767+0000","last_unstale":"2023-09-23T15:31:48.708849+0000","last_undegraded":"2023-09-23T15:31:48.708849+0000","last_fullsized":"2023-09-23T15:31:48.708849+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.703270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":431,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685112+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:31:48.685112+0000","last_peered":"2023-09-23T15:31:48.685112+0000","last_clean":"2023-09-23T15:31:48.685112+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:31:48.685112+0000","last_undegraded":"2023-09-23T15:31:48.685112+0000","last_fullsized":"2023-09-23T15:31:48.685112+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"439'7748","reported_seq":7780,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:53.975747+0000","last_change":"2023-09-23T15:29:48.679894+0000","last_active":"2023-09-23T15:31:53.975747+0000","last_peered":"2023-09-23T15:31:53.975747+0000","last_clean":"2023-09-23T15:31:53.975747+0000","last_became_active":"2023-09-23T15:29:48.679648+0000","last_became_peered":"2023-09-23T15:29:48.679648+0000","last_unstale":"2023-09-23T15:31:53.975747+0000","last_undegraded":"2023-09-23T15:31:53.975747+0000","last_fullsized":"2023-09-23T15:31:53.975747+0000","mapping_epoch":413,"log_start":"417'1300","ondisk_log_start":"417'1300","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6448,"log_dups_size":0,"ondisk_log_size":6448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:23.079554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347209728,"num_objects":5298,"num_object_clones":0,"num_object_copies":15894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7748,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":783,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694827+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:31:48.694827+0000","last_peered":"2023-09-23T15:31:48.694827+0000","last_clean":"2023-09-23T15:31:48.694827+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:31:48.694827+0000","last_undegraded":"2023-09-23T15:31:48.694827+0000","last_fullsized":"2023-09-23T15:31:48.694827+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":152,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.708734+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:31:48.708734+0000","last_peered":"2023-09-23T15:31:48.708734+0000","last_clean":"2023-09-23T15:31:48.708734+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:31:48.708734+0000","last_undegraded":"2023-09-23T15:31:48.708734+0000","last_fullsized":"2023-09-23T15:31:48.708734+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":427,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693634+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:31:48.693634+0000","last_peered":"2023-09-23T15:31:48.693634+0000","last_clean":"2023-09-23T15:31:48.693634+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:31:48.693634+0000","last_undegraded":"2023-09-23T15:31:48.693634+0000","last_fullsized":"2023-09-23T15:31:48.693634+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":420,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692583+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:31:48.692583+0000","last_peered":"2023-09-23T15:31:48.692583+0000","last_clean":"2023-09-23T15:31:48.692583+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:31:48.692583+0000","last_undegraded":"2023-09-23T15:31:48.692583+0000","last_fullsized":"2023-09-23T15:31:48.692583+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"439'8070","reported_seq":8103,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.592700+0000","last_change":"2023-09-23T15:29:48.683852+0000","last_active":"2023-09-23T15:31:54.592700+0000","last_peered":"2023-09-23T15:31:54.592700+0000","last_clean":"2023-09-23T15:31:54.592700+0000","last_became_active":"2023-09-23T15:29:48.683560+0000","last_became_peered":"2023-09-23T15:29:48.683560+0000","last_unstale":"2023-09-23T15:31:54.592700+0000","last_undegraded":"2023-09-23T15:31:54.592700+0000","last_fullsized":"2023-09-23T15:31:54.592700+0000","mapping_epoch":413,"log_start":"418'1600","ondisk_log_start":"418'1600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6470,"log_dups_size":0,"ondisk_log_size":6470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:38.866091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356057112,"num_objects":5434,"num_object_clones":0,"num_object_copies":16302,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5434,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8070,"num_write_kb":432065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":765,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.684937+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:31:48.684937+0000","last_peered":"2023-09-23T15:31:48.684937+0000","last_clean":"2023-09-23T15:31:48.684937+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:31:48.684937+0000","last_undegraded":"2023-09-23T15:31:48.684937+0000","last_fullsized":"2023-09-23T15:31:48.684937+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1178,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.709741+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:31:48.709741+0000","last_peered":"2023-09-23T15:31:48.709741+0000","last_clean":"2023-09-23T15:31:48.709741+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:31:48.709741+0000","last_undegraded":"2023-09-23T15:31:48.709741+0000","last_fullsized":"2023-09-23T15:31:48.709741+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":319,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693985+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:31:48.693985+0000","last_peered":"2023-09-23T15:31:48.693985+0000","last_clean":"2023-09-23T15:31:48.693985+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:31:48.693985+0000","last_undegraded":"2023-09-23T15:31:48.693985+0000","last_fullsized":"2023-09-23T15:31:48.693985+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":420,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692143+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:31:48.692143+0000","last_peered":"2023-09-23T15:31:48.692143+0000","last_clean":"2023-09-23T15:31:48.692143+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:31:48.692143+0000","last_undegraded":"2023-09-23T15:31:48.692143+0000","last_fullsized":"2023-09-23T15:31:48.692143+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"439'7994","reported_seq":8028,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.158499+0000","last_change":"2023-09-23T15:29:48.685734+0000","last_active":"2023-09-23T15:31:54.158499+0000","last_peered":"2023-09-23T15:31:54.158499+0000","last_clean":"2023-09-23T15:31:54.158499+0000","last_became_active":"2023-09-23T15:29:48.684906+0000","last_became_peered":"2023-09-23T15:29:48.684906+0000","last_unstale":"2023-09-23T15:31:54.158499+0000","last_undegraded":"2023-09-23T15:31:54.158499+0000","last_fullsized":"2023-09-23T15:31:54.158499+0000","mapping_epoch":413,"log_start":"418'2200","ondisk_log_start":"418'2200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":5794,"log_dups_size":0,"ondisk_log_size":5794,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:06.890491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348127232,"num_objects":5312,"num_object_clones":0,"num_object_copies":15936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7994,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":730,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693180+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:31:48.693180+0000","last_peered":"2023-09-23T15:31:48.693180+0000","last_clean":"2023-09-23T15:31:48.693180+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:31:48.693180+0000","last_undegraded":"2023-09-23T15:31:48.693180+0000","last_fullsized":"2023-09-23T15:31:48.693180+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":212,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.707612+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:31:48.707612+0000","last_peered":"2023-09-23T15:31:48.707612+0000","last_clean":"2023-09-23T15:31:48.707612+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:31:48.707612+0000","last_undegraded":"2023-09-23T15:31:48.707612+0000","last_fullsized":"2023-09-23T15:31:48.707612+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1073,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694748+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:31:48.694748+0000","last_peered":"2023-09-23T15:31:48.694748+0000","last_clean":"2023-09-23T15:31:48.694748+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:31:48.694748+0000","last_undegraded":"2023-09-23T15:31:48.694748+0000","last_fullsized":"2023-09-23T15:31:48.694748+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":386,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.712069+0000","last_change":"2023-09-23T15:30:26.710395+0000","last_active":"2023-09-23T15:31:48.712069+0000","last_peered":"2023-09-23T15:31:48.712069+0000","last_clean":"2023-09-23T15:31:48.712069+0000","last_became_active":"2023-09-23T15:30:26.710076+0000","last_became_peered":"2023-09-23T15:30:26.710076+0000","last_unstale":"2023-09-23T15:31:48.712069+0000","last_undegraded":"2023-09-23T15:31:48.712069+0000","last_fullsized":"2023-09-23T15:31:48.712069+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.313559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"439'8015","reported_seq":8049,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:54.572845+0000","last_change":"2023-09-23T15:29:48.697095+0000","last_active":"2023-09-23T15:31:54.572845+0000","last_peered":"2023-09-23T15:31:54.572845+0000","last_clean":"2023-09-23T15:31:54.572845+0000","last_became_active":"2023-09-23T15:29:48.696636+0000","last_became_peered":"2023-09-23T15:29:48.696636+0000","last_unstale":"2023-09-23T15:31:54.572845+0000","last_undegraded":"2023-09-23T15:31:54.572845+0000","last_fullsized":"2023-09-23T15:31:54.572845+0000","mapping_epoch":413,"log_start":"418'1600","ondisk_log_start":"418'1600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6415,"log_dups_size":0,"ondisk_log_size":6415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:52.284501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351338496,"num_objects":5361,"num_object_clones":0,"num_object_copies":16083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8015,"num_write_kb":428032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":579,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706478+0000","last_change":"2023-09-23T15:30:26.695170+0000","last_active":"2023-09-23T15:31:48.706478+0000","last_peered":"2023-09-23T15:31:48.706478+0000","last_clean":"2023-09-23T15:31:48.706478+0000","last_became_active":"2023-09-23T15:30:26.694799+0000","last_became_peered":"2023-09-23T15:30:26.694799+0000","last_unstale":"2023-09-23T15:31:48.706478+0000","last_undegraded":"2023-09-23T15:31:48.706478+0000","last_fullsized":"2023-09-23T15:31:48.706478+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:29.288370+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":892,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685307+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:31:48.685307+0000","last_peered":"2023-09-23T15:31:48.685307+0000","last_clean":"2023-09-23T15:31:48.685307+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:31:48.685307+0000","last_undegraded":"2023-09-23T15:31:48.685307+0000","last_fullsized":"2023-09-23T15:31:48.685307+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":479,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692678+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:31:48.692678+0000","last_peered":"2023-09-23T15:31:48.692678+0000","last_clean":"2023-09-23T15:31:48.692678+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:31:48.692678+0000","last_undegraded":"2023-09-23T15:31:48.692678+0000","last_fullsized":"2023-09-23T15:31:48.692678+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692768+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:31:48.692768+0000","last_peered":"2023-09-23T15:31:48.692768+0000","last_clean":"2023-09-23T15:31:48.692768+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:31:48.692768+0000","last_undegraded":"2023-09-23T15:31:48.692768+0000","last_fullsized":"2023-09-23T15:31:48.692768+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":648,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693461+0000","last_change":"2023-09-23T15:30:26.740237+0000","last_active":"2023-09-23T15:31:48.693461+0000","last_peered":"2023-09-23T15:31:48.693461+0000","last_clean":"2023-09-23T15:31:48.693461+0000","last_became_active":"2023-09-23T15:30:26.690718+0000","last_became_peered":"2023-09-23T15:30:26.690718+0000","last_unstale":"2023-09-23T15:31:48.693461+0000","last_undegraded":"2023-09-23T15:31:48.693461+0000","last_fullsized":"2023-09-23T15:31:48.693461+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:00.439552+0000","scrub_duration":0.013441308000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":458,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692072+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:31:48.692072+0000","last_peered":"2023-09-23T15:31:48.692072+0000","last_clean":"2023-09-23T15:31:48.692072+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:31:48.692072+0000","last_undegraded":"2023-09-23T15:31:48.692072+0000","last_fullsized":"2023-09-23T15:31:48.692072+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":444,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.711773+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:31:48.711773+0000","last_peered":"2023-09-23T15:31:48.711773+0000","last_clean":"2023-09-23T15:31:48.711773+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:31:48.711773+0000","last_undegraded":"2023-09-23T15:31:48.711773+0000","last_fullsized":"2023-09-23T15:31:48.711773+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":616,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685038+0000","last_change":"2023-09-23T15:30:28.373196+0000","last_active":"2023-09-23T15:31:48.685038+0000","last_peered":"2023-09-23T15:31:48.685038+0000","last_clean":"2023-09-23T15:31:48.685038+0000","last_became_active":"2023-09-23T15:30:26.699434+0000","last_became_peered":"2023-09-23T15:30:26.699434+0000","last_unstale":"2023-09-23T15:31:48.685038+0000","last_undegraded":"2023-09-23T15:31:48.685038+0000","last_fullsized":"2023-09-23T15:31:48.685038+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:26.970670+0000","scrub_duration":0.0018471900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":479,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706715+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:31:48.706715+0000","last_peered":"2023-09-23T15:31:48.706715+0000","last_clean":"2023-09-23T15:31:48.706715+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:31:48.706715+0000","last_undegraded":"2023-09-23T15:31:48.706715+0000","last_fullsized":"2023-09-23T15:31:48.706715+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693283+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:31:48.693283+0000","last_peered":"2023-09-23T15:31:48.693283+0000","last_clean":"2023-09-23T15:31:48.693283+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:31:48.693283+0000","last_undegraded":"2023-09-23T15:31:48.693283+0000","last_fullsized":"2023-09-23T15:31:48.693283+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":459,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.706881+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:31:48.706881+0000","last_peered":"2023-09-23T15:31:48.706881+0000","last_clean":"2023-09-23T15:31:48.706881+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:31:48.706881+0000","last_undegraded":"2023-09-23T15:31:48.706881+0000","last_fullsized":"2023-09-23T15:31:48.706881+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693481+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:31:48.693481+0000","last_peered":"2023-09-23T15:31:48.693481+0000","last_clean":"2023-09-23T15:31:48.693481+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:31:48.693481+0000","last_undegraded":"2023-09-23T15:31:48.693481+0000","last_fullsized":"2023-09-23T15:31:48.693481+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":626,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.711732+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:31:48.711732+0000","last_peered":"2023-09-23T15:31:48.711732+0000","last_clean":"2023-09-23T15:31:48.711732+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:31:48.711732+0000","last_undegraded":"2023-09-23T15:31:48.711732+0000","last_fullsized":"2023-09-23T15:31:48.711732+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":71,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.685249+0000","last_change":"2023-09-23T15:30:26.699597+0000","last_active":"2023-09-23T15:31:48.685249+0000","last_peered":"2023-09-23T15:31:48.685249+0000","last_clean":"2023-09-23T15:31:48.685249+0000","last_became_active":"2023-09-23T15:30:26.699233+0000","last_became_peered":"2023-09-23T15:30:26.699233+0000","last_unstale":"2023-09-23T15:31:48.685249+0000","last_undegraded":"2023-09-23T15:31:48.685249+0000","last_fullsized":"2023-09-23T15:31:48.685249+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.804745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":363,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.710631+0000","last_change":"2023-09-23T15:30:26.685257+0000","last_active":"2023-09-23T15:31:48.710631+0000","last_peered":"2023-09-23T15:31:48.710631+0000","last_clean":"2023-09-23T15:31:48.710631+0000","last_became_active":"2023-09-23T15:30:26.684954+0000","last_became_peered":"2023-09-23T15:30:26.684954+0000","last_unstale":"2023-09-23T15:31:48.710631+0000","last_undegraded":"2023-09-23T15:31:48.710631+0000","last_fullsized":"2023-09-23T15:31:48.710631+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:49.244950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":619,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693594+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:31:48.693594+0000","last_peered":"2023-09-23T15:31:48.693594+0000","last_clean":"2023-09-23T15:31:48.693594+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:31:48.693594+0000","last_undegraded":"2023-09-23T15:31:48.693594+0000","last_fullsized":"2023-09-23T15:31:48.693594+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":394,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693186+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:31:48.693186+0000","last_peered":"2023-09-23T15:31:48.693186+0000","last_clean":"2023-09-23T15:31:48.693186+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:31:48.693186+0000","last_undegraded":"2023-09-23T15:31:48.693186+0000","last_fullsized":"2023-09-23T15:31:48.693186+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":81,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.692585+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:31:48.692585+0000","last_peered":"2023-09-23T15:31:48.692585+0000","last_clean":"2023-09-23T15:31:48.692585+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:31:48.692585+0000","last_undegraded":"2023-09-23T15:31:48.692585+0000","last_fullsized":"2023-09-23T15:31:48.692585+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5590614040,"num_objects":85307,"num_object_clones":0,"num_object_copies":255921,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":85307,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":126911,"num_write_kb":6790913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16775852032,"data_stored":16775839816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108211,"ondisk_log_size":108211,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768756,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1506580,"kb_used_data":1371660,"kb_used_omap":13,"kb_used_meta":134898,"kb_avail":92238572,"statfs":{"total":95995035648,"available":94452297728,"internally_reserved":0,"allocated":1404579840,"data_stored":1403143007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":138135646},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":7.742,"5min":5.9420000000000002,"15min":5.8419999999999996},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":75.201999999999998,"5min":80,"15min":80},"last":10.452999999999999},{"interface":"front","average":{"1min":7.6130000000000004,"5min":5.8869999999999996,"15min":5.9009999999999998},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":75.186000000000007,"5min":80.117000000000004,"15min":80.117000000000004},"last":10.467000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":10.943,"15min":17.835999999999999},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.9930000000000003,"5min":157.096,"15min":157.096},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.083,"5min":10.696,"15min":17.853000000000002},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8730000000000002,"5min":157.011,"15min":157.011},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":1.895,"15min":1.1399999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.1500000000000004,"5min":44.451999999999998,"15min":44.451999999999998},"last":4.1559999999999997},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6890000000000001,"15min":1.202},"min":{"1min":0.33100000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.069,"5min":44.384999999999998,"15min":44.384999999999998},"last":4.3079999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":3.3620000000000001,"15min":3.8340000000000001},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.0309999999999997,"5min":76.019000000000005,"15min":76.019000000000005},"last":10.061},{"interface":"front","average":{"1min":1.9339999999999999,"5min":3.7229999999999999,"15min":4.2279999999999998},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0709999999999997,"5min":76.317999999999998,"15min":76.317999999999998},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":2.851,"15min":1.329},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.923,"5min":170.15799999999999,"15min":170.15799999999999},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.851,"15min":1.28},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.1150000000000002,"5min":170.017,"15min":170.017},"last":0.55100000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.78000000000000003,"15min":0.79800000000000004},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9870000000000001,"5min":3.742,"15min":3.742},"last":0.44900000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.0109999999999999,"15min":0.82099999999999995},"min":{"1min":0.36699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.2329999999999997,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.35399999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":3.9550000000000001,"15min":1.8029999999999999},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.4140000000000001,"5min":262.38099999999997,"15min":262.38099999999997},"last":0.379},{"interface":"front","average":{"1min":1.522,"5min":4.0780000000000003,"15min":1.7909999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.2850000000000001,"5min":261.488,"15min":261.488},"last":0.46500000000000002}]}]},{"osd":6,"up_from":304,"seq":1305670058099,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2567564,"kb_used_data":2393668,"kb_used_omap":13,"kb_used_meta":173874,"kb_avail":91177588,"statfs":{"total":95995035648,"available":93365850112,"internally_reserved":0,"allocated":2451116032,"data_stored":2449686744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":178047331},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":2.5619999999999998,"15min":2.9089999999999998},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":17.914999999999999,"5min":49.441000000000003,"15min":184.53200000000001},"last":0.26500000000000001},{"interface":"front","average":{"1min":5.1619999999999999,"5min":2.7789999999999999,"15min":3.194},"min":{"1min":0.42899999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.510999999999999,"5min":48.880000000000003,"15min":86.227999999999994},"last":0.42499999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":1.7549999999999999,"15min":2.7069999999999999},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.7290000000000001,"5min":9.9710000000000001,"15min":86.534000000000006},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.3799999999999999,"15min":2.5350000000000001},"min":{"1min":0.29799999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.9580000000000002,"5min":6.5419999999999998,"15min":86.144000000000005},"last":0.73999999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.0190000000000001,"15min":1.7849999999999999},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.371,"5min":10.371,"15min":10.371},"last":4.048},{"interface":"front","average":{"1min":3.387,"5min":2.0110000000000001,"15min":1.8759999999999999},"min":{"1min":0.38900000000000001,"5min":0.36499999999999999,"15min":0.307},"max":{"1min":10.388,"5min":10.388,"15min":10.388},"last":4.0670000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":3.371,"15min":2.6579999999999999},"min":{"1min":0.30299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.644,"5min":142.67599999999999,"15min":142.67599999999999},"last":9.6829999999999998},{"interface":"front","average":{"1min":4.0039999999999996,"5min":4.1360000000000001,"15min":3.0059999999999998},"min":{"1min":0.46000000000000002,"5min":0.34699999999999998,"15min":0.33500000000000002},"max":{"1min":10.478,"5min":142.541,"15min":142.541},"last":0.47899999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":2.532,"15min":1.516},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.23999999999999999},"max":{"1min":4.3710000000000004,"5min":128.11600000000001,"15min":128.11600000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":2.5960000000000001,"5min":2.964,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.26400000000000001},"max":{"1min":8.2449999999999992,"5min":128.095,"15min":128.095},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":4.4539999999999997,"15min":2.8300000000000001},"min":{"1min":0.35099999999999998,"5min":0.248,"15min":0.248},"max":{"1min":9.2560000000000002,"5min":130.91300000000001,"15min":130.91300000000001},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.83099999999999996,"5min":3.9950000000000001,"15min":2.4220000000000002},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1960000000000002,"5min":130.92599999999999,"15min":130.92599999999999},"last":0.99299999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.5029999999999999,"15min":1.325},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.5090000000000003,"5min":9.5090000000000003,"15min":9.5090000000000003},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.018,"5min":1.3169999999999999,"15min":1.4219999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":3.4660000000000002,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.78400000000000003}]}]},{"osd":1,"up_from":183,"seq":785979015400,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1649032,"kb_used_data":1023104,"kb_used_omap":6,"kb_used_meta":625913,"kb_avail":92096120,"statfs":{"total":95995035648,"available":94306426880,"internally_reserved":0,"allocated":1047658496,"data_stored":1045438632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":640934987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,0,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5339999999999998,"5min":4.4089999999999998,"15min":3.4399999999999999},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":96.323999999999998,"5min":101.27800000000001,"15min":101.27800000000001},"last":7.2290000000000001},{"interface":"front","average":{"1min":6.7939999999999996,"5min":4.3129999999999997,"15min":3.7469999999999999},"min":{"1min":0.26400000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":96.188999999999993,"5min":101.051,"15min":101.051},"last":2.02}]},{"osd":2,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":4.8289999999999997,"15min":4.0789999999999997},"min":{"1min":0.251,"5min":0.247,"15min":0.221},"max":{"1min":13.984999999999999,"5min":186.304,"15min":186.304},"last":0.27900000000000003},{"interface":"front","average":{"1min":2.8130000000000002,"5min":4.4989999999999997,"15min":3.3620000000000001},"min":{"1min":0.32300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":13.795999999999999,"5min":186.541,"15min":186.541},"last":6.1260000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.419,"15min":4.5129999999999999},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.744,"5min":15.191000000000001,"15min":211.405},"last":11.896000000000001},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2429999999999999,"15min":4.5700000000000003},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.246},"max":{"1min":6.915,"5min":14.609999999999999,"15min":210.988},"last":13.98}]},{"osd":4,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":3.3740000000000001,"15min":3.4900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29799999999999999},"max":{"1min":13.840999999999999,"5min":72.602000000000004,"15min":80.570999999999998},"last":0.53100000000000003},{"interface":"front","average":{"1min":2.2949999999999999,"5min":2.4889999999999999,"15min":2.9420000000000002},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.308},"max":{"1min":7.0209999999999999,"5min":73.454999999999998,"15min":80.75},"last":5.1269999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":3.7040000000000002,"15min":3.363},"min":{"1min":0.42699999999999999,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":7.976,"5min":173.18799999999999,"15min":173.18799999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":3.476,"5min":4.415,"15min":3.7759999999999998},"min":{"1min":0.42799999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":13.872,"5min":172.822,"15min":172.822},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:31:43 2023","interfaces":[{"interface":"back","average":{"1min":3.0760000000000001,"5min":2.8250000000000002,"15min":2.9990000000000001},"min":{"1min":0.377,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":13.912000000000001,"5min":54.198,"15min":54.198},"last":0.623},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.4710000000000001,"15min":2.0430000000000001},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.1210000000000004,"5min":8.5800000000000001,"15min":15.605},"last":0.50700000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:31:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.8169999999999999,"15min":2.0310000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":4.9139999999999997,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":0.56999999999999995},{"interface":"front","average":{"1min":2.484,"5min":3.512,"15min":4.5519999999999996},"min":{"1min":0.46400000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.2759999999999998,"5min":54.223999999999997,"15min":54.223999999999997},"last":5.2130000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893446,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4267244,"kb_used_data":3451876,"kb_used_omap":8,"kb_used_meta":815351,"kb_avail":89477908,"statfs":{"total":95995035648,"available":91625377792,"internally_reserved":0,"allocated":3534721024,"data_stored":3532642409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":834920020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":4.827,"15min":4.6230000000000002},"min":{"1min":0.25900000000000001,"5min":0.216,"15min":0.18099999999999999},"max":{"1min":12.516,"5min":141.52199999999999,"15min":141.52199999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":2.5270000000000001,"5min":4.8819999999999997,"15min":4.2990000000000004},"min":{"1min":0.24099999999999999,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":8.2260000000000009,"5min":141.62799999999999,"15min":141.62799999999999},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":6.968,"5min":3.073,"15min":2.5659999999999998},"min":{"1min":0.33600000000000002,"5min":0.23400000000000001,"15min":0.20200000000000001},"max":{"1min":66.132999999999996,"5min":66.132999999999996,"15min":66.132999999999996},"last":2.403},{"interface":"front","average":{"1min":8.0739999999999998,"5min":2.7440000000000002,"15min":2.7240000000000002},"min":{"1min":0.26300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":68.296999999999997,"5min":68.296999999999997,"15min":68.296999999999997},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":5.6840000000000002,"15min":5.4950000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.722,"5min":162.154,"15min":197.78899999999999},"last":4.2649999999999997},{"interface":"front","average":{"1min":2.5139999999999998,"5min":5.1989999999999998,"15min":5.7460000000000004},"min":{"1min":0.40699999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":12.866,"5min":162.215,"15min":200.58799999999999},"last":4.7439999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":4.7480000000000002,"5min":2.6800000000000002,"15min":2.0979999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":36.393000000000001,"5min":36.393000000000001,"15min":36.393000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":3.734,"5min":2.5960000000000001,"15min":2.1040000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":36.219999999999999,"5min":36.219999999999999,"15min":36.219999999999999},"last":7.6870000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4119999999999999,"5min":2.4420000000000002,"15min":2.3039999999999998},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":33.158999999999999,"5min":44.57,"15min":64.349999999999994},"last":0.64000000000000001},{"interface":"front","average":{"1min":4.5940000000000003,"5min":2.8919999999999999,"15min":2.4020000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.312},"max":{"1min":33.140000000000001,"5min":44.588000000000001,"15min":44.588000000000001},"last":0.65200000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1529999999999996,"5min":3.0150000000000001,"15min":2.7400000000000002},"min":{"1min":0.505,"5min":0.40500000000000003,"15min":0.379},"max":{"1min":41.317999999999998,"5min":41.317999999999998,"15min":41.317999999999998},"last":5.7649999999999997},{"interface":"front","average":{"1min":4.3639999999999999,"5min":3.0630000000000002,"15min":3.0630000000000002},"min":{"1min":0.39000000000000001,"5min":0.377,"15min":0.371},"max":{"1min":41.195,"5min":41.195,"15min":41.195},"last":7.7309999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.552,"15min":2.3799999999999999},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.318},"max":{"1min":12.593999999999999,"5min":36.848999999999997,"15min":36.848999999999997},"last":4.7060000000000004},{"interface":"front","average":{"1min":3.4689999999999999,"5min":3.3849999999999998,"15min":3.8969999999999998},"min":{"1min":0.54200000000000004,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":11.214,"5min":36.947000000000003,"15min":36.947000000000003},"last":0.51600000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566396,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2775320,"kb_used_data":2057488,"kb_used_omap":11,"kb_used_meta":717812,"kb_avail":90969832,"statfs":{"total":95995035648,"available":93153107968,"internally_reserved":0,"allocated":2106867712,"data_stored":2104805538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":735040272},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0960000000000001,"5min":2.7290000000000001,"15min":5.157},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.17199999999999999},"max":{"1min":12.43,"5min":70.307000000000002,"15min":235.517},"last":0.248},{"interface":"front","average":{"1min":2.3069999999999999,"5min":2.359,"15min":5.1180000000000003},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5380000000000003,"5min":70.286000000000001,"15min":235.33799999999999},"last":4.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:31:25 2023","interfaces":[{"interface":"back","average":{"1min":4.4379999999999997,"5min":3.2109999999999999,"15min":3.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":65.906999999999996,"5min":65.906999999999996,"15min":71.819000000000003},"last":3.4319999999999999},{"interface":"front","average":{"1min":4.391,"5min":3.0129999999999999,"15min":3.3180000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":65.222999999999999,"5min":65.222999999999999,"15min":71.805000000000007},"last":4.5389999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:31:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2309999999999999,"5min":3.1880000000000002,"15min":4.4589999999999996},"min":{"1min":0.216,"5min":0.216,"15min":0.216},"max":{"1min":18.606999999999999,"5min":75.701999999999998,"15min":77.031999999999996},"last":0.65200000000000002},{"interface":"front","average":{"1min":2.0920000000000001,"5min":2.8639999999999999,"15min":4.6280000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":8.3450000000000006,"5min":75.388999999999996,"15min":76.742000000000004},"last":5.1100000000000003}]},{"osd":4,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":5.1719999999999997,"5min":2.262,"15min":1.9019999999999999},"min":{"1min":0.29799999999999999,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":63.747999999999998,"5min":63.747999999999998,"15min":63.747999999999998},"last":4.4589999999999996},{"interface":"front","average":{"1min":6.2210000000000001,"5min":2.7160000000000002,"15min":2.0590000000000002},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":63.954000000000001,"5min":63.954000000000001,"15min":63.954000000000001},"last":0.59799999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":3.3919999999999999,"15min":2.399},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":6.2460000000000004,"5min":107.021,"15min":107.021},"last":3.48},{"interface":"front","average":{"1min":2.262,"5min":3.2629999999999999,"15min":2.3559999999999999},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.317},"max":{"1min":8.4559999999999995,"5min":107.136,"15min":107.136},"last":4.5060000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:31:38 2023","interfaces":[{"interface":"back","average":{"1min":9.9670000000000005,"5min":3.5009999999999999,"15min":2.6000000000000001},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":156.16,"5min":156.16,"15min":156.16},"last":3.5049999999999999},{"interface":"front","average":{"1min":11.045999999999999,"5min":3.625,"15min":2.3719999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":155.226,"5min":155.226,"15min":155.226},"last":0.627}]},{"osd":7,"last update":"Sat Sep 23 15:31:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":3.5499999999999998,"15min":2.843},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":12.282,"5min":112.70699999999999,"15min":112.70699999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":2.2570000000000001,"5min":3.661,"15min":3.423},"min":{"1min":0.438,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.5519999999999996,"5min":109.402,"15min":109.402},"last":3.5379999999999998}]}]},{"osd":3,"up_from":110,"seq":472446402868,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2530032,"kb_used_data":1714664,"kb_used_omap":3,"kb_used_meta":815356,"kb_avail":91215120,"statfs":{"total":95995035648,"available":93404282880,"internally_reserved":0,"allocated":1755815936,"data_stored":1753732860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":834924713},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":8.9039999999999999,"5min":5.4000000000000004,"15min":4.056},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.183},"max":{"1min":141.934,"5min":141.934,"15min":141.934},"last":2.1499999999999999},{"interface":"front","average":{"1min":9.2690000000000001,"5min":5.9290000000000003,"15min":4.54},"min":{"1min":0.28899999999999998,"5min":0.189,"15min":0.184},"max":{"1min":141.72800000000001,"5min":141.72800000000001,"15min":141.72800000000001},"last":9.0830000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":4.1909999999999998,"15min":4.4039999999999999},"min":{"1min":0.28999999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.96,"5min":121.26900000000001,"15min":126.875},"last":1.6579999999999999},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.6130000000000004,"15min":4.8150000000000004},"min":{"1min":0.30099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.401999999999999,"5min":121.012,"15min":128.249},"last":1.9299999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":11.366,"5min":3.5179999999999998,"15min":2.8900000000000001},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":144.35599999999999,"5min":144.35599999999999,"15min":144.35599999999999},"last":7.601},{"interface":"front","average":{"1min":11.470000000000001,"5min":3.988,"15min":3.7519999999999998},"min":{"1min":0.26100000000000001,"5min":0.246,"15min":0.246},"max":{"1min":144.494,"5min":144.494,"15min":144.494},"last":7.8970000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.712,"15min":1.9239999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":4.6040000000000001,"5min":17.027999999999999,"15min":36.551000000000002},"last":1.0580000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.5429999999999999,"15min":1.754},"min":{"1min":0.373,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":5.3360000000000003,"5min":7.6630000000000003,"15min":13.115},"last":1.635}]},{"osd":5,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.3620000000000001,"5min":3.9889999999999999,"15min":3.282},"min":{"1min":0.49199999999999999,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":122.452,"5min":122.452,"15min":148.619},"last":1.123},{"interface":"front","average":{"1min":6.9669999999999996,"5min":3.6619999999999999,"15min":3.004},"min":{"1min":0.42299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":122.33499999999999,"5min":122.33499999999999,"15min":148.56999999999999},"last":1.6080000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:31:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":2.4870000000000001,"15min":2.637},"min":{"1min":0.36399999999999999,"5min":0.34999999999999998,"15min":0.32700000000000001},"max":{"1min":7.5359999999999996,"5min":69.721999999999994,"15min":84.665000000000006},"last":1.575},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.867,"15min":3.359},"min":{"1min":0.46800000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.1399999999999997,"5min":69.852999999999994,"15min":84.897999999999996},"last":1.0920000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:31:50 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":7.1130000000000004,"15min":10.536},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":9.3729999999999993,"5min":216.56899999999999,"15min":216.56899999999999},"last":0.95299999999999996},{"interface":"front","average":{"1min":3.0960000000000001,"5min":7.9459999999999997,"15min":11.596},"min":{"1min":0.58099999999999996,"5min":0.432,"15min":0.432},"max":{"1min":13.43,"5min":216.548,"15min":216.548},"last":1.163}]}]},{"osd":4,"up_from":31,"seq":133143986635,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3206928,"kb_used_data":2748936,"kb_used_omap":0,"kb_used_meta":457856,"kb_avail":90538224,"statfs":{"total":95995035648,"available":92711141376,"internally_reserved":0,"allocated":2814910464,"data_stored":2813477137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":468844544},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,2,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.8519999999999999,"5min":2.4670000000000001,"15min":4.0609999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":33.500999999999998,"5min":36.865000000000002,"15min":162.75999999999999},"last":2.1379999999999999},{"interface":"front","average":{"1min":3.9119999999999999,"5min":2.601,"15min":4.0970000000000004},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":33.021999999999998,"5min":37.081000000000003,"15min":162.34800000000001},"last":0.26700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":5.0419999999999998,"15min":3.548},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.245},"max":{"1min":6.8789999999999996,"5min":144.76900000000001,"15min":144.76900000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.6499999999999999,"5min":5.3399999999999999,"15min":3.7559999999999998},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":6.6440000000000001,"5min":144.58799999999999,"15min":144.58799999999999},"last":2.2519999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9580000000000002,"5min":1.8149999999999999,"15min":2.6379999999999999},"min":{"1min":0.433,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":30.489000000000001,"5min":30.489000000000001,"15min":78.751000000000005},"last":0.35899999999999999},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.637,"15min":2.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.315,"15min":0.28999999999999998},"max":{"1min":30.524000000000001,"5min":30.524000000000001,"15min":77.930999999999997},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:31:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":2.698,"15min":3.7709999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":8.3320000000000007,"5min":96.587000000000003,"15min":156.46000000000001},"last":2.3969999999999998},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.8069999999999999,"15min":3.8879999999999999},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.6920000000000002,"5min":96.784999999999997,"15min":156.74299999999999},"last":2.7000000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3720000000000001,"15min":1.069},"min":{"1min":0.33000000000000002,"5min":0.26500000000000001,"15min":0.221},"max":{"1min":8.3829999999999991,"5min":40.685000000000002,"15min":40.685000000000002},"last":2.1890000000000001},{"interface":"front","average":{"1min":2.508,"5min":1.6140000000000001,"15min":1.2050000000000001},"min":{"1min":0.29099999999999998,"5min":0.27500000000000002,"15min":0.246},"max":{"1min":12.882999999999999,"5min":38.439,"15min":38.439},"last":4.7679999999999998}]},{"osd":6,"last update":"Sat Sep 23 15:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":3.7469999999999999,"15min":2.1179999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.28999999999999998},"max":{"1min":9.6280000000000001,"5min":228.93199999999999,"15min":228.93199999999999},"last":2.6709999999999998},{"interface":"front","average":{"1min":2.8969999999999998,"5min":3.8359999999999999,"15min":2.5699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.446,"5min":232.54900000000001,"15min":232.54900000000001},"last":3.23}]},{"osd":7,"last update":"Sat Sep 23 15:31:44 2023","interfaces":[{"interface":"back","average":{"1min":11.157999999999999,"5min":3.2000000000000002,"15min":2},"min":{"1min":0.36399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":183.94800000000001,"5min":183.94800000000001,"15min":183.94800000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":10.839,"5min":3.0590000000000002,"15min":1.724},"min":{"1min":0.41899999999999998,"5min":0.314,"15min":0.314},"max":{"1min":183.72399999999999,"5min":183.72399999999999,"15min":183.72399999999999},"last":7.1820000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724998,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2249152,"kb_used_data":1735288,"kb_used_omap":0,"kb_used_meta":513792,"kb_avail":91496000,"statfs":{"total":95995035648,"available":93691904000,"internally_reserved":0,"allocated":1776934912,"data_stored":1775514182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":526123008},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:35 2023","interfaces":[{"interface":"back","average":{"1min":5.7169999999999996,"5min":10.461,"15min":6.4630000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":54.673999999999999,"5min":397.84199999999998,"15min":397.84199999999998},"last":6.609},{"interface":"front","average":{"1min":5.8179999999999996,"5min":10.352,"15min":6.3979999999999997},"min":{"1min":0.28799999999999998,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":54.338999999999999,"5min":397.28100000000001,"15min":397.28100000000001},"last":1.24}]},{"osd":1,"last update":"Sat Sep 23 15:31:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":4.0129999999999999,"15min":3.9550000000000001},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":13.972,"5min":193.66499999999999,"15min":193.66499999999999},"last":1.897},{"interface":"front","average":{"1min":2.7040000000000002,"5min":4.1900000000000004,"15min":3.673},"min":{"1min":0.33200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.850000000000001,"5min":193.52099999999999,"15min":193.52099999999999},"last":1.2949999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":3.3679999999999999,"15min":3.1040000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.25},"max":{"1min":15.904,"5min":97.126000000000005,"15min":193.00999999999999},"last":2.2709999999999999},{"interface":"front","average":{"1min":2.363,"5min":3.4740000000000002,"15min":3.3279999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27900000000000003},"max":{"1min":19.812999999999999,"5min":96.516000000000005,"15min":193.15700000000001},"last":3.641}]},{"osd":3,"last update":"Sat Sep 23 15:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":3.4020000000000001,"15min":5.8399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":14.082000000000001,"5min":150.75899999999999,"15min":266.70299999999997},"last":1.3939999999999999},{"interface":"front","average":{"1min":2.2080000000000002,"5min":3.3069999999999999,"15min":6.0110000000000001},"min":{"1min":0.313,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":15.26,"5min":150.62299999999999,"15min":266.80399999999997},"last":2.4129999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:31:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":1.294,"15min":1.214},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":14.068,"5min":14.068,"15min":14.068},"last":0.438},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.2829999999999999,"15min":1.2330000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":19.879999999999999,"5min":19.879999999999999,"15min":19.879999999999999},"last":0.75900000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.7310000000000001,"15min":2.0569999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.4139999999999997,"5min":35.667000000000002,"15min":35.667000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.734,"5min":1.7,"15min":2.0830000000000002},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.1829999999999998,"5min":36.140999999999998,"15min":36.140999999999998},"last":1.3420000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.474,"15min":1.3460000000000001},"min":{"1min":0.34799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.609,"5min":16.065999999999999,"15min":16.065999999999999},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.631,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.3320000000000007,"5min":19.908000000000001,"15min":19.908000000000001},"last":0.61399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3508670464,"data_stored":3508666392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1036058624,"data_stored":1036058624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2099183616,"data_stored":2099183616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1736900608,"data_stored":1736900608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2802388992,"data_stored":2802384920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1760624640,"data_stored":1760624640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2442268672,"data_stored":2442264600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1389756416,"data_stored":1389756416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:31:59.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:32:00.040 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:00.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:00.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:00.165 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:00.253 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:00.578 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:00 smithi017 ceph-mon[215646]: pgmap v1296: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T15:32:00.661 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:00.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:00.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:00.766 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:00.807 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:00.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:00 smithi027 ceph-mon[195233]: pgmap v1296: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T15:32:00.855 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:00 smithi017 ceph-mon[222878]: pgmap v1296: 119 pgs: 119 active+clean; 5.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2023-09-23T15:32:00.883 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:00.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:00.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:00.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:01.064 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:01.091 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:01.130 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:01.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:01.136 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:01.236 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:01.327 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:01.492 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:01.615 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:01.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:01.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:01.629 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:32:01.629 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:32:01.726 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:01.810 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:01.985 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:01.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:02.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:02.147 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:02.229 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:02.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:02.340 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:02.401 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:02.630 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:32:02.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:02 smithi017 ceph-mon[215646]: pgmap v1297: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:02.667 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:02.668 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:02 smithi017 ceph-mon[222878]: pgmap v1297: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:02.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:02.721 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:02.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:02.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:02.835 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:02.888 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:02 smithi027 ceph-mon[195233]: pgmap v1297: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:03.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:03.552 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.620 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[215646]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.670 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:03.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:03.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:03.776 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:03.824 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:03.937 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:03.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:03.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:03.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T15:32:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:03 smithi017 ceph-mon[222878]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:03.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:03.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:03.947 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1601306023' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:03.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: osdmap e440: 8 total, 8 up, 8 in 2023-09-23T15:32:03.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: failed to encode map e440 with expected crc 2023-09-23T15:32:03.948 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:03 smithi027 ceph-mon[195233]: failed to encode map e440 with expected crc 2023-09-23T15:32:04.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:04.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:04.883 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:04.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:04.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:04.896 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:04 smithi017 ceph-mon[215646]: pgmap v1299: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:04.896 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:04 smithi017 ceph-mon[222878]: pgmap v1299: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:04.994 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:04 smithi027 ceph-mon[195233]: pgmap v1299: 119 pgs: 119 active+clean; 5.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:32:05.108 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:05.213 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:05.260 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:05.260 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:05.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:05.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:05.311 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:05.348 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:05.364 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:05.384 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:05.411 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:05.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:05.481 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:05.497 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:05.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:05.516 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:05.582 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:05.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.590 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:32:05.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:05.652 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:05.759 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:05.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:05.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:05.771 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2778024160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:05.828 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2778024160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:05.829 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2778024160' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:05.868 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:05.964 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:06.083 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:06.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:06.093 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:06.183 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:32:06.193 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:06.211 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:32:06.252 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:06.319 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:06.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:06.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:06.438 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:06.489 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:06.562 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:06.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:06.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:06.678 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:06.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:32:06.765 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:06.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:06 smithi027 ceph-mon[195233]: pgmap v1300: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-09-23T15:32:06.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:06 smithi017 ceph-mon[222878]: pgmap v1300: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-09-23T15:32:06.914 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:06 smithi017 ceph-mon[215646]: pgmap v1300: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 803 op/s 2023-09-23T15:32:07.640 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:07.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:07.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:07.749 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:07.816 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1300,"stamp":"2023-09-23T15:32:04.977731+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5084813259,"num_objects":77544,"num_object_clones":0,"num_object_copies":232632,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77544,"num_whiteouts":0,"num_read":5996,"num_read_kb":13213,"num_write":135609,"num_write_kb":6803638,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":157,"num_bytes_recovered":4466766,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":115326,"ondisk_log_size":115326,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":357,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19292736,"kb_used_data":15017536,"kb_used_omap":57,"kb_used_meta":4274822,"kb_avail":730668480,"statfs":{"total":767960285184,"available":748204523520,"internally_reserved":0,"allocated":15377956864,"data_stored":15363764467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4377417753},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-526712832,"num_objects":-8037,"num_object_clones":0,"num_object_copies":-24111,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8037,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8037,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004193"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":787,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900423+0000","last_change":"2023-09-23T15:28:39.656073+0000","last_active":"2023-09-23T15:31:52.900423+0000","last_peered":"2023-09-23T15:31:52.900423+0000","last_clean":"2023-09-23T15:31:52.900423+0000","last_became_active":"2023-09-23T15:28:39.655630+0000","last_became_peered":"2023-09-23T15:28:39.655630+0000","last_unstale":"2023-09-23T15:31:52.900423+0000","last_undegraded":"2023-09-23T15:31:52.900423+0000","last_fullsized":"2023-09-23T15:31:52.900423+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:07.786522+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:07.786522+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:25:10.734918+0000","scrub_duration":0.0018254809999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894031+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:31:52.894031+0000","last_peered":"2023-09-23T15:31:52.894031+0000","last_clean":"2023-09-23T15:31:52.894031+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:31:52.894031+0000","last_undegraded":"2023-09-23T15:31:52.894031+0000","last_fullsized":"2023-09-23T15:31:52.894031+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":345,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901260+0000","last_change":"2023-09-23T15:30:26.692661+0000","last_active":"2023-09-23T15:31:52.901260+0000","last_peered":"2023-09-23T15:31:52.901260+0000","last_clean":"2023-09-23T15:31:52.901260+0000","last_became_active":"2023-09-23T15:30:26.692331+0000","last_became_peered":"2023-09-23T15:30:26.692331+0000","last_unstale":"2023-09-23T15:31:52.901260+0000","last_undegraded":"2023-09-23T15:31:52.901260+0000","last_fullsized":"2023-09-23T15:31:52.901260+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:03.180374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":521,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901209+0000","last_change":"2023-09-23T15:28:55.922068+0000","last_active":"2023-09-23T15:31:52.901209+0000","last_peered":"2023-09-23T15:31:52.901209+0000","last_clean":"2023-09-23T15:31:52.901209+0000","last_became_active":"2023-09-23T15:28:39.666685+0000","last_became_peered":"2023-09-23T15:28:39.666685+0000","last_unstale":"2023-09-23T15:31:52.901209+0000","last_undegraded":"2023-09-23T15:31:52.901209+0000","last_fullsized":"2023-09-23T15:31:52.901209+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:55.921999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:55.921999+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:38.044680+0000","scrub_duration":0.0074994720000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":364,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693598+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:31:48.693598+0000","last_peered":"2023-09-23T15:31:48.693598+0000","last_clean":"2023-09-23T15:31:48.693598+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:31:48.693598+0000","last_undegraded":"2023-09-23T15:31:48.693598+0000","last_fullsized":"2023-09-23T15:31:48.693598+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":436,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.910872+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:31:52.910872+0000","last_peered":"2023-09-23T15:31:52.910872+0000","last_clean":"2023-09-23T15:31:52.910872+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:31:52.910872+0000","last_undegraded":"2023-09-23T15:31:52.910872+0000","last_fullsized":"2023-09-23T15:31:52.910872+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":575,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894186+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:31:52.894186+0000","last_peered":"2023-09-23T15:31:52.894186+0000","last_clean":"2023-09-23T15:31:52.894186+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:31:52.894186+0000","last_undegraded":"2023-09-23T15:31:52.894186+0000","last_fullsized":"2023-09-23T15:31:52.894186+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":376,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902958+0000","last_change":"2023-09-23T15:30:26.692470+0000","last_active":"2023-09-23T15:31:52.902958+0000","last_peered":"2023-09-23T15:31:52.902958+0000","last_clean":"2023-09-23T15:31:52.902958+0000","last_became_active":"2023-09-23T15:30:26.692148+0000","last_became_peered":"2023-09-23T15:30:26.692148+0000","last_unstale":"2023-09-23T15:31:52.902958+0000","last_undegraded":"2023-09-23T15:31:52.902958+0000","last_fullsized":"2023-09-23T15:31:52.902958+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:32:36.258289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":390,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.899143+0000","last_change":"2023-09-23T15:22:26.307623+0000","last_active":"2023-09-23T15:31:52.899143+0000","last_peered":"2023-09-23T15:31:52.899143+0000","last_clean":"2023-09-23T15:31:52.899143+0000","last_became_active":"2023-09-23T15:22:26.307335+0000","last_became_peered":"2023-09-23T15:22:26.307335+0000","last_unstale":"2023-09-23T15:31:52.899143+0000","last_undegraded":"2023-09-23T15:31:52.899143+0000","last_fullsized":"2023-09-23T15:31:52.899143+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:47.201651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":708,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897939+0000","last_change":"2023-09-23T15:28:39.659556+0000","last_active":"2023-09-23T15:31:52.897939+0000","last_peered":"2023-09-23T15:31:52.897939+0000","last_clean":"2023-09-23T15:31:52.897939+0000","last_became_active":"2023-09-23T15:28:39.659112+0000","last_became_peered":"2023-09-23T15:28:39.659112+0000","last_unstale":"2023-09-23T15:31:52.897939+0000","last_undegraded":"2023-09-23T15:31:52.897939+0000","last_fullsized":"2023-09-23T15:31:52.897939+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:40.874644+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":390,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.908507+0000","last_change":"2023-09-23T15:06:16.126708+0000","last_active":"2023-09-23T15:31:52.908507+0000","last_peered":"2023-09-23T15:31:52.908507+0000","last_clean":"2023-09-23T15:31:52.908507+0000","last_became_active":"2023-09-23T15:06:16.126377+0000","last_became_peered":"2023-09-23T15:06:16.126377+0000","last_unstale":"2023-09-23T15:31:52.908507+0000","last_undegraded":"2023-09-23T15:31:52.908507+0000","last_fullsized":"2023-09-23T15:31:52.908507+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:28.939343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":372,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695531+0000","last_change":"2023-09-23T15:30:26.706619+0000","last_active":"2023-09-23T15:31:48.695531+0000","last_peered":"2023-09-23T15:31:48.695531+0000","last_clean":"2023-09-23T15:31:48.695531+0000","last_became_active":"2023-09-23T15:30:26.706343+0000","last_became_peered":"2023-09-23T15:30:26.706343+0000","last_unstale":"2023-09-23T15:31:48.695531+0000","last_undegraded":"2023-09-23T15:31:48.695531+0000","last_fullsized":"2023-09-23T15:31:48.695531+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:47.811881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":850,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893240+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:31:52.893240+0000","last_peered":"2023-09-23T15:31:52.893240+0000","last_clean":"2023-09-23T15:31:52.893240+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:31:52.893240+0000","last_undegraded":"2023-09-23T15:31:52.893240+0000","last_fullsized":"2023-09-23T15:31:52.893240+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":456,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902304+0000","last_change":"2023-09-23T15:27:41.416575+0000","last_active":"2023-09-23T15:31:52.902304+0000","last_peered":"2023-09-23T15:31:52.902304+0000","last_clean":"2023-09-23T15:31:52.902304+0000","last_became_active":"2023-09-23T15:27:41.416235+0000","last_became_peered":"2023-09-23T15:27:41.416235+0000","last_unstale":"2023-09-23T15:31:52.902304+0000","last_undegraded":"2023-09-23T15:31:52.902304+0000","last_fullsized":"2023-09-23T15:31:52.902304+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:57.901092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":346,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898593+0000","last_change":"2023-09-23T15:30:26.703714+0000","last_active":"2023-09-23T15:31:52.898593+0000","last_peered":"2023-09-23T15:31:52.898593+0000","last_clean":"2023-09-23T15:31:52.898593+0000","last_became_active":"2023-09-23T15:30:26.703402+0000","last_became_peered":"2023-09-23T15:30:26.703402+0000","last_unstale":"2023-09-23T15:31:52.898593+0000","last_undegraded":"2023-09-23T15:31:52.898593+0000","last_fullsized":"2023-09-23T15:31:52.898593+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:48.014628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":542,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901499+0000","last_change":"2023-09-23T15:28:59.984242+0000","last_active":"2023-09-23T15:31:52.901499+0000","last_peered":"2023-09-23T15:31:52.901499+0000","last_clean":"2023-09-23T15:31:52.901499+0000","last_became_active":"2023-09-23T15:28:39.668715+0000","last_became_peered":"2023-09-23T15:28:39.668715+0000","last_unstale":"2023-09-23T15:31:52.901499+0000","last_undegraded":"2023-09-23T15:31:52.901499+0000","last_fullsized":"2023-09-23T15:31:52.901499+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:59.984131+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:59.984131+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:34.209362+0000","scrub_duration":0.015840911999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":185,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898125+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:31:52.898125+0000","last_peered":"2023-09-23T15:31:52.898125+0000","last_clean":"2023-09-23T15:31:52.898125+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:31:52.898125+0000","last_undegraded":"2023-09-23T15:31:52.898125+0000","last_fullsized":"2023-09-23T15:31:52.898125+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":333,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.696788+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:31:48.696788+0000","last_peered":"2023-09-23T15:31:48.696788+0000","last_clean":"2023-09-23T15:31:48.696788+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:31:48.696788+0000","last_undegraded":"2023-09-23T15:31:48.696788+0000","last_fullsized":"2023-09-23T15:31:48.696788+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":590,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694501+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:31:48.694501+0000","last_peered":"2023-09-23T15:31:48.694501+0000","last_clean":"2023-09-23T15:31:48.694501+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:31:48.694501+0000","last_undegraded":"2023-09-23T15:31:48.694501+0000","last_fullsized":"2023-09-23T15:31:48.694501+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":106,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894812+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:31:52.894812+0000","last_peered":"2023-09-23T15:31:52.894812+0000","last_clean":"2023-09-23T15:31:52.894812+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:31:52.894812+0000","last_undegraded":"2023-09-23T15:31:52.894812+0000","last_fullsized":"2023-09-23T15:31:52.894812+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894883+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:31:52.894883+0000","last_peered":"2023-09-23T15:31:52.894883+0000","last_clean":"2023-09-23T15:31:52.894883+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:31:52.894883+0000","last_undegraded":"2023-09-23T15:31:52.894883+0000","last_fullsized":"2023-09-23T15:31:52.894883+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":751,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902971+0000","last_change":"2023-09-23T15:16:46.272529+0000","last_active":"2023-09-23T15:31:52.902971+0000","last_peered":"2023-09-23T15:31:52.902971+0000","last_clean":"2023-09-23T15:31:52.902971+0000","last_became_active":"2023-09-23T15:16:46.272092+0000","last_became_peered":"2023-09-23T15:16:46.272092+0000","last_unstale":"2023-09-23T15:31:52.902971+0000","last_undegraded":"2023-09-23T15:31:52.902971+0000","last_fullsized":"2023-09-23T15:31:52.902971+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:01:14.381625+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":189,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897683+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:31:52.897683+0000","last_peered":"2023-09-23T15:31:52.897683+0000","last_clean":"2023-09-23T15:31:52.897683+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:31:52.897683+0000","last_undegraded":"2023-09-23T15:31:52.897683+0000","last_fullsized":"2023-09-23T15:31:52.897683+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":394,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693839+0000","last_change":"2023-09-23T15:30:26.705220+0000","last_active":"2023-09-23T15:31:48.693839+0000","last_peered":"2023-09-23T15:31:48.693839+0000","last_clean":"2023-09-23T15:31:48.693839+0000","last_became_active":"2023-09-23T15:30:26.704922+0000","last_became_peered":"2023-09-23T15:30:26.704922+0000","last_unstale":"2023-09-23T15:31:48.693839+0000","last_undegraded":"2023-09-23T15:31:48.693839+0000","last_fullsized":"2023-09-23T15:31:48.693839+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:03:25.436067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":814,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900669+0000","last_change":"2023-09-23T15:27:41.402702+0000","last_active":"2023-09-23T15:31:52.900669+0000","last_peered":"2023-09-23T15:31:52.900669+0000","last_clean":"2023-09-23T15:31:52.900669+0000","last_became_active":"2023-09-23T15:27:41.402368+0000","last_became_peered":"2023-09-23T15:27:41.402368+0000","last_unstale":"2023-09-23T15:31:52.900669+0000","last_undegraded":"2023-09-23T15:31:52.900669+0000","last_fullsized":"2023-09-23T15:31:52.900669+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:32.445525+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":434,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.907687+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:31:52.907687+0000","last_peered":"2023-09-23T15:31:52.907687+0000","last_clean":"2023-09-23T15:31:52.907687+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:31:52.907687+0000","last_undegraded":"2023-09-23T15:31:52.907687+0000","last_fullsized":"2023-09-23T15:31:52.907687+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":441,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901190+0000","last_change":"2023-09-23T15:27:41.435902+0000","last_active":"2023-09-23T15:31:52.901190+0000","last_peered":"2023-09-23T15:31:52.901190+0000","last_clean":"2023-09-23T15:31:52.901190+0000","last_became_active":"2023-09-23T15:27:41.435594+0000","last_became_peered":"2023-09-23T15:27:41.435594+0000","last_unstale":"2023-09-23T15:31:52.901190+0000","last_undegraded":"2023-09-23T15:31:52.901190+0000","last_fullsized":"2023-09-23T15:31:52.901190+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:58:40.063101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":766,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695071+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:31:48.695071+0000","last_peered":"2023-09-23T15:31:48.695071+0000","last_clean":"2023-09-23T15:31:48.695071+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:31:48.695071+0000","last_undegraded":"2023-09-23T15:31:48.695071+0000","last_fullsized":"2023-09-23T15:31:48.695071+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":183,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898780+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:31:52.898780+0000","last_peered":"2023-09-23T15:31:52.898780+0000","last_clean":"2023-09-23T15:31:52.898780+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:31:52.898780+0000","last_undegraded":"2023-09-23T15:31:52.898780+0000","last_fullsized":"2023-09-23T15:31:52.898780+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":423,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.910892+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:31:52.910892+0000","last_peered":"2023-09-23T15:31:52.910892+0000","last_clean":"2023-09-23T15:31:52.910892+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:31:52.910892+0000","last_undegraded":"2023-09-23T15:31:52.910892+0000","last_fullsized":"2023-09-23T15:31:52.910892+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":755,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902004+0000","last_change":"2023-09-23T15:22:26.307902+0000","last_active":"2023-09-23T15:31:52.902004+0000","last_peered":"2023-09-23T15:31:52.902004+0000","last_clean":"2023-09-23T15:31:52.902004+0000","last_became_active":"2023-09-23T15:22:26.307205+0000","last_became_peered":"2023-09-23T15:22:26.307205+0000","last_unstale":"2023-09-23T15:31:52.902004+0000","last_undegraded":"2023-09-23T15:31:52.902004+0000","last_fullsized":"2023-09-23T15:31:52.902004+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:34.783976+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":376,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894265+0000","last_change":"2023-09-23T15:22:26.307690+0000","last_active":"2023-09-23T15:31:52.894265+0000","last_peered":"2023-09-23T15:31:52.894265+0000","last_clean":"2023-09-23T15:31:52.894265+0000","last_became_active":"2023-09-23T15:22:26.307350+0000","last_became_peered":"2023-09-23T15:22:26.307350+0000","last_unstale":"2023-09-23T15:31:52.894265+0000","last_undegraded":"2023-09-23T15:31:52.894265+0000","last_fullsized":"2023-09-23T15:31:52.894265+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:05.215963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":396,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898809+0000","last_change":"2023-09-23T15:30:26.706427+0000","last_active":"2023-09-23T15:31:52.898809+0000","last_peered":"2023-09-23T15:31:52.898809+0000","last_clean":"2023-09-23T15:31:52.898809+0000","last_became_active":"2023-09-23T15:30:26.706086+0000","last_became_peered":"2023-09-23T15:30:26.706086+0000","last_unstale":"2023-09-23T15:31:52.898809+0000","last_undegraded":"2023-09-23T15:31:52.898809+0000","last_fullsized":"2023-09-23T15:31:52.898809+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:42.578372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.6","version":"440'8351","reported_seq":8385,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:03.950220+0000","last_change":"2023-09-23T15:29:48.688598+0000","last_active":"2023-09-23T15:32:03.950220+0000","last_peered":"2023-09-23T15:32:03.950220+0000","last_clean":"2023-09-23T15:32:03.950220+0000","last_became_active":"2023-09-23T15:29:48.688227+0000","last_became_peered":"2023-09-23T15:29:48.688227+0000","last_unstale":"2023-09-23T15:32:03.950220+0000","last_undegraded":"2023-09-23T15:32:03.950220+0000","last_fullsized":"2023-09-23T15:32:03.950220+0000","mapping_epoch":413,"log_start":"417'600","ondisk_log_start":"417'600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":7751,"log_dups_size":0,"ondisk_log_size":7751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:04.960560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317259776,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8351,"num_write_kb":422144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":736,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898213+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:31:52.898213+0000","last_peered":"2023-09-23T15:31:52.898213+0000","last_clean":"2023-09-23T15:31:52.898213+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:31:52.898213+0000","last_undegraded":"2023-09-23T15:31:52.898213+0000","last_fullsized":"2023-09-23T15:31:52.898213+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":179,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898904+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:31:52.898904+0000","last_peered":"2023-09-23T15:31:52.898904+0000","last_clean":"2023-09-23T15:31:52.898904+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:31:52.898904+0000","last_undegraded":"2023-09-23T15:31:52.898904+0000","last_fullsized":"2023-09-23T15:31:52.898904+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":448,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900513+0000","last_change":"2023-09-23T15:27:41.402338+0000","last_active":"2023-09-23T15:31:52.900513+0000","last_peered":"2023-09-23T15:31:52.900513+0000","last_clean":"2023-09-23T15:31:52.900513+0000","last_became_active":"2023-09-23T15:27:41.401830+0000","last_became_peered":"2023-09-23T15:27:41.401830+0000","last_unstale":"2023-09-23T15:31:52.900513+0000","last_undegraded":"2023-09-23T15:31:52.900513+0000","last_fullsized":"2023-09-23T15:31:52.900513+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:16.413863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"440'8245","reported_seq":8280,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.580866+0000","last_change":"2023-09-23T15:29:48.696982+0000","last_active":"2023-09-23T15:32:04.580866+0000","last_peered":"2023-09-23T15:32:04.580866+0000","last_clean":"2023-09-23T15:32:04.580866+0000","last_became_active":"2023-09-23T15:29:48.696666+0000","last_became_peered":"2023-09-23T15:29:48.696666+0000","last_unstale":"2023-09-23T15:32:04.580866+0000","last_undegraded":"2023-09-23T15:32:04.580866+0000","last_fullsized":"2023-09-23T15:32:04.580866+0000","mapping_epoch":413,"log_start":"418'1800","ondisk_log_start":"418'1800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6445,"log_dups_size":0,"ondisk_log_size":6445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:34.479018+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309395456,"num_objects":4721,"num_object_clones":0,"num_object_copies":14163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8245,"num_write_kb":414912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":607,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.911563+0000","last_change":"2023-09-23T15:30:48.890337+0000","last_active":"2023-09-23T15:31:52.911563+0000","last_peered":"2023-09-23T15:31:52.911563+0000","last_clean":"2023-09-23T15:31:52.911563+0000","last_became_active":"2023-09-23T15:30:26.693580+0000","last_became_peered":"2023-09-23T15:30:26.693580+0000","last_unstale":"2023-09-23T15:31:52.911563+0000","last_undegraded":"2023-09-23T15:31:52.911563+0000","last_fullsized":"2023-09-23T15:31:52.911563+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:25.353085+0000","scrub_duration":0.011955009000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":179,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898391+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:31:52.898391+0000","last_peered":"2023-09-23T15:31:52.898391+0000","last_clean":"2023-09-23T15:31:52.898391+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:31:52.898391+0000","last_undegraded":"2023-09-23T15:31:52.898391+0000","last_fullsized":"2023-09-23T15:31:52.898391+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.896682+0000","last_change":"2023-09-23T15:27:23.316634+0000","last_active":"2023-09-23T15:31:52.896682+0000","last_peered":"2023-09-23T15:31:52.896682+0000","last_clean":"2023-09-23T15:31:52.896682+0000","last_became_active":"2023-09-23T15:27:23.315982+0000","last_became_peered":"2023-09-23T15:27:23.315982+0000","last_unstale":"2023-09-23T15:31:52.896682+0000","last_undegraded":"2023-09-23T15:31:52.896682+0000","last_fullsized":"2023-09-23T15:31:52.896682+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:18.550044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"440'8230","reported_seq":8265,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:03.988458+0000","last_change":"2023-09-23T15:29:48.697177+0000","last_active":"2023-09-23T15:32:03.988458+0000","last_peered":"2023-09-23T15:32:03.988458+0000","last_clean":"2023-09-23T15:32:03.988458+0000","last_became_active":"2023-09-23T15:29:48.696842+0000","last_became_peered":"2023-09-23T15:29:48.696842+0000","last_unstale":"2023-09-23T15:32:03.988458+0000","last_undegraded":"2023-09-23T15:32:03.988458+0000","last_fullsized":"2023-09-23T15:32:03.988458+0000","mapping_epoch":413,"log_start":"418'1800","ondisk_log_start":"418'1800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6430,"log_dups_size":0,"ondisk_log_size":6430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:18.816926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318373888,"num_objects":4858,"num_object_clones":0,"num_object_copies":14574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8230,"num_write_kb":418816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":687,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898975+0000","last_change":"2023-09-23T15:27:26.319447+0000","last_active":"2023-09-23T15:31:52.898975+0000","last_peered":"2023-09-23T15:31:52.898975+0000","last_clean":"2023-09-23T15:31:52.898975+0000","last_became_active":"2023-09-23T15:27:26.319045+0000","last_became_peered":"2023-09-23T15:27:26.319045+0000","last_unstale":"2023-09-23T15:31:52.898975+0000","last_undegraded":"2023-09-23T15:31:52.898975+0000","last_fullsized":"2023-09-23T15:31:52.898975+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:02.128694+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":167,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898008+0000","last_change":"2023-09-23T15:27:41.423515+0000","last_active":"2023-09-23T15:31:52.898008+0000","last_peered":"2023-09-23T15:31:52.898008+0000","last_clean":"2023-09-23T15:31:52.898008+0000","last_became_active":"2023-09-23T15:27:41.423041+0000","last_became_peered":"2023-09-23T15:27:41.423041+0000","last_unstale":"2023-09-23T15:31:52.898008+0000","last_undegraded":"2023-09-23T15:31:52.898008+0000","last_fullsized":"2023-09-23T15:31:52.898008+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:17.879937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":70,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,6],"acting":[7,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":420,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897922+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:31:52.897922+0000","last_peered":"2023-09-23T15:31:52.897922+0000","last_clean":"2023-09-23T15:31:52.897922+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:31:52.897922+0000","last_undegraded":"2023-09-23T15:31:52.897922+0000","last_fullsized":"2023-09-23T15:31:52.897922+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"440'8545","reported_seq":8581,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.296415+0000","last_change":"2023-09-23T15:29:48.698992+0000","last_active":"2023-09-23T15:32:04.296415+0000","last_peered":"2023-09-23T15:32:04.296415+0000","last_clean":"2023-09-23T15:32:04.296415+0000","last_became_active":"2023-09-23T15:29:48.698673+0000","last_became_peered":"2023-09-23T15:29:48.698673+0000","last_unstale":"2023-09-23T15:32:04.296415+0000","last_undegraded":"2023-09-23T15:32:04.296415+0000","last_fullsized":"2023-09-23T15:32:04.296415+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":8545,"log_dups_size":0,"ondisk_log_size":8545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:12.184760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314769408,"num_objects":4803,"num_object_clones":0,"num_object_copies":14409,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4803,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8545,"num_write_kb":427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,3],"acting":[2,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":657,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901890+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:31:52.901890+0000","last_peered":"2023-09-23T15:31:52.901890+0000","last_clean":"2023-09-23T15:31:52.901890+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:31:52.901890+0000","last_undegraded":"2023-09-23T15:31:52.901890+0000","last_fullsized":"2023-09-23T15:31:52.901890+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":415,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900750+0000","last_change":"2023-09-23T15:22:26.309259+0000","last_active":"2023-09-23T15:31:52.900750+0000","last_peered":"2023-09-23T15:31:52.900750+0000","last_clean":"2023-09-23T15:31:52.900750+0000","last_became_active":"2023-09-23T15:22:26.308957+0000","last_became_peered":"2023-09-23T15:22:26.308957+0000","last_unstale":"2023-09-23T15:31:52.900750+0000","last_undegraded":"2023-09-23T15:31:52.900750+0000","last_fullsized":"2023-09-23T15:31:52.900750+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:22.594084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":442,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.907909+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:31:52.907909+0000","last_peered":"2023-09-23T15:31:52.907909+0000","last_clean":"2023-09-23T15:31:52.907909+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:31:52.907909+0000","last_undegraded":"2023-09-23T15:31:52.907909+0000","last_fullsized":"2023-09-23T15:31:52.907909+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"439'8268","reported_seq":8304,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:32:00.697855+0000","last_change":"2023-09-23T15:29:48.679906+0000","last_active":"2023-09-23T15:32:00.697855+0000","last_peered":"2023-09-23T15:32:00.697855+0000","last_clean":"2023-09-23T15:32:00.697855+0000","last_became_active":"2023-09-23T15:29:48.679584+0000","last_became_peered":"2023-09-23T15:29:48.679584+0000","last_unstale":"2023-09-23T15:32:00.697855+0000","last_undegraded":"2023-09-23T15:32:00.697855+0000","last_fullsized":"2023-09-23T15:32:00.697855+0000","mapping_epoch":413,"log_start":"417'1200","ondisk_log_start":"417'1200","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:53.805715+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330301440,"num_objects":5040,"num_object_clones":0,"num_object_copies":15120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8268,"num_write_kb":425856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":692,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898684+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:31:52.898684+0000","last_peered":"2023-09-23T15:31:52.898684+0000","last_clean":"2023-09-23T15:31:52.898684+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:31:52.898684+0000","last_undegraded":"2023-09-23T15:31:52.898684+0000","last_fullsized":"2023-09-23T15:31:52.898684+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":92,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.907958+0000","last_change":"2023-09-23T15:30:26.690336+0000","last_active":"2023-09-23T15:31:52.907958+0000","last_peered":"2023-09-23T15:31:52.907958+0000","last_clean":"2023-09-23T15:31:52.907958+0000","last_became_active":"2023-09-23T15:30:26.690141+0000","last_became_peered":"2023-09-23T15:30:26.690141+0000","last_unstale":"2023-09-23T15:31:52.907958+0000","last_undegraded":"2023-09-23T15:31:52.907958+0000","last_fullsized":"2023-09-23T15:31:52.907958+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:44.578933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894724+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:31:52.894724+0000","last_peered":"2023-09-23T15:31:52.894724+0000","last_clean":"2023-09-23T15:31:52.894724+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:31:52.894724+0000","last_undegraded":"2023-09-23T15:31:52.894724+0000","last_fullsized":"2023-09-23T15:31:52.894724+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.3","version":"440'8588","reported_seq":8624,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.450314+0000","last_change":"2023-09-23T15:29:48.698981+0000","last_active":"2023-09-23T15:32:04.450314+0000","last_peered":"2023-09-23T15:32:04.450314+0000","last_clean":"2023-09-23T15:32:04.450314+0000","last_became_active":"2023-09-23T15:29:48.698707+0000","last_became_peered":"2023-09-23T15:29:48.698707+0000","last_unstale":"2023-09-23T15:32:04.450314+0000","last_undegraded":"2023-09-23T15:32:04.450314+0000","last_fullsized":"2023-09-23T15:32:04.450314+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:28.109594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323747840,"num_objects":4940,"num_object_clones":0,"num_object_copies":14820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8588,"num_write_kb":432896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,3],"acting":[2,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":763,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900751+0000","last_change":"2023-09-23T15:27:01.846028+0000","last_active":"2023-09-23T15:31:52.900751+0000","last_peered":"2023-09-23T15:31:52.900751+0000","last_clean":"2023-09-23T15:31:52.900751+0000","last_became_active":"2023-09-23T15:26:50.302447+0000","last_became_peered":"2023-09-23T15:26:50.302447+0000","last_unstale":"2023-09-23T15:31:52.900751+0000","last_undegraded":"2023-09-23T15:31:52.900751+0000","last_fullsized":"2023-09-23T15:31:52.900751+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:27:01.845960+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.845960+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:41.626305+0000","scrub_duration":0.001719754,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":407,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898012+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:31:52.898012+0000","last_peered":"2023-09-23T15:31:52.898012+0000","last_clean":"2023-09-23T15:31:52.898012+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:31:52.898012+0000","last_undegraded":"2023-09-23T15:31:52.898012+0000","last_fullsized":"2023-09-23T15:31:52.898012+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.895969+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:31:52.895969+0000","last_peered":"2023-09-23T15:31:52.895969+0000","last_clean":"2023-09-23T15:31:52.895969+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:31:52.895969+0000","last_undegraded":"2023-09-23T15:31:52.895969+0000","last_fullsized":"2023-09-23T15:31:52.895969+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.0","version":"440'8374","reported_seq":8409,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.596286+0000","last_change":"2023-09-23T15:29:48.683338+0000","last_active":"2023-09-23T15:32:04.596286+0000","last_peered":"2023-09-23T15:32:04.596286+0000","last_clean":"2023-09-23T15:32:04.596286+0000","last_became_active":"2023-09-23T15:29:48.682970+0000","last_became_peered":"2023-09-23T15:29:48.682970+0000","last_unstale":"2023-09-23T15:32:04.596286+0000","last_undegraded":"2023-09-23T15:32:04.596286+0000","last_fullsized":"2023-09-23T15:32:04.596286+0000","mapping_epoch":413,"log_start":"418'1900","ondisk_log_start":"418'1900","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6474,"log_dups_size":0,"ondisk_log_size":6474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:59:20.273166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315490304,"num_objects":4814,"num_object_clones":0,"num_object_copies":14442,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4814,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8374,"num_write_kb":422016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":719,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694906+0000","last_change":"2023-09-23T15:28:50.442371+0000","last_active":"2023-09-23T15:31:48.694906+0000","last_peered":"2023-09-23T15:31:48.694906+0000","last_clean":"2023-09-23T15:31:48.694906+0000","last_became_active":"2023-09-23T15:26:50.292178+0000","last_became_peered":"2023-09-23T15:26:50.292178+0000","last_unstale":"2023-09-23T15:31:48.694906+0000","last_undegraded":"2023-09-23T15:31:48.694906+0000","last_fullsized":"2023-09-23T15:31:48.694906+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:43.892312+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":96,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.896684+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:31:52.896684+0000","last_peered":"2023-09-23T15:31:52.896684+0000","last_clean":"2023-09-23T15:31:52.896684+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:31:52.896684+0000","last_undegraded":"2023-09-23T15:31:52.896684+0000","last_fullsized":"2023-09-23T15:31:52.896684+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901974+0000","last_change":"2023-09-23T15:27:23.313558+0000","last_active":"2023-09-23T15:31:52.901974+0000","last_peered":"2023-09-23T15:31:52.901974+0000","last_clean":"2023-09-23T15:31:52.901974+0000","last_became_active":"2023-09-23T15:27:23.313182+0000","last_became_peered":"2023-09-23T15:27:23.313182+0000","last_unstale":"2023-09-23T15:31:52.901974+0000","last_undegraded":"2023-09-23T15:31:52.901974+0000","last_fullsized":"2023-09-23T15:31:52.901974+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:52:29.489340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.1","version":"440'8283","reported_seq":8318,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.061674+0000","last_change":"2023-09-23T15:29:48.680338+0000","last_active":"2023-09-23T15:32:04.061674+0000","last_peered":"2023-09-23T15:32:04.061674+0000","last_clean":"2023-09-23T15:32:04.061674+0000","last_became_active":"2023-09-23T15:29:48.679979+0000","last_became_peered":"2023-09-23T15:29:48.679979+0000","last_unstale":"2023-09-23T15:32:04.061674+0000","last_undegraded":"2023-09-23T15:32:04.061674+0000","last_fullsized":"2023-09-23T15:32:04.061674+0000","mapping_epoch":413,"log_start":"418'2500","ondisk_log_start":"418'2500","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":5783,"log_dups_size":0,"ondisk_log_size":5783,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:54.650904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311492608,"num_objects":4753,"num_object_clones":0,"num_object_copies":14259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8283,"num_write_kb":417152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":775,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901928+0000","last_change":"2023-09-23T15:26:58.912943+0000","last_active":"2023-09-23T15:31:52.901928+0000","last_peered":"2023-09-23T15:31:52.901928+0000","last_clean":"2023-09-23T15:31:52.901928+0000","last_became_active":"2023-09-23T15:26:50.301056+0000","last_became_peered":"2023-09-23T15:26:50.301056+0000","last_unstale":"2023-09-23T15:31:52.901928+0000","last_undegraded":"2023-09-23T15:31:52.901928+0000","last_fullsized":"2023-09-23T15:31:52.901928+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.912879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.912879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:22:28.375791+0000","scrub_duration":0.001554277,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":405,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695455+0000","last_change":"2023-09-23T15:30:26.705171+0000","last_active":"2023-09-23T15:31:48.695455+0000","last_peered":"2023-09-23T15:31:48.695455+0000","last_clean":"2023-09-23T15:31:48.695455+0000","last_became_active":"2023-09-23T15:30:26.704853+0000","last_became_peered":"2023-09-23T15:30:26.704853+0000","last_unstale":"2023-09-23T15:31:48.695455+0000","last_undegraded":"2023-09-23T15:31:48.695455+0000","last_fullsized":"2023-09-23T15:31:48.695455+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:11.875343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":374,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695029+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:31:48.695029+0000","last_peered":"2023-09-23T15:31:48.695029+0000","last_clean":"2023-09-23T15:31:48.695029+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:31:48.695029+0000","last_undegraded":"2023-09-23T15:31:48.695029+0000","last_fullsized":"2023-09-23T15:31:48.695029+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"440'8490","reported_seq":8525,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.454101+0000","last_change":"2023-09-23T15:29:48.682296+0000","last_active":"2023-09-23T15:32:04.454101+0000","last_peered":"2023-09-23T15:32:04.454101+0000","last_clean":"2023-09-23T15:32:04.454101+0000","last_became_active":"2023-09-23T15:29:48.681890+0000","last_became_peered":"2023-09-23T15:29:48.681890+0000","last_unstale":"2023-09-23T15:32:04.454101+0000","last_undegraded":"2023-09-23T15:32:04.454101+0000","last_fullsized":"2023-09-23T15:32:04.454101+0000","mapping_epoch":413,"log_start":"418'2100","ondisk_log_start":"418'2100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6390,"log_dups_size":0,"ondisk_log_size":6390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:34.635613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314703872,"num_objects":4802,"num_object_clones":0,"num_object_copies":14406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8490,"num_write_kb":425344,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":680,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.899065+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:31:52.899065+0000","last_peered":"2023-09-23T15:31:52.899065+0000","last_clean":"2023-09-23T15:31:52.899065+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:31:52.899065+0000","last_undegraded":"2023-09-23T15:31:52.899065+0000","last_fullsized":"2023-09-23T15:31:52.899065+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'71","reported_seq":657,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694141+0000","last_change":"2023-09-23T15:27:41.438552+0000","last_active":"2023-09-23T15:31:48.694141+0000","last_peered":"2023-09-23T15:31:48.694141+0000","last_clean":"2023-09-23T15:31:48.694141+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:31:48.694141+0000","last_undegraded":"2023-09-23T15:31:48.694141+0000","last_fullsized":"2023-09-23T15:31:48.694141+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":71,"log_dups_size":0,"ondisk_log_size":71,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000455909,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":343,"num_read_kb":312,"num_write":75,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":415,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901046+0000","last_change":"2023-09-23T15:06:16.137756+0000","last_active":"2023-09-23T15:31:52.901046+0000","last_peered":"2023-09-23T15:31:52.901046+0000","last_clean":"2023-09-23T15:31:52.901046+0000","last_became_active":"2023-09-23T15:06:16.137412+0000","last_became_peered":"2023-09-23T15:06:16.137412+0000","last_unstale":"2023-09-23T15:31:52.901046+0000","last_undegraded":"2023-09-23T15:31:52.901046+0000","last_fullsized":"2023-09-23T15:31:52.901046+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:53:41.424885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":404,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893085+0000","last_change":"2023-09-23T15:27:23.312164+0000","last_active":"2023-09-23T15:31:52.893085+0000","last_peered":"2023-09-23T15:31:52.893085+0000","last_clean":"2023-09-23T15:31:52.893085+0000","last_became_active":"2023-09-23T15:27:23.311846+0000","last_became_peered":"2023-09-23T15:27:23.311846+0000","last_unstale":"2023-09-23T15:31:52.893085+0000","last_undegraded":"2023-09-23T15:31:52.893085+0000","last_fullsized":"2023-09-23T15:31:52.893085+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:48.886520+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"10.a","version":"440'8551","reported_seq":8586,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.609686+0000","last_change":"2023-09-23T15:29:48.686875+0000","last_active":"2023-09-23T15:32:04.609686+0000","last_peered":"2023-09-23T15:32:04.609686+0000","last_clean":"2023-09-23T15:32:04.609686+0000","last_became_active":"2023-09-23T15:29:48.686520+0000","last_became_peered":"2023-09-23T15:29:48.686520+0000","last_unstale":"2023-09-23T15:32:04.609686+0000","last_undegraded":"2023-09-23T15:32:04.609686+0000","last_fullsized":"2023-09-23T15:32:04.609686+0000","mapping_epoch":413,"log_start":"418'2100","ondisk_log_start":"418'2100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6451,"log_dups_size":0,"ondisk_log_size":6451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:52.851173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322764800,"num_objects":4925,"num_object_clones":0,"num_object_copies":14775,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4925,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8551,"num_write_kb":431232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":736,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902917+0000","last_change":"2023-09-23T15:16:48.533283+0000","last_active":"2023-09-23T15:31:52.902917+0000","last_peered":"2023-09-23T15:31:52.902917+0000","last_clean":"2023-09-23T15:31:52.902917+0000","last_became_active":"2023-09-23T15:16:46.268601+0000","last_became_peered":"2023-09-23T15:16:46.268601+0000","last_unstale":"2023-09-23T15:31:52.902917+0000","last_undegraded":"2023-09-23T15:31:52.902917+0000","last_fullsized":"2023-09-23T15:31:52.902917+0000","mapping_epoch":225,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":226,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.533166+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.533166+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:24.010783+0000","scrub_duration":0.00181952,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"85'79","reported_seq":974,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.904239+0000","last_change":"2023-09-23T15:22:26.308817+0000","last_active":"2023-09-23T15:31:52.904239+0000","last_peered":"2023-09-23T15:31:52.904239+0000","last_clean":"2023-09-23T15:31:52.904239+0000","last_became_active":"2023-09-23T15:22:26.307608+0000","last_became_peered":"2023-09-23T15:22:26.307608+0000","last_unstale":"2023-09-23T15:31:52.904239+0000","last_undegraded":"2023-09-23T15:31:52.904239+0000","last_fullsized":"2023-09-23T15:31:52.904239+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:21.287048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011646060000000001,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":750,"num_read_kb":2399,"num_write":55,"num_write_kb":1393,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894666+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:31:52.894666+0000","last_peered":"2023-09-23T15:31:52.894666+0000","last_clean":"2023-09-23T15:31:52.894666+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:31:52.894666+0000","last_undegraded":"2023-09-23T15:31:52.894666+0000","last_fullsized":"2023-09-23T15:31:52.894666+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":339,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693765+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:31:48.693765+0000","last_peered":"2023-09-23T15:31:48.693765+0000","last_clean":"2023-09-23T15:31:48.693765+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:31:48.693765+0000","last_undegraded":"2023-09-23T15:31:48.693765+0000","last_fullsized":"2023-09-23T15:31:48.693765+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.9","version":"440'8569","reported_seq":8603,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.043353+0000","last_change":"2023-09-23T15:29:48.686854+0000","last_active":"2023-09-23T15:32:04.043353+0000","last_peered":"2023-09-23T15:32:04.043353+0000","last_clean":"2023-09-23T15:32:04.043353+0000","last_became_active":"2023-09-23T15:29:48.686539+0000","last_became_peered":"2023-09-23T15:29:48.686539+0000","last_unstale":"2023-09-23T15:32:04.043353+0000","last_undegraded":"2023-09-23T15:32:04.043353+0000","last_fullsized":"2023-09-23T15:32:04.043353+0000","mapping_epoch":413,"log_start":"417'800","ondisk_log_start":"417'800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":7769,"log_dups_size":0,"ondisk_log_size":7769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:35:33.671531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317390848,"num_objects":4843,"num_object_clones":0,"num_object_copies":14529,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4843,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8569,"num_write_kb":429184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":803,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897585+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:31:52.897585+0000","last_peered":"2023-09-23T15:31:52.897585+0000","last_clean":"2023-09-23T15:31:52.897585+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:31:52.897585+0000","last_undegraded":"2023-09-23T15:31:52.897585+0000","last_fullsized":"2023-09-23T15:31:52.897585+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'182","reported_seq":1059,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893175+0000","last_change":"2023-09-23T15:06:16.129635+0000","last_active":"2023-09-23T15:31:52.893175+0000","last_peered":"2023-09-23T15:31:52.893175+0000","last_clean":"2023-09-23T15:31:52.893175+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:31:52.893175+0000","last_undegraded":"2023-09-23T15:31:52.893175+0000","last_fullsized":"2023-09-23T15:31:52.893175+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035283100000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1088,"num_read_kb":2260,"num_write":168,"num_write_kb":1291,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":334,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.695609+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:31:48.695609+0000","last_peered":"2023-09-23T15:31:48.695609+0000","last_clean":"2023-09-23T15:31:48.695609+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:31:48.695609+0000","last_undegraded":"2023-09-23T15:31:48.695609+0000","last_fullsized":"2023-09-23T15:31:48.695609+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":342,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694959+0000","last_change":"2023-09-23T15:12:41.934464+0000","last_active":"2023-09-23T15:31:48.694959+0000","last_peered":"2023-09-23T15:31:48.694959+0000","last_clean":"2023-09-23T15:31:48.694959+0000","last_became_active":"2023-09-23T15:12:41.934190+0000","last_became_peered":"2023-09-23T15:12:41.934190+0000","last_unstale":"2023-09-23T15:31:48.694959+0000","last_undegraded":"2023-09-23T15:31:48.694959+0000","last_fullsized":"2023-09-23T15:31:48.694959+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:47:00.826598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.b","version":"440'8393","reported_seq":8428,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.550299+0000","last_change":"2023-09-23T15:29:48.684663+0000","last_active":"2023-09-23T15:32:04.550299+0000","last_peered":"2023-09-23T15:32:04.550299+0000","last_clean":"2023-09-23T15:32:04.550299+0000","last_became_active":"2023-09-23T15:29:48.684293+0000","last_became_peered":"2023-09-23T15:29:48.684293+0000","last_unstale":"2023-09-23T15:32:04.550299+0000","last_undegraded":"2023-09-23T15:32:04.550299+0000","last_fullsized":"2023-09-23T15:32:04.550299+0000","mapping_epoch":413,"log_start":"418'2000","ondisk_log_start":"418'2000","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6393,"log_dups_size":0,"ondisk_log_size":6393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:18.182468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311885824,"num_objects":4759,"num_object_clones":0,"num_object_copies":14277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8393,"num_write_kb":420864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'144","reported_seq":785,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.901410+0000","last_change":"2023-09-23T15:27:41.422593+0000","last_active":"2023-09-23T15:31:52.901410+0000","last_peered":"2023-09-23T15:31:52.901410+0000","last_clean":"2023-09-23T15:31:52.901410+0000","last_became_active":"2023-09-23T15:27:41.421998+0000","last_became_peered":"2023-09-23T15:27:41.421998+0000","last_unstale":"2023-09-23T15:31:52.901410+0000","last_undegraded":"2023-09-23T15:31:52.901410+0000","last_fullsized":"2023-09-23T15:31:52.901410+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":144,"log_dups_size":0,"ondisk_log_size":144,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:53.165941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00046560999999999998,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":287,"num_read_kb":396,"num_write":150,"num_write_kb":181,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":746,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898677+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:31:52.898677+0000","last_peered":"2023-09-23T15:31:52.898677+0000","last_clean":"2023-09-23T15:31:52.898677+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:31:52.898677+0000","last_undegraded":"2023-09-23T15:31:52.898677+0000","last_fullsized":"2023-09-23T15:31:52.898677+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":309,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902077+0000","last_change":"2023-09-23T15:30:26.694100+0000","last_active":"2023-09-23T15:31:52.902077+0000","last_peered":"2023-09-23T15:31:52.902077+0000","last_clean":"2023-09-23T15:31:52.902077+0000","last_became_active":"2023-09-23T15:30:26.693767+0000","last_became_peered":"2023-09-23T15:30:26.693767+0000","last_unstale":"2023-09-23T15:31:52.902077+0000","last_undegraded":"2023-09-23T15:31:52.902077+0000","last_fullsized":"2023-09-23T15:31:52.902077+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:49.703270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":432,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897454+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:31:52.897454+0000","last_peered":"2023-09-23T15:31:52.897454+0000","last_clean":"2023-09-23T15:31:52.897454+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:31:52.897454+0000","last_undegraded":"2023-09-23T15:31:52.897454+0000","last_fullsized":"2023-09-23T15:31:52.897454+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"440'8269","reported_seq":8305,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:03.971995+0000","last_change":"2023-09-23T15:29:48.679894+0000","last_active":"2023-09-23T15:32:03.971995+0000","last_peered":"2023-09-23T15:32:03.971995+0000","last_clean":"2023-09-23T15:32:03.971995+0000","last_became_active":"2023-09-23T15:29:48.679648+0000","last_became_peered":"2023-09-23T15:29:48.679648+0000","last_unstale":"2023-09-23T15:32:03.971995+0000","last_undegraded":"2023-09-23T15:32:03.971995+0000","last_fullsized":"2023-09-23T15:32:03.971995+0000","mapping_epoch":413,"log_start":"418'1800","ondisk_log_start":"418'1800","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6469,"log_dups_size":0,"ondisk_log_size":6469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:23.079554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313065472,"num_objects":4777,"num_object_clones":0,"num_object_copies":14331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8269,"num_write_kb":417472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":783,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694827+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:31:48.694827+0000","last_peered":"2023-09-23T15:31:48.694827+0000","last_clean":"2023-09-23T15:31:48.694827+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:31:48.694827+0000","last_undegraded":"2023-09-23T15:31:48.694827+0000","last_fullsized":"2023-09-23T15:31:48.694827+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"85'86","reported_seq":153,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898538+0000","last_change":"2023-09-23T15:27:41.449626+0000","last_active":"2023-09-23T15:31:52.898538+0000","last_peered":"2023-09-23T15:31:52.898538+0000","last_clean":"2023-09-23T15:31:52.898538+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:31:52.898538+0000","last_undegraded":"2023-09-23T15:31:52.898538+0000","last_fullsized":"2023-09-23T15:31:52.898538+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":86,"log_dups_size":0,"ondisk_log_size":86,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00061428800000000005,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":367,"num_read_kb":1330,"num_write":88,"num_write_kb":5172,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":428,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902954+0000","last_change":"2023-09-23T15:27:23.315599+0000","last_active":"2023-09-23T15:31:52.902954+0000","last_peered":"2023-09-23T15:31:52.902954+0000","last_clean":"2023-09-23T15:31:52.902954+0000","last_became_active":"2023-09-23T15:27:23.315252+0000","last_became_peered":"2023-09-23T15:27:23.315252+0000","last_unstale":"2023-09-23T15:31:52.902954+0000","last_undegraded":"2023-09-23T15:31:52.902954+0000","last_fullsized":"2023-09-23T15:31:52.902954+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:23.860510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":421,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.910785+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:31:52.910785+0000","last_peered":"2023-09-23T15:31:52.910785+0000","last_clean":"2023-09-23T15:31:52.910785+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:31:52.910785+0000","last_undegraded":"2023-09-23T15:31:52.910785+0000","last_fullsized":"2023-09-23T15:31:52.910785+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.d","version":"440'8576","reported_seq":8612,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.610029+0000","last_change":"2023-09-23T15:29:48.683852+0000","last_active":"2023-09-23T15:32:04.610029+0000","last_peered":"2023-09-23T15:32:04.610029+0000","last_clean":"2023-09-23T15:32:04.610029+0000","last_became_active":"2023-09-23T15:29:48.683560+0000","last_became_peered":"2023-09-23T15:29:48.683560+0000","last_unstale":"2023-09-23T15:32:04.610029+0000","last_undegraded":"2023-09-23T15:32:04.610029+0000","last_fullsized":"2023-09-23T15:32:04.610029+0000","mapping_epoch":413,"log_start":"418'2100","ondisk_log_start":"418'2100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6476,"log_dups_size":0,"ondisk_log_size":6476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:38.866091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322895896,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8576,"num_write_kb":432065,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":766,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897387+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:31:52.897387+0000","last_peered":"2023-09-23T15:31:52.897387+0000","last_clean":"2023-09-23T15:31:52.897387+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:31:52.897387+0000","last_undegraded":"2023-09-23T15:31:52.897387+0000","last_fullsized":"2023-09-23T15:31:52.897387+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"85'154","reported_seq":1179,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898052+0000","last_change":"2023-09-23T15:27:42.383080+0000","last_active":"2023-09-23T15:31:52.898052+0000","last_peered":"2023-09-23T15:31:52.898052+0000","last_clean":"2023-09-23T15:31:52.898052+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:31:52.898052+0000","last_undegraded":"2023-09-23T15:31:52.898052+0000","last_fullsized":"2023-09-23T15:31:52.898052+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":154,"log_dups_size":0,"ondisk_log_size":154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00035022500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1285,"num_read_kb":1488,"num_write":118,"num_write_kb":240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":319,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.693985+0000","last_change":"2023-09-23T15:12:41.925116+0000","last_active":"2023-09-23T15:31:48.693985+0000","last_peered":"2023-09-23T15:31:48.693985+0000","last_clean":"2023-09-23T15:31:48.693985+0000","last_became_active":"2023-09-23T15:12:41.924832+0000","last_became_peered":"2023-09-23T15:12:41.924832+0000","last_unstale":"2023-09-23T15:31:48.693985+0000","last_undegraded":"2023-09-23T15:31:48.693985+0000","last_fullsized":"2023-09-23T15:31:48.693985+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:45.746932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":421,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.908043+0000","last_change":"2023-09-23T15:27:41.466066+0000","last_active":"2023-09-23T15:31:52.908043+0000","last_peered":"2023-09-23T15:31:52.908043+0000","last_clean":"2023-09-23T15:31:52.908043+0000","last_became_active":"2023-09-23T15:27:41.465722+0000","last_became_peered":"2023-09-23T15:27:41.465722+0000","last_unstale":"2023-09-23T15:31:52.908043+0000","last_undegraded":"2023-09-23T15:31:52.908043+0000","last_fullsized":"2023-09-23T15:31:52.908043+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:36.915837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.e","version":"440'8453","reported_seq":8488,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.094056+0000","last_change":"2023-09-23T15:29:48.685734+0000","last_active":"2023-09-23T15:32:04.094056+0000","last_peered":"2023-09-23T15:32:04.094056+0000","last_clean":"2023-09-23T15:32:04.094056+0000","last_became_active":"2023-09-23T15:29:48.684906+0000","last_became_peered":"2023-09-23T15:29:48.684906+0000","last_unstale":"2023-09-23T15:32:04.094056+0000","last_undegraded":"2023-09-23T15:32:04.094056+0000","last_fullsized":"2023-09-23T15:32:04.094056+0000","mapping_epoch":413,"log_start":"418'2600","ondisk_log_start":"418'2600","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":5853,"log_dups_size":0,"ondisk_log_size":5853,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:06.890491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046208,"num_objects":4853,"num_object_clones":0,"num_object_copies":14559,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4853,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8453,"num_write_kb":425792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":731,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.896061+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:31:52.896061+0000","last_peered":"2023-09-23T15:31:52.896061+0000","last_clean":"2023-09-23T15:31:52.896061+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:31:52.896061+0000","last_undegraded":"2023-09-23T15:31:52.896061+0000","last_fullsized":"2023-09-23T15:31:52.896061+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":213,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893640+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:31:52.893640+0000","last_peered":"2023-09-23T15:31:52.893640+0000","last_clean":"2023-09-23T15:31:52.893640+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:31:52.893640+0000","last_undegraded":"2023-09-23T15:31:52.893640+0000","last_fullsized":"2023-09-23T15:31:52.893640+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"85'152","reported_seq":1073,"reported_epoch":438,"state":"active+clean","last_fresh":"2023-09-23T15:31:48.694748+0000","last_change":"2023-09-23T15:27:26.323343+0000","last_active":"2023-09-23T15:31:48.694748+0000","last_peered":"2023-09-23T15:31:48.694748+0000","last_clean":"2023-09-23T15:31:48.694748+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:31:48.694748+0000","last_undegraded":"2023-09-23T15:31:48.694748+0000","last_fullsized":"2023-09-23T15:31:48.694748+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":152,"log_dups_size":0,"ondisk_log_size":152,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029414799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1022,"num_read_kb":2898,"num_write":104,"num_write_kb":2096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":387,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.898751+0000","last_change":"2023-09-23T15:30:26.710395+0000","last_active":"2023-09-23T15:31:52.898751+0000","last_peered":"2023-09-23T15:31:52.898751+0000","last_clean":"2023-09-23T15:31:52.898751+0000","last_became_active":"2023-09-23T15:30:26.710076+0000","last_became_peered":"2023-09-23T15:30:26.710076+0000","last_unstale":"2023-09-23T15:31:52.898751+0000","last_undegraded":"2023-09-23T15:31:52.898751+0000","last_fullsized":"2023-09-23T15:31:52.898751+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.313559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"10.f","version":"440'8517","reported_seq":8552,"reported_epoch":440,"state":"active+clean","last_fresh":"2023-09-23T15:32:04.581617+0000","last_change":"2023-09-23T15:29:48.697095+0000","last_active":"2023-09-23T15:32:04.581617+0000","last_peered":"2023-09-23T15:32:04.581617+0000","last_clean":"2023-09-23T15:32:04.581617+0000","last_became_active":"2023-09-23T15:29:48.696636+0000","last_became_peered":"2023-09-23T15:29:48.696636+0000","last_unstale":"2023-09-23T15:32:04.581617+0000","last_undegraded":"2023-09-23T15:32:04.581617+0000","last_fullsized":"2023-09-23T15:32:04.581617+0000","mapping_epoch":413,"log_start":"418'2100","ondisk_log_start":"418'2100","created":413,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T15:29:47.658752+0000","last_clean_scrub_stamp":"2023-09-23T15:29:47.658752+0000","objects_scrubbed":0,"log_size":6417,"log_dups_size":0,"ondisk_log_size":6417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:52.284501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8517,"num_write_kb":428032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":580,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900618+0000","last_change":"2023-09-23T15:30:26.695170+0000","last_active":"2023-09-23T15:31:52.900618+0000","last_peered":"2023-09-23T15:31:52.900618+0000","last_clean":"2023-09-23T15:31:52.900618+0000","last_became_active":"2023-09-23T15:30:26.694799+0000","last_became_peered":"2023-09-23T15:30:26.694799+0000","last_unstale":"2023-09-23T15:31:52.900618+0000","last_undegraded":"2023-09-23T15:31:52.900618+0000","last_fullsized":"2023-09-23T15:31:52.900618+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:29.288370+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"85'108","reported_seq":893,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.899365+0000","last_change":"2023-09-23T15:27:41.428920+0000","last_active":"2023-09-23T15:31:52.899365+0000","last_peered":"2023-09-23T15:31:52.899365+0000","last_clean":"2023-09-23T15:31:52.899365+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:31:52.899365+0000","last_undegraded":"2023-09-23T15:31:52.899365+0000","last_fullsized":"2023-09-23T15:31:52.899365+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":108,"log_dups_size":0,"ondisk_log_size":108,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011099370000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":853,"num_read_kb":2129,"num_write":88,"num_write_kb":2290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":480,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.910814+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:31:52.910814+0000","last_peered":"2023-09-23T15:31:52.910814+0000","last_clean":"2023-09-23T15:31:52.910814+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:31:52.910814+0000","last_undegraded":"2023-09-23T15:31:52.910814+0000","last_fullsized":"2023-09-23T15:31:52.910814+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.896766+0000","last_change":"2023-09-23T15:27:23.313542+0000","last_active":"2023-09-23T15:31:52.896766+0000","last_peered":"2023-09-23T15:31:52.896766+0000","last_clean":"2023-09-23T15:31:52.896766+0000","last_became_active":"2023-09-23T15:27:23.313183+0000","last_became_peered":"2023-09-23T15:27:23.313183+0000","last_unstale":"2023-09-23T15:31:52.896766+0000","last_undegraded":"2023-09-23T15:31:52.896766+0000","last_fullsized":"2023-09-23T15:31:52.896766+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:11.296952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":649,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.902670+0000","last_change":"2023-09-23T15:30:26.740237+0000","last_active":"2023-09-23T15:31:52.902670+0000","last_peered":"2023-09-23T15:31:52.902670+0000","last_clean":"2023-09-23T15:31:52.902670+0000","last_became_active":"2023-09-23T15:30:26.690718+0000","last_became_peered":"2023-09-23T15:30:26.690718+0000","last_unstale":"2023-09-23T15:31:52.902670+0000","last_undegraded":"2023-09-23T15:31:52.902670+0000","last_fullsized":"2023-09-23T15:31:52.902670+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:00.439552+0000","scrub_duration":0.013441308000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":459,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.908003+0000","last_change":"2023-09-23T15:27:42.380321+0000","last_active":"2023-09-23T15:31:52.908003+0000","last_peered":"2023-09-23T15:31:52.908003+0000","last_clean":"2023-09-23T15:31:52.908003+0000","last_became_active":"2023-09-23T15:27:42.379989+0000","last_became_peered":"2023-09-23T15:27:42.379989+0000","last_unstale":"2023-09-23T15:31:52.908003+0000","last_undegraded":"2023-09-23T15:31:52.908003+0000","last_fullsized":"2023-09-23T15:31:52.908003+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:10:47.037399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":445,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893373+0000","last_change":"2023-09-23T15:12:41.858157+0000","last_active":"2023-09-23T15:31:52.893373+0000","last_peered":"2023-09-23T15:31:52.893373+0000","last_clean":"2023-09-23T15:31:52.893373+0000","last_became_active":"2023-09-23T15:12:41.857745+0000","last_became_peered":"2023-09-23T15:12:41.857745+0000","last_unstale":"2023-09-23T15:31:52.893373+0000","last_undegraded":"2023-09-23T15:31:52.893373+0000","last_fullsized":"2023-09-23T15:31:52.893373+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:28:40.111967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":617,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897472+0000","last_change":"2023-09-23T15:30:28.373196+0000","last_active":"2023-09-23T15:31:52.897472+0000","last_peered":"2023-09-23T15:31:52.897472+0000","last_clean":"2023-09-23T15:31:52.897472+0000","last_became_active":"2023-09-23T15:30:26.699434+0000","last_became_peered":"2023-09-23T15:30:26.699434+0000","last_unstale":"2023-09-23T15:31:52.897472+0000","last_undegraded":"2023-09-23T15:31:52.897472+0000","last_fullsized":"2023-09-23T15:31:52.897472+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:26.970670+0000","scrub_duration":0.0018471900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":480,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900825+0000","last_change":"2023-09-23T15:27:23.328159+0000","last_active":"2023-09-23T15:31:52.900825+0000","last_peered":"2023-09-23T15:31:52.900825+0000","last_clean":"2023-09-23T15:31:52.900825+0000","last_became_active":"2023-09-23T15:27:23.314466+0000","last_became_peered":"2023-09-23T15:27:23.314466+0000","last_unstale":"2023-09-23T15:31:52.900825+0000","last_undegraded":"2023-09-23T15:31:52.900825+0000","last_fullsized":"2023-09-23T15:31:52.900825+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:43:12.910890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894806+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:31:52.894806+0000","last_peered":"2023-09-23T15:31:52.894806+0000","last_clean":"2023-09-23T15:31:52.894806+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:31:52.894806+0000","last_undegraded":"2023-09-23T15:31:52.894806+0000","last_fullsized":"2023-09-23T15:31:52.894806+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":460,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.900974+0000","last_change":"2023-09-23T15:27:23.322079+0000","last_active":"2023-09-23T15:31:52.900974+0000","last_peered":"2023-09-23T15:31:52.900974+0000","last_clean":"2023-09-23T15:31:52.900974+0000","last_became_active":"2023-09-23T15:27:23.314487+0000","last_became_peered":"2023-09-23T15:27:23.314487+0000","last_unstale":"2023-09-23T15:31:52.900974+0000","last_undegraded":"2023-09-23T15:31:52.900974+0000","last_fullsized":"2023-09-23T15:31:52.900974+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:18.961610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.894261+0000","last_change":"2023-09-23T15:27:23.319962+0000","last_active":"2023-09-23T15:31:52.894261+0000","last_peered":"2023-09-23T15:31:52.894261+0000","last_clean":"2023-09-23T15:31:52.894261+0000","last_became_active":"2023-09-23T15:27:23.319424+0000","last_became_peered":"2023-09-23T15:27:23.319424+0000","last_unstale":"2023-09-23T15:31:52.894261+0000","last_undegraded":"2023-09-23T15:31:52.894261+0000","last_fullsized":"2023-09-23T15:31:52.894261+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:51.167361+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":627,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.899141+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:31:52.899141+0000","last_peered":"2023-09-23T15:31:52.899141+0000","last_clean":"2023-09-23T15:31:52.899141+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:31:52.899141+0000","last_undegraded":"2023-09-23T15:31:52.899141+0000","last_fullsized":"2023-09-23T15:31:52.899141+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":72,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.897710+0000","last_change":"2023-09-23T15:30:26.699597+0000","last_active":"2023-09-23T15:31:52.897710+0000","last_peered":"2023-09-23T15:31:52.897710+0000","last_clean":"2023-09-23T15:31:52.897710+0000","last_became_active":"2023-09-23T15:30:26.699233+0000","last_became_peered":"2023-09-23T15:30:26.699233+0000","last_unstale":"2023-09-23T15:31:52.897710+0000","last_undegraded":"2023-09-23T15:31:52.897710+0000","last_fullsized":"2023-09-23T15:31:52.897710+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.804745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":364,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893032+0000","last_change":"2023-09-23T15:30:26.685257+0000","last_active":"2023-09-23T15:31:52.893032+0000","last_peered":"2023-09-23T15:31:52.893032+0000","last_clean":"2023-09-23T15:31:52.893032+0000","last_became_active":"2023-09-23T15:30:26.684954+0000","last_became_peered":"2023-09-23T15:30:26.684954+0000","last_unstale":"2023-09-23T15:31:52.893032+0000","last_undegraded":"2023-09-23T15:31:52.893032+0000","last_fullsized":"2023-09-23T15:31:52.893032+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:49.244950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":620,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.905918+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:31:52.905918+0000","last_peered":"2023-09-23T15:31:52.905918+0000","last_clean":"2023-09-23T15:31:52.905918+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:31:52.905918+0000","last_undegraded":"2023-09-23T15:31:52.905918+0000","last_fullsized":"2023-09-23T15:31:52.905918+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":395,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.908530+0000","last_change":"2023-09-23T15:05:44.722465+0000","last_active":"2023-09-23T15:31:52.908530+0000","last_peered":"2023-09-23T15:31:52.908530+0000","last_clean":"2023-09-23T15:31:52.908530+0000","last_became_active":"2023-09-23T15:05:44.722036+0000","last_became_peered":"2023-09-23T15:05:44.722036+0000","last_unstale":"2023-09-23T15:31:52.908530+0000","last_undegraded":"2023-09-23T15:31:52.908530+0000","last_fullsized":"2023-09-23T15:31:52.908530+0000","mapping_epoch":105,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":106,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:41:50.985875+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":82,"reported_epoch":439,"state":"active+clean","last_fresh":"2023-09-23T15:31:52.893855+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:31:52.893855+0000","last_peered":"2023-09-23T15:31:52.893855+0000","last_clean":"2023-09-23T15:31:52.893855+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:31:52.893855+0000","last_undegraded":"2023-09-23T15:31:52.893855+0000","last_fullsized":"2023-09-23T15:31:52.893855+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5080023064,"num_objects":77516,"num_object_clones":0,"num_object_copies":232548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":77516,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":134702,"num_write_kb":6790913,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15244144640,"data_stored":15244132424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109302,"ondisk_log_size":109302,"up":48,"acting":48,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":34,"num_bytes_recovered":9774,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":122,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768758,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1364704,"kb_used_data":1236632,"kb_used_omap":13,"kb_used_meta":128050,"kb_avail":92380448,"statfs":{"total":95995035648,"available":94597578752,"internally_reserved":0,"allocated":1266311168,"data_stored":1264870265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":131123294},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":7.742,"5min":5.9420000000000002,"15min":5.8419999999999996},"min":{"1min":0.252,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":75.201999999999998,"5min":80,"15min":80},"last":1.524},{"interface":"front","average":{"1min":7.6130000000000004,"5min":5.8869999999999996,"15min":5.9009999999999998},"min":{"1min":0.32100000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":75.186000000000007,"5min":80.117000000000004,"15min":80.117000000000004},"last":1.5389999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.905,"5min":10.943,"15min":17.835999999999999},"min":{"1min":0.26700000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":8.9930000000000003,"5min":157.096,"15min":157.096},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.083,"5min":10.696,"15min":17.853000000000002},"min":{"1min":0.32200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.8730000000000002,"5min":157.011,"15min":157.011},"last":0.27700000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0790000000000002,"5min":1.895,"15min":1.1399999999999999},"min":{"1min":0.29599999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.1500000000000004,"5min":44.451999999999998,"15min":44.451999999999998},"last":3.5870000000000002},{"interface":"front","average":{"1min":1.6579999999999999,"5min":1.6890000000000001,"15min":1.202},"min":{"1min":0.33100000000000002,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":6.069,"5min":44.384999999999998,"15min":44.384999999999998},"last":1.806}]},{"osd":3,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.73,"5min":3.3620000000000001,"15min":3.8340000000000001},"min":{"1min":0.30099999999999999,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":7.0309999999999997,"5min":76.019000000000005,"15min":76.019000000000005},"last":2.4220000000000002},{"interface":"front","average":{"1min":1.9339999999999999,"5min":3.7229999999999999,"15min":4.2279999999999998},"min":{"1min":0.33500000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.0709999999999997,"5min":76.317999999999998,"15min":76.317999999999998},"last":3.6589999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.587,"5min":2.851,"15min":1.329},"min":{"1min":0.29199999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":9.923,"5min":170.15799999999999,"15min":170.15799999999999},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.5369999999999999,"5min":2.851,"15min":1.28},"min":{"1min":0.30399999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":5.1150000000000002,"5min":170.017,"15min":170.017},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.78000000000000003,"15min":0.79800000000000004},"min":{"1min":0.32600000000000001,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":1.9870000000000001,"5min":3.742,"15min":3.742},"last":0.32700000000000001},{"interface":"front","average":{"1min":1.409,"5min":1.0109999999999999,"15min":0.82099999999999995},"min":{"1min":0.36699999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":6.2329999999999997,"5min":6.2329999999999997,"15min":6.2329999999999997},"last":0.46200000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:31:47 2023","interfaces":[{"interface":"back","average":{"1min":1.228,"5min":3.9550000000000001,"15min":1.8029999999999999},"min":{"1min":0.34799999999999998,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":3.4140000000000001,"5min":262.38099999999997,"15min":262.38099999999997},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.522,"5min":4.0780000000000003,"15min":1.7909999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.2850000000000001,"5min":261.488,"15min":261.488},"last":0.5}]}]},{"osd":6,"up_from":304,"seq":1305670058101,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2347736,"kb_used_data":2175760,"kb_used_omap":13,"kb_used_meta":171954,"kb_avail":91397416,"statfs":{"total":95995035648,"available":93590953984,"internally_reserved":0,"allocated":2227978240,"data_stored":2226544882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":176081251},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":3.2909999999999999,"5min":2.5619999999999998,"15min":2.9089999999999998},"min":{"1min":0.26700000000000002,"5min":0.224,"15min":0.224},"max":{"1min":17.914999999999999,"5min":49.441000000000003,"15min":184.53200000000001},"last":44.853999999999999},{"interface":"front","average":{"1min":5.1619999999999999,"5min":2.7789999999999999,"15min":3.194},"min":{"1min":0.42899999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":17.510999999999999,"5min":48.880000000000003,"15min":86.227999999999994},"last":40.959000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":1.7549999999999999,"15min":2.7069999999999999},"min":{"1min":0.25800000000000001,"5min":0.221,"15min":0.221},"max":{"1min":6.7290000000000001,"5min":9.9710000000000001,"15min":86.534000000000006},"last":0.83899999999999997},{"interface":"front","average":{"1min":1.4119999999999999,"5min":1.3799999999999999,"15min":2.5350000000000001},"min":{"1min":0.29799999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.9580000000000002,"5min":6.5419999999999998,"15min":86.144000000000005},"last":0.58999999999999997}]},{"osd":2,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.0190000000000001,"15min":1.7849999999999999},"min":{"1min":0.28299999999999997,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":10.371,"5min":10.371,"15min":10.371},"last":0.28000000000000003},{"interface":"front","average":{"1min":3.387,"5min":2.0110000000000001,"15min":1.8759999999999999},"min":{"1min":0.38900000000000001,"5min":0.36499999999999999,"15min":0.307},"max":{"1min":10.388,"5min":10.388,"15min":10.388},"last":0.879}]},{"osd":3,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1890000000000001,"5min":3.371,"15min":2.6579999999999999},"min":{"1min":0.30299999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.644,"5min":142.67599999999999,"15min":142.67599999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":4.0039999999999996,"5min":4.1360000000000001,"15min":3.0059999999999998},"min":{"1min":0.46000000000000002,"5min":0.34699999999999998,"15min":0.33500000000000002},"max":{"1min":10.478,"5min":142.541,"15min":142.541},"last":0.36099999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":2.532,"15min":1.516},"min":{"1min":0.26500000000000001,"5min":0.24099999999999999,"15min":0.23999999999999999},"max":{"1min":4.3710000000000004,"5min":128.11600000000001,"15min":128.11600000000001},"last":0.52000000000000002},{"interface":"front","average":{"1min":2.5960000000000001,"5min":2.964,"15min":2.173},"min":{"1min":0.31900000000000001,"5min":0.29699999999999999,"15min":0.26400000000000001},"max":{"1min":8.2449999999999992,"5min":128.095,"15min":128.095},"last":0.32800000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.371,"5min":4.4539999999999997,"15min":2.8300000000000001},"min":{"1min":0.35099999999999998,"5min":0.248,"15min":0.248},"max":{"1min":9.2560000000000002,"5min":130.91300000000001,"15min":130.91300000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.83099999999999996,"5min":3.9950000000000001,"15min":2.4220000000000002},"min":{"1min":0.40400000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":2.1960000000000002,"5min":130.92599999999999,"15min":130.92599999999999},"last":4.4400000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:31:51 2023","interfaces":[{"interface":"back","average":{"1min":2.4119999999999999,"5min":1.5029999999999999,"15min":1.325},"min":{"1min":0.38900000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.5090000000000003,"5min":9.5090000000000003,"15min":9.5090000000000003},"last":10.087},{"interface":"front","average":{"1min":1.018,"5min":1.3169999999999999,"15min":1.4219999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":3.4660000000000002,"5min":6.5709999999999997,"15min":6.5709999999999997},"last":0.61899999999999999}]}]},{"osd":1,"up_from":183,"seq":785979015401,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1601352,"kb_used_data":975424,"kb_used_omap":6,"kb_used_meta":625913,"kb_avail":92143800,"statfs":{"total":95995035648,"available":94355251200,"internally_reserved":0,"allocated":998834176,"data_stored":996614312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":640934987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":6.5339999999999998,"5min":4.4089999999999998,"15min":3.4399999999999999},"min":{"1min":0.22,"5min":0.19900000000000001,"15min":0.191},"max":{"1min":96.323999999999998,"5min":101.27800000000001,"15min":101.27800000000001},"last":0.216},{"interface":"front","average":{"1min":6.7939999999999996,"5min":4.3129999999999997,"15min":3.7469999999999999},"min":{"1min":0.26400000000000001,"5min":0.22500000000000001,"15min":0.21099999999999999},"max":{"1min":96.188999999999993,"5min":101.051,"15min":101.051},"last":2.6499999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.6829999999999998,"5min":4.8289999999999997,"15min":4.0789999999999997},"min":{"1min":0.251,"5min":0.247,"15min":0.221},"max":{"1min":13.984999999999999,"5min":186.304,"15min":186.304},"last":3.6520000000000001},{"interface":"front","average":{"1min":2.8130000000000002,"5min":4.4989999999999997,"15min":3.3620000000000001},"min":{"1min":0.32300000000000001,"5min":0.216,"15min":0.216},"max":{"1min":13.795999999999999,"5min":186.541,"15min":186.541},"last":0.57099999999999995}]},{"osd":3,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.274,"5min":2.419,"15min":4.5129999999999999},"min":{"1min":0.27900000000000003,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":13.744,"5min":15.191000000000001,"15min":211.405},"last":0.36499999999999999},{"interface":"front","average":{"1min":2.2170000000000001,"5min":2.2429999999999999,"15min":4.5700000000000003},"min":{"1min":0.29199999999999998,"5min":0.26400000000000001,"15min":0.246},"max":{"1min":6.915,"5min":14.609999999999999,"15min":210.988},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":3.1339999999999999,"5min":3.3740000000000001,"15min":3.4900000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.29799999999999999},"max":{"1min":13.840999999999999,"5min":72.602000000000004,"15min":80.570999999999998},"last":3.722},{"interface":"front","average":{"1min":2.2949999999999999,"5min":2.4889999999999999,"15min":2.9420000000000002},"min":{"1min":0.36699999999999999,"5min":0.312,"15min":0.308},"max":{"1min":7.0209999999999999,"5min":73.454999999999998,"15min":80.75},"last":0.33100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:31:24 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":3.7040000000000002,"15min":3.363},"min":{"1min":0.42699999999999999,"5min":0.312,"15min":0.27400000000000002},"max":{"1min":7.976,"5min":173.18799999999999,"15min":173.18799999999999},"last":2.722},{"interface":"front","average":{"1min":3.476,"5min":4.415,"15min":3.7759999999999998},"min":{"1min":0.42799999999999999,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":13.872,"5min":172.822,"15min":172.822},"last":3.694}]},{"osd":6,"last update":"Sat Sep 23 15:31:43 2023","interfaces":[{"interface":"back","average":{"1min":3.0760000000000001,"5min":2.8250000000000002,"15min":2.9990000000000001},"min":{"1min":0.377,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":13.912000000000001,"5min":54.198,"15min":54.198},"last":3.7530000000000001},{"interface":"front","average":{"1min":1.4159999999999999,"5min":1.4710000000000001,"15min":2.0430000000000001},"min":{"1min":0.41899999999999998,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":4.1210000000000004,"5min":8.5800000000000001,"15min":15.605},"last":2.6920000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:31:50 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.8169999999999999,"15min":2.0310000000000001},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":4.9139999999999997,"5min":8.6120000000000001,"15min":8.6120000000000001},"last":2.7490000000000001},{"interface":"front","average":{"1min":2.484,"5min":3.512,"15min":4.5519999999999996},"min":{"1min":0.46400000000000002,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":6.2759999999999998,"5min":54.223999999999997,"15min":54.223999999999997},"last":3.7839999999999998}]}]},{"osd":0,"up_from":95,"seq":408021893448,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3944056,"kb_used_data":3128560,"kb_used_omap":8,"kb_used_meta":815351,"kb_avail":89801096,"statfs":{"total":95995035648,"available":91956322304,"internally_reserved":0,"allocated":3203645440,"data_stored":3201562755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":834920020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0950000000000002,"5min":4.827,"15min":4.6230000000000002},"min":{"1min":0.25900000000000001,"5min":0.216,"15min":0.18099999999999999},"max":{"1min":12.516,"5min":141.52199999999999,"15min":141.52199999999999},"last":1.6519999999999999},{"interface":"front","average":{"1min":2.5270000000000001,"5min":4.8819999999999997,"15min":4.2990000000000004},"min":{"1min":0.24099999999999999,"5min":0.216,"15min":0.20999999999999999},"max":{"1min":8.2260000000000009,"5min":141.62799999999999,"15min":141.62799999999999},"last":3.4009999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":6.968,"5min":3.073,"15min":2.5659999999999998},"min":{"1min":0.33600000000000002,"5min":0.23400000000000001,"15min":0.20200000000000001},"max":{"1min":66.132999999999996,"5min":66.132999999999996,"15min":66.132999999999996},"last":5.4100000000000001},{"interface":"front","average":{"1min":8.0739999999999998,"5min":2.7440000000000002,"15min":2.7240000000000002},"min":{"1min":0.26300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":68.296999999999997,"5min":68.296999999999997,"15min":68.296999999999997},"last":1.3049999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":2.726,"5min":5.6840000000000002,"15min":5.4950000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":10.722,"5min":162.154,"15min":197.78899999999999},"last":2.4319999999999999},{"interface":"front","average":{"1min":2.5139999999999998,"5min":5.1989999999999998,"15min":5.7460000000000004},"min":{"1min":0.40699999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":12.866,"5min":162.215,"15min":200.58799999999999},"last":1.6859999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":4.7480000000000002,"5min":2.6800000000000002,"15min":2.0979999999999999},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28000000000000003},"max":{"1min":36.393000000000001,"5min":36.393000000000001,"15min":36.393000000000001},"last":3.2890000000000001},{"interface":"front","average":{"1min":3.734,"5min":2.5960000000000001,"15min":2.1040000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":36.219999999999999,"5min":36.219999999999999,"15min":36.219999999999999},"last":4.2240000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4119999999999999,"5min":2.4420000000000002,"15min":2.3039999999999998},"min":{"1min":0.312,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":33.158999999999999,"5min":44.57,"15min":64.349999999999994},"last":1.331},{"interface":"front","average":{"1min":4.5940000000000003,"5min":2.8919999999999999,"15min":2.4020000000000001},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.312},"max":{"1min":33.140000000000001,"5min":44.588000000000001,"15min":44.588000000000001},"last":3.3730000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":5.1529999999999996,"5min":3.0150000000000001,"15min":2.7400000000000002},"min":{"1min":0.505,"5min":0.40500000000000003,"15min":0.379},"max":{"1min":41.317999999999998,"5min":41.317999999999998,"15min":41.317999999999998},"last":3.3149999999999999},{"interface":"front","average":{"1min":4.3639999999999999,"5min":3.0630000000000002,"15min":3.0630000000000002},"min":{"1min":0.39000000000000001,"5min":0.377,"15min":0.371},"max":{"1min":41.195,"5min":41.195,"15min":41.195},"last":4.2590000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:31:46 2023","interfaces":[{"interface":"back","average":{"1min":3.4580000000000002,"5min":2.552,"15min":2.3799999999999999},"min":{"1min":0.34399999999999997,"5min":0.318,"15min":0.318},"max":{"1min":12.593999999999999,"5min":36.848999999999997,"15min":36.848999999999997},"last":1.349},{"interface":"front","average":{"1min":3.4689999999999999,"5min":3.3849999999999998,"15min":3.8969999999999998},"min":{"1min":0.54200000000000004,"5min":0.41799999999999998,"15min":0.41799999999999998},"max":{"1min":11.214,"5min":36.947000000000003,"15min":36.947000000000003},"last":3.339}]}]},{"osd":2,"up_from":105,"seq":450971566398,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2607908,"kb_used_data":1881884,"kb_used_omap":11,"kb_used_meta":726004,"kb_avail":91137244,"statfs":{"total":95995035648,"available":93324537856,"internally_reserved":0,"allocated":1927049216,"data_stored":1924982972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":743428880},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":3.0960000000000001,"5min":2.7290000000000001,"15min":5.157},"min":{"1min":0.224,"5min":0.20499999999999999,"15min":0.17199999999999999},"max":{"1min":12.43,"5min":70.307000000000002,"15min":235.517},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.3069999999999999,"5min":2.359,"15min":5.1180000000000003},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.5380000000000003,"5min":70.286000000000001,"15min":235.33799999999999},"last":0.70299999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:31:25 2023","interfaces":[{"interface":"back","average":{"1min":4.4379999999999997,"5min":3.2109999999999999,"15min":3.5510000000000002},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.20499999999999999},"max":{"1min":65.906999999999996,"5min":65.906999999999996,"15min":71.819000000000003},"last":1.7250000000000001},{"interface":"front","average":{"1min":4.391,"5min":3.0129999999999999,"15min":3.3180000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":65.222999999999999,"5min":65.222999999999999,"15min":71.805000000000007},"last":0.66100000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:32:01 2023","interfaces":[{"interface":"back","average":{"1min":2.2709999999999999,"5min":3.202,"15min":4.069},"min":{"1min":0.26800000000000002,"5min":0.216,"15min":0.216},"max":{"1min":6.6529999999999996,"5min":75.701999999999998,"15min":77.031999999999996},"last":0.55800000000000005},{"interface":"front","average":{"1min":2.6589999999999998,"5min":3.1560000000000001,"15min":4.2789999999999999},"min":{"1min":0.28100000000000003,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":11.420999999999999,"5min":75.388999999999996,"15min":76.742000000000004},"last":1.8540000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":5.1719999999999997,"5min":2.262,"15min":1.9019999999999999},"min":{"1min":0.29799999999999999,"5min":0.28899999999999998,"15min":0.28000000000000003},"max":{"1min":63.747999999999998,"5min":63.747999999999998,"15min":63.747999999999998},"last":0.51400000000000001},{"interface":"front","average":{"1min":6.2210000000000001,"5min":2.7160000000000002,"15min":2.0590000000000002},"min":{"1min":0.36599999999999999,"5min":0.31900000000000001,"15min":0.26200000000000001},"max":{"1min":63.954000000000001,"5min":63.954000000000001,"15min":63.954000000000001},"last":0.72199999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:31:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":3.3919999999999999,"15min":2.399},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.31900000000000001},"max":{"1min":6.2460000000000004,"5min":107.021,"15min":107.021},"last":1.4379999999999999},{"interface":"front","average":{"1min":2.262,"5min":3.2629999999999999,"15min":2.3559999999999999},"min":{"1min":0.33000000000000002,"5min":0.318,"15min":0.317},"max":{"1min":8.4559999999999995,"5min":107.136,"15min":107.136},"last":0.54300000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:31:38 2023","interfaces":[{"interface":"back","average":{"1min":9.9670000000000005,"5min":3.5009999999999999,"15min":2.6000000000000001},"min":{"1min":0.35499999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":156.16,"5min":156.16,"15min":156.16},"last":1.409},{"interface":"front","average":{"1min":11.045999999999999,"5min":3.625,"15min":2.3719999999999999},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":155.226,"5min":155.226,"15min":155.226},"last":0.63700000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:31:49 2023","interfaces":[{"interface":"back","average":{"1min":2.9809999999999999,"5min":3.5499999999999998,"15min":2.843},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":12.282,"5min":112.70699999999999,"15min":112.70699999999999},"last":0.753},{"interface":"front","average":{"1min":2.2570000000000001,"5min":3.661,"15min":3.423},"min":{"1min":0.438,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":6.5519999999999996,"5min":109.402,"15min":109.402},"last":1.4610000000000001}]}]},{"osd":3,"up_from":110,"seq":472446402870,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2382076,"kb_used_data":1558516,"kb_used_omap":3,"kb_used_meta":823548,"kb_avail":91363076,"statfs":{"total":95995035648,"available":93555789824,"internally_reserved":0,"allocated":1595920384,"data_stored":1593833238,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":843313321},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":8.9039999999999999,"5min":5.4000000000000004,"15min":4.056},"min":{"1min":0.22500000000000001,"5min":0.221,"15min":0.183},"max":{"1min":141.934,"5min":141.934,"15min":141.934},"last":0.61899999999999999},{"interface":"front","average":{"1min":9.2690000000000001,"5min":5.9290000000000003,"15min":4.54},"min":{"1min":0.28899999999999998,"5min":0.189,"15min":0.184},"max":{"1min":141.72800000000001,"5min":141.72800000000001,"15min":141.72800000000001},"last":3.3889999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:31:25 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":4.1909999999999998,"15min":4.4039999999999999},"min":{"1min":0.28999999999999998,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":7.96,"5min":121.26900000000001,"15min":126.875},"last":0.45300000000000001},{"interface":"front","average":{"1min":2.1240000000000001,"5min":4.6130000000000004,"15min":4.8150000000000004},"min":{"1min":0.30099999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.401999999999999,"5min":121.012,"15min":128.249},"last":0.33300000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":11.366,"5min":3.5179999999999998,"15min":2.8900000000000001},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":144.35599999999999,"5min":144.35599999999999,"15min":144.35599999999999},"last":0.29799999999999999},{"interface":"front","average":{"1min":11.470000000000001,"5min":3.988,"15min":3.7519999999999998},"min":{"1min":0.26100000000000001,"5min":0.246,"15min":0.246},"max":{"1min":144.494,"5min":144.494,"15min":144.494},"last":2.2360000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.712,"15min":1.9239999999999999},"min":{"1min":0.39300000000000002,"5min":0.28999999999999998,"15min":0.27500000000000002},"max":{"1min":4.6040000000000001,"5min":17.027999999999999,"15min":36.551000000000002},"last":1.9470000000000001},{"interface":"front","average":{"1min":1.611,"5min":1.5429999999999999,"15min":1.754},"min":{"1min":0.373,"5min":0.28599999999999998,"15min":0.27400000000000002},"max":{"1min":5.3360000000000003,"5min":7.6630000000000003,"15min":13.115},"last":1.829}]},{"osd":5,"last update":"Sat Sep 23 15:31:03 2023","interfaces":[{"interface":"back","average":{"1min":7.3620000000000001,"5min":3.9889999999999999,"15min":3.282},"min":{"1min":0.49199999999999999,"5min":0.315,"15min":0.29599999999999999},"max":{"1min":122.452,"5min":122.452,"15min":148.619},"last":1.853},{"interface":"front","average":{"1min":6.9669999999999996,"5min":3.6619999999999999,"15min":3.004},"min":{"1min":0.42299999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":122.33499999999999,"5min":122.33499999999999,"15min":148.56999999999999},"last":0.68999999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:31:34 2023","interfaces":[{"interface":"back","average":{"1min":2.2530000000000001,"5min":2.4870000000000001,"15min":2.637},"min":{"1min":0.36399999999999999,"5min":0.34999999999999998,"15min":0.32700000000000001},"max":{"1min":7.5359999999999996,"5min":69.721999999999994,"15min":84.665000000000006},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.4239999999999999,"5min":2.867,"15min":3.359},"min":{"1min":0.46800000000000003,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":5.1399999999999997,"5min":69.852999999999994,"15min":84.897999999999996},"last":1.9059999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:31:50 2023","interfaces":[{"interface":"back","average":{"1min":2.637,"5min":7.1130000000000004,"15min":10.536},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":9.3729999999999993,"5min":216.56899999999999,"15min":216.56899999999999},"last":0.51500000000000001},{"interface":"front","average":{"1min":3.0960000000000001,"5min":7.9459999999999997,"15min":11.596},"min":{"1min":0.58099999999999996,"5min":0.432,"15min":0.432},"max":{"1min":13.43,"5min":216.548,"15min":216.548},"last":2.8519999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986637,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2946780,"kb_used_data":2486164,"kb_used_omap":0,"kb_used_meta":460480,"kb_avail":90798372,"statfs":{"total":95995035648,"available":92977532928,"internally_reserved":0,"allocated":2545831936,"data_stored":2544394539,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":471531520},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.8519999999999999,"5min":2.4670000000000001,"15min":4.0609999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":33.500999999999998,"5min":36.865000000000002,"15min":162.75999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":3.9119999999999999,"5min":2.601,"15min":4.0970000000000004},"min":{"1min":0.25900000000000001,"5min":0.22700000000000001,"15min":0.22600000000000001},"max":{"1min":33.021999999999998,"5min":37.081000000000003,"15min":162.34800000000001},"last":5.4770000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":5.0419999999999998,"15min":3.548},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.245},"max":{"1min":6.8789999999999996,"5min":144.76900000000001,"15min":144.76900000000001},"last":0.29999999999999999},{"interface":"front","average":{"1min":1.6499999999999999,"5min":5.3399999999999999,"15min":3.7559999999999998},"min":{"1min":0.27700000000000002,"5min":0.25,"15min":0.25},"max":{"1min":6.6440000000000001,"5min":144.58799999999999,"15min":144.58799999999999},"last":1.1579999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:31:27 2023","interfaces":[{"interface":"back","average":{"1min":3.9580000000000002,"5min":1.8149999999999999,"15min":2.6379999999999999},"min":{"1min":0.433,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":30.489000000000001,"5min":30.489000000000001,"15min":78.751000000000005},"last":1.6659999999999999},{"interface":"front","average":{"1min":2.9780000000000002,"5min":1.637,"15min":2.6949999999999998},"min":{"1min":0.43099999999999999,"5min":0.315,"15min":0.28999999999999998},"max":{"1min":30.524000000000001,"5min":30.524000000000001,"15min":77.930999999999997},"last":4.4029999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:31:10 2023","interfaces":[{"interface":"back","average":{"1min":1.6479999999999999,"5min":2.698,"15min":3.7709999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.30399999999999999},"max":{"1min":8.3320000000000007,"5min":96.587000000000003,"15min":156.46000000000001},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.6779999999999999,"5min":2.8069999999999999,"15min":3.8879999999999999},"min":{"1min":0.371,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":6.6920000000000002,"5min":96.784999999999997,"15min":156.74299999999999},"last":4.0389999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:31:16 2023","interfaces":[{"interface":"back","average":{"1min":1.6160000000000001,"5min":1.3720000000000001,"15min":1.069},"min":{"1min":0.33000000000000002,"5min":0.26500000000000001,"15min":0.221},"max":{"1min":8.3829999999999991,"5min":40.685000000000002,"15min":40.685000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":2.508,"5min":1.6140000000000001,"15min":1.2050000000000001},"min":{"1min":0.29099999999999998,"5min":0.27500000000000002,"15min":0.246},"max":{"1min":12.882999999999999,"5min":38.439,"15min":38.439},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:31:39 2023","interfaces":[{"interface":"back","average":{"1min":2.3330000000000002,"5min":3.7469999999999999,"15min":2.1179999999999999},"min":{"1min":0.309,"5min":0.309,"15min":0.28999999999999998},"max":{"1min":9.6280000000000001,"5min":228.93199999999999,"15min":228.93199999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":2.8969999999999998,"5min":3.8359999999999999,"15min":2.5699999999999998},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":10.446,"5min":232.54900000000001,"15min":232.54900000000001},"last":0.65100000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:31:44 2023","interfaces":[{"interface":"back","average":{"1min":11.157999999999999,"5min":3.2000000000000002,"15min":2},"min":{"1min":0.36399999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":183.94800000000001,"5min":183.94800000000001,"15min":183.94800000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":10.839,"5min":3.0590000000000002,"15min":1.724},"min":{"1min":0.41899999999999998,"5min":0.314,"15min":0.314},"max":{"1min":183.72399999999999,"5min":183.72399999999999,"15min":183.72399999999999},"last":0.69399999999999995}]}]},{"osd":5,"up_from":39,"seq":167503725000,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2098124,"kb_used_data":1574596,"kb_used_omap":0,"kb_used_meta":523520,"kb_avail":91647028,"statfs":{"total":95995035648,"available":93846556672,"internally_reserved":0,"allocated":1612386304,"data_stored":1610961504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":536084480},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:31:35 2023","interfaces":[{"interface":"back","average":{"1min":5.7169999999999996,"5min":10.461,"15min":6.4630000000000001},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":54.673999999999999,"5min":397.84199999999998,"15min":397.84199999999998},"last":3.5009999999999999},{"interface":"front","average":{"1min":5.8179999999999996,"5min":10.352,"15min":6.3979999999999997},"min":{"1min":0.28799999999999998,"5min":0.25800000000000001,"15min":0.23300000000000001},"max":{"1min":54.338999999999999,"5min":397.28100000000001,"15min":397.28100000000001},"last":0.25600000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:31:21 2023","interfaces":[{"interface":"back","average":{"1min":2.1779999999999999,"5min":4.0129999999999999,"15min":3.9550000000000001},"min":{"1min":0.24299999999999999,"5min":0.23699999999999999,"15min":0.23499999999999999},"max":{"1min":13.972,"5min":193.66499999999999,"15min":193.66499999999999},"last":0.48399999999999999},{"interface":"front","average":{"1min":2.7040000000000002,"5min":4.1900000000000004,"15min":3.673},"min":{"1min":0.33200000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":19.850000000000001,"5min":193.52099999999999,"15min":193.52099999999999},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:31:35 2023","interfaces":[{"interface":"back","average":{"1min":2.7589999999999999,"5min":3.3679999999999999,"15min":3.1040000000000001},"min":{"1min":0.315,"5min":0.253,"15min":0.25},"max":{"1min":15.904,"5min":97.126000000000005,"15min":193.00999999999999},"last":2.7330000000000001},{"interface":"front","average":{"1min":2.363,"5min":3.4740000000000002,"15min":3.3279999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.27900000000000003},"max":{"1min":19.812999999999999,"5min":96.516000000000005,"15min":193.15700000000001},"last":0.76700000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:31:05 2023","interfaces":[{"interface":"back","average":{"1min":2.2170000000000001,"5min":3.4020000000000001,"15min":5.8399999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":14.082000000000001,"5min":150.75899999999999,"15min":266.70299999999997},"last":1.256},{"interface":"front","average":{"1min":2.2080000000000002,"5min":3.3069999999999999,"15min":6.0110000000000001},"min":{"1min":0.313,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":15.26,"5min":150.62299999999999,"15min":266.80399999999997},"last":0.52100000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:31:21 2023","interfaces":[{"interface":"back","average":{"1min":2.0640000000000001,"5min":1.294,"15min":1.214},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":14.068,"5min":14.068,"15min":14.068},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.4649999999999999,"5min":1.2829999999999999,"15min":1.2330000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":19.879999999999999,"5min":19.879999999999999,"15min":19.879999999999999},"last":1.3140000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.7310000000000001,"15min":2.0569999999999999},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":8.4139999999999997,"5min":35.667000000000002,"15min":35.667000000000002},"last":2.2370000000000001},{"interface":"front","average":{"1min":1.734,"5min":1.7,"15min":2.0830000000000002},"min":{"1min":0.34000000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.1829999999999998,"5min":36.140999999999998,"15min":36.140999999999998},"last":1.224}]},{"osd":7,"last update":"Sat Sep 23 15:31:48 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.474,"15min":1.3460000000000001},"min":{"1min":0.34799999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":4.609,"5min":16.065999999999999,"15min":16.065999999999999},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.8169999999999999,"5min":1.631,"15min":1.627},"min":{"1min":0.40100000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":8.3320000000000007,"5min":19.908000000000001,"15min":19.908000000000001},"last":0.59499999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3174109184,"data_stored":3174105112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":985071616,"data_stored":985071616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1902903296,"data_stored":1902903296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1576271872,"data_stored":1576271872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2539851776,"data_stored":2539847704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1595342848,"data_stored":1595342848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2217611264,"data_stored":2217607192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1252982784,"data_stored":1252982784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:32:07.823 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:32:07.823 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:32:07.823 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2023-09-23T15:32:07.823 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T15:32:07.824 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:07.878 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:07.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:07.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:07.899 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:07 smithi017 ceph-mon[215646]: from='client.46706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:32:07.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:07 smithi017 ceph-mon[222878]: from='client.46706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:32:07.986 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:08.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:07 smithi027 ceph-mon[195233]: from='client.46706 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:32:08.128 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:08.199 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:08] "GET /metrics HTTP/1.1" 200 36421 "" "Prometheus/2.43.0" 2023-09-23T15:32:08.234 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:08.312 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:08.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:08.436 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:08.520 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:08.674 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:08.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:08.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:08.687 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:32:08.704 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:32:08.704 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:32:08.704 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:32:08.705 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:32:08.705 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:32:08.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:08.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:08 smithi017 ceph-mon[215646]: pgmap v1301: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:08.924 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:08 smithi017 ceph-mon[222878]: pgmap v1301: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:08.944 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:09.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:08 smithi027 ceph-mon[195233]: pgmap v1301: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:09.422 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:09.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:09.526 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:09.597 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:10.071 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:10.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:10.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:10.118 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:32:10.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:10.231 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:10.270 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:10.290 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:10.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:10.395 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:10.465 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:10.490 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:10.587 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:10.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:10.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:10.706 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:10.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:10 smithi017 ceph-mon[222878]: pgmap v1302: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:10.801 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:10 smithi017 ceph-mon[215646]: pgmap v1302: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:10.801 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:11.052 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:11.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:11.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:10 smithi027 ceph-mon[195233]: pgmap v1302: 119 pgs: 119 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 805 op/s 2023-09-23T15:32:11.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:11.194 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:11.274 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:11.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:11.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:11.387 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:11.470 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:11.862 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:32:12.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1519816940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.102 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1519816940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1519816940' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:12.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: pgmap v1303: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: failed to encode map e441 with expected crc 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: failed to encode map e441 with expected crc 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[215646]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: pgmap v1303: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:32:12.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: failed to encode map e441 with expected crc 2023-09-23T15:32:12.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: failed to encode map e441 with expected crc 2023-09-23T15:32:12.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:12.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:12 smithi017 ceph-mon[222878]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: pgmap v1303: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: failed to encode map e441 with expected crc 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: failed to encode map e441 with expected crc 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:12 smithi027 ceph-mon[195233]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T15:32:13.126 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:13.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:13.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:13.232 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:13.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:13.368 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:13.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:13.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:13.479 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:13.483 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:13.561 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:13.688 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:13.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:13.696 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:13.796 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:13.832 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:13.930 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:13.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:14.044 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:14.152 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:14.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[215646]: pgmap v1305: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/137017709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[222878]: pgmap v1305: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:14.262 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/137017709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:14.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:14.320 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:14.335 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:14.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:14.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:14.415 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:14.458 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:14.464 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:14.543 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:14.556 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:14.572 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:14 smithi027 ceph-mon[195233]: pgmap v1305: 119 pgs: 119 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s 2023-09-23T15:32:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:14.601 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:14.602 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/137017709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.604 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:32:14.681 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:14.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:14.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:14.796 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:14.859 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:14 smithi017 conmon[215622]: 2023-09-23T15:32:14.649+0000 7f618a1a9700 -1 mon.a@0(leader).osd e441 definitely_dead 0 2023-09-23T15:32:14.866 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:14.969 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:14.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:15.076 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:15.124 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:15.244 INFO:teuthology.orchestra.run.smithi017.stderr:marked out osd.5. 2023-09-23T15:32:15.379 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:32:15.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T15:32:15.407 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:15 smithi017 ceph-mon[222878]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T15:32:15.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T15:32:15.408 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:15 smithi017 ceph-mon[215646]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T15:32:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T15:32:15.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:15 smithi027 ceph-mon[195233]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T15:32:15.726 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:15.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:15.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:15.830 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:15.862 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:15.914 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:15.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:15.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:16.019 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:16.047 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:16.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[222878]: pgmap v1306: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2023-09-23T15:32:16.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T15:32:16.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[222878]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T15:32:16.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[222878]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.363 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[222878]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[215646]: pgmap v1306: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2023-09-23T15:32:16.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T15:32:16.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[215646]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T15:32:16.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[215646]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.364 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:16 smithi017 ceph-mon[215646]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.508 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:16.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:16.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:16 smithi027 ceph-mon[195233]: pgmap v1306: 119 pgs: 119 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2023-09-23T15:32:16.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2636416275' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T15:32:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:16 smithi027 ceph-mon[195233]: osdmap e442: 8 total, 8 up, 7 in 2023-09-23T15:32:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:16 smithi027 ceph-mon[195233]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:16 smithi027 ceph-mon[195233]: failed to encode map e442 with expected crc 2023-09-23T15:32:16.611 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:16.643 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:16.793 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:16.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:16.824 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:16.898 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:16.901 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:32:16.901 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:32:16.928 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:16.977 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:16.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:16.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:17.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:17.111 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:17.217 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:17.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:17.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:17.322 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:17.451 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:17.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.606 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:17 smithi027 ceph-mon[195233]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.632 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:17.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:17.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[215646]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: osdmap e443: 8 total, 8 up, 7 in 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:17 smithi017 ceph-mon[222878]: failed to encode map e443 with expected crc 2023-09-23T15:32:17.742 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:17.796 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:17.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:32:18.075 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:18] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T15:32:18.099 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:18.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:18.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:18.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:18.304 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:18.413 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:32:18.445 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: pgmap v1309: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 441'8960 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.446 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9122 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 441'9070 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 441'8719 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9098 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.447 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: pgmap v1309: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 441'8960 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.448 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9122 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 441'9070 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 441'8719 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9098 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.449 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.450 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:18 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: pgmap v1309: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 631 op/s 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: 10.6 starting backfill to osd.1 from (0'0,0'0] MAX to 441'8960 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.603 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: 10.9 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9122 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: osdmap e444: 8 total, 8 up, 7 in 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e444 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: 10.a starting backfill to osd.2 from (0'0,0'0] MAX to 441'9070 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: 10.7 starting backfill to osd.2 from (0'0,0'0] MAX to 441'8719 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: 10.3 starting backfill to osd.4 from (0'0,0'0] MAX to 441'9098 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: osdmap e445: 8 total, 8 up, 7 in 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:18 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:18.843 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:18.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:18.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:19.102 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:19.179 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:19.219 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:19.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:19.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:19.350 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:19.430 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.535 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.536 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.537 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:19 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e445 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: osdmap e446: 8 total, 8 up, 7 in 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:19 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:19.728 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:19.800 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:19.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:19.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:19.912 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:19.974 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:20.054 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:20.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:20.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:20.160 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:20.298 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[215646]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[215646]: pgmap v1313: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 57 op/s 2023-09-23T15:32:20.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[222878]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.446 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:20 smithi017 ceph-mon[222878]: pgmap v1313: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 57 op/s 2023-09-23T15:32:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:20 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:20 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:20 smithi027 ceph-mon[195233]: failed to encode map e446 with expected crc 2023-09-23T15:32:20.600 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:20 smithi027 ceph-mon[195233]: pgmap v1313: 119 pgs: 1 unknown, 5 peering, 113 active+clean; 4.2 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 57 op/s 2023-09-23T15:32:20.870 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:20.876 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:20.977 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:21.056 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:21.338 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:32:21.380 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:32:21.380 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:32:21.381 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:32:21.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T15:32:21.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3644783743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:21.599 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:21 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:21.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3644783743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:21.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:21 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:21.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3644783743' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:21.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:21 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: pgmap v1314: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 451 op/s; 20255/202359 objects misplaced (10.009%); 12 MiB/s, 193 objects/s recovering 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:32:22.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[215646]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: pgmap v1314: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 451 op/s; 20255/202359 objects misplaced (10.009%); 12 MiB/s, 193 objects/s recovering 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:32:22.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:22 smithi017 ceph-mon[222878]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: pgmap v1314: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 12 KiB/s rd, 31 KiB/s wr, 451 op/s; 20255/202359 objects misplaced (10.009%); 12 MiB/s, 193 objects/s recovering 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: failed to encode map e447 with expected crc 2023-09-23T15:32:22.604 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:32:22.605 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:22 smithi027 ceph-mon[195233]: osdmap e447: 8 total, 8 up, 7 in 2023-09-23T15:32:23.635 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:23.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:23.742 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:23.793 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:23.870 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:23.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:23.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:23.979 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:24.071 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:24.073 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:24.231 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:32:24.231 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:32:24.264 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:24.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:24.373 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:24.441 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:24.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:24 smithi017 ceph-mon[215646]: pgmap v1316: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 373 op/s; 20255/202359 objects misplaced (10.009%); 10 MiB/s, 159 objects/s recovering 2023-09-23T15:32:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:24 smithi017 ceph-mon[222878]: pgmap v1316: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 373 op/s; 20255/202359 objects misplaced (10.009%); 10 MiB/s, 159 objects/s recovering 2023-09-23T15:32:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:24 smithi027 ceph-mon[195233]: pgmap v1316: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 9.5 KiB/s rd, 26 KiB/s wr, 373 op/s; 20255/202359 objects misplaced (10.009%); 10 MiB/s, 159 objects/s recovering 2023-09-23T15:32:25.107 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:25.107 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:25.107 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:25.134 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:25.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:25.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:25.250 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:25.294 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:25.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.302 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.303 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:32:25.519 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:32:25.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/446613734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:25.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/446613734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/446613734' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:26.123 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:26.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:26.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:26.210 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:32:26.233 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:26.274 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:26.388 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:26.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:26.496 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:26.539 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:26.644 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:26.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:26.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:26.699 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:26 smithi017 ceph-mon[215646]: pgmap v1317: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 755 op/s; 17294/192003 objects misplaced (9.007%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:32:26.700 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:26 smithi017 ceph-mon[222878]: pgmap v1317: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 755 op/s; 17294/192003 objects misplaced (9.007%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:32:26.758 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:26.804 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:26 smithi027 ceph-mon[195233]: pgmap v1317: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 7.4 KiB/s rd, 20 KiB/s wr, 755 op/s; 17294/192003 objects misplaced (9.007%); 24 MiB/s, 378 objects/s recovering 2023-09-23T15:32:26.818 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:26.898 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:26.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:26.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:27.005 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:27.061 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:27.181 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:27.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:27.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:27.297 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:27.350 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:27.442 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:27.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:27.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:27.551 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:27.643 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:27.670 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:27.779 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:27.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:27.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:27.844 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:27.896 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:27.982 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:28.302 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:28] "GET /metrics HTTP/1.1" 200 36422 "" "Prometheus/2.43.0" 2023-09-23T15:32:28.423 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:28.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:28.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:28.509 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:32:28.532 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:28.591 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:28.646 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:28.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:28.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:28.753 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:moved 27 / 357 (7.56302%) 2023-09-23T15:32:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:32:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 8.24621 (expected baseline 6.61168) 2023-09-23T15:32:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 44 pgs (0.72549 -> 0.862745 * mean) 2023-09-23T15:32:28.826 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 52 pgs (1.15686 -> 1.01961 * mean) 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0230 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0253 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 1.0000 -> 0.6293 2023-09-23T15:32:28.827 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 weight 1.0000 -> 0.8329 2023-09-23T15:32:28.832 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:28.838 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:32:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:28 smithi027 ceph-mon[195233]: pgmap v1318: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 692 op/s; 17294/191373 objects misplaced (9.037%); 21 MiB/s, 333 objects/s recovering 2023-09-23T15:32:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:28.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:28.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[215646]: pgmap v1318: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 692 op/s; 17294/191373 objects misplaced (9.037%); 21 MiB/s, 333 objects/s recovering 2023-09-23T15:32:28.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:28.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:28.874 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:28 smithi017 conmon[202315]: 2023-09-23T15:32:28.823+0000 7fdcb8de7700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T15:32:28.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[222878]: pgmap v1318: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 6.5 KiB/s rd, 17 KiB/s wr, 692 op/s; 17294/191373 objects misplaced (9.037%); 21 MiB/s, 333 objects/s recovering 2023-09-23T15:32:28.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:28.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:30.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[215646]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:30.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:30.165 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[215646]: pgmap v1319: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 5.5 KiB/s rd, 15 KiB/s wr, 581 op/s; 17294/191373 objects misplaced (9.037%); 17 MiB/s, 280 objects/s recovering 2023-09-23T15:32:30.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[222878]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:30.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:30.167 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:29 smithi017 ceph-mon[222878]: pgmap v1319: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 5.5 KiB/s rd, 15 KiB/s wr, 581 op/s; 17294/191373 objects misplaced (9.037%); 17 MiB/s, 280 objects/s recovering 2023-09-23T15:32:30.353 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:29 smithi027 ceph-mon[195233]: from='client.46733 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:30.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:30.354 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:29 smithi027 ceph-mon[195233]: pgmap v1319: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.9 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 5.5 KiB/s rd, 15 KiB/s wr, 581 op/s; 17294/191373 objects misplaced (9.037%); 17 MiB/s, 280 objects/s recovering 2023-09-23T15:32:31.382 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:31.392 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:31.493 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:31.566 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:31.671 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:32:31.672 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 3.0 2023-09-23T15:32:31.739 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:31.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:31.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:31.849 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:32.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:32.153 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:32.190 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:32.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:32 smithi017 ceph-mon[215646]: pgmap v1320: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 14719/180201 objects misplaced (8.168%); 22 MiB/s, 345 objects/s recovering 2023-09-23T15:32:32.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:32 smithi017 ceph-mon[222878]: pgmap v1320: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 14719/180201 objects misplaced (8.168%); 22 MiB/s, 345 objects/s recovering 2023-09-23T15:32:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:32.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:32.317 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:32.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:32 smithi027 ceph-mon[195233]: pgmap v1320: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 14719/180201 objects misplaced (8.168%); 22 MiB/s, 345 objects/s recovering 2023-09-23T15:32:32.371 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:32.460 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:32.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:32.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:32.575 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:32.619 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:32.632 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:32:32.633 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:32:32.633 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:32:32.633 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:32:32.633 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:32:32.679 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:32.689 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:32.790 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:32.882 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:33.292 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:33.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:33.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:33.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:33.472 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:33.542 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:33.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:33.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:33.608 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:32:33.647 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:33.676 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:33.769 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:33.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:33.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:33 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3244114205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:33.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:33 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:33.879 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:33.938 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:33.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:33 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3244114205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:33.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:33 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:33.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:33 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3244114205' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:33.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:33 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:34.036 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:34.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:34.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:34.146 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:34.211 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:34.317 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:34.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:34.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:34.433 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:34.485 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:34.583 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:34.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:34.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: pgmap v1321: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14719/180201 objects misplaced (8.168%); 18 MiB/s, 293 objects/s recovering 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: failed to encode map e448 with expected crc 2023-09-23T15:32:34.603 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[215646]: failed to encode map e448 with expected crc 2023-09-23T15:32:34.700 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: pgmap v1321: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14719/180201 objects misplaced (8.168%); 18 MiB/s, 293 objects/s recovering 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: failed to encode map e448 with expected crc 2023-09-23T15:32:34.725 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:34 smithi027 ceph-mon[195233]: failed to encode map e448 with expected crc 2023-09-23T15:32:34.727 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:34.783 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:34.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:34.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:34.893 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:34.931 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:34.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: pgmap v1321: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.7 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 14719/180201 objects misplaced (8.168%); 18 MiB/s, 293 objects/s recovering 2023-09-23T15:32:34.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: osdmap e448: 8 total, 8 up, 7 in 2023-09-23T15:32:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: failed to encode map e448 with expected crc 2023-09-23T15:32:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:34 smithi017 ceph-mon[222878]: failed to encode map e448 with expected crc 2023-09-23T15:32:35.020 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:35.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:35.026 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:35.054 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:32:35.126 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:35.162 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:35.225 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:35.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:35.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:35.332 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:35.406 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:35.420 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:35.482 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:35.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:35.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:35.595 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:35.703 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:35.923 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:35.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:36.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:36.120 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:36.388 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:36.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:36.398 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:36.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:36.544 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:36.618 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:36 smithi017 ceph-mon[215646]: pgmap v1323: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 11894/170505 objects misplaced (6.976%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:36.653 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:36.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:36.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:36.830 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:36 smithi027 ceph-mon[195233]: pgmap v1323: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 11894/170505 objects misplaced (6.976%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:36.835 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:36.897 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:36 smithi017 ceph-mon[222878]: pgmap v1323: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 715 op/s; 11894/170505 objects misplaced (6.976%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:36.914 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:36.922 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:36.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:36.963 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:37.000 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:37.032 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:37.053 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:37.103 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:37.111 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:37.147 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:37.191 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:37.195 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:37.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:37.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:37.241 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.256 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:32:37.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:37.385 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:37.641 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:37 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/936493167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:37.645 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:37.655 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:37.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:37.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:37.745 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:32:37.758 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:37.887 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:37.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:37 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/936493167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:38.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:37 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/936493167' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:38.353 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:38.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:38.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:38.360 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:38] "GET /metrics HTTP/1.1" 200 36473 "" "Prometheus/2.43.0" 2023-09-23T15:32:38.460 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:38.549 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:38.637 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:38 smithi017 ceph-mon[215646]: pgmap v1324: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:38.637 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:38 smithi017 ceph-mon[222878]: pgmap v1324: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:38.780 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:38.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:38.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:38.892 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:38.973 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:38 smithi027 ceph-mon[195233]: pgmap v1324: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:39.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:39.323 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:39.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:39.424 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:39.469 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:39.547 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:39.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:39.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 6.32456 (expected baseline 6.61168) 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 59 pgs (1.15686 -> 1.15686 * mean) 2023-09-23T15:32:39.582 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:32:39.583 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:32:39.583 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2023-09-23T15:32:39.583 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:32:39.583 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0215 2023-09-23T15:32:39.583 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.2146 2023-09-23T15:32:39.657 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:39.684 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:39.755 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:39.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:39.828 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:32:39.856 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:39.916 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:40.378 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:40.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:40.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:40.485 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:40.512 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:32:40.513 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:40 smithi017 ceph-mon[215646]: pgmap v1325: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:40.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:40 smithi017 ceph-mon[215646]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:40 smithi017 ceph-mon[222878]: pgmap v1325: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:40 smithi017 ceph-mon[222878]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:40.981 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:40.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:41.029 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:41.067 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:32:41.067 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T15:32:41.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:41.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:40 smithi027 ceph-mon[195233]: pgmap v1325: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.5 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 11894/169860 objects misplaced (7.002%); 23 MiB/s, 360 objects/s recovering 2023-09-23T15:32:41.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:40 smithi027 ceph-mon[195233]: from='client.36840 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:41.174 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:41.475 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:41.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:41.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:41.582 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:41.686 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:41.846 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:41.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:41.851 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:41.951 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:41.971 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:41.997 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:42.058 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:32:42.059 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:32:42.090 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:42.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:42.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:42.199 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:42.275 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:42.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:42 smithi017 ceph-mon[215646]: pgmap v1326: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:42.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:42 smithi017 ceph-mon[222878]: pgmap v1326: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:42 smithi027 ceph-mon[195233]: pgmap v1326: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:43.834 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:43.836 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:43.836 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:43.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:43.939 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:43.969 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:43.973 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:44.048 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:44.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:44.157 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:44.198 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:44.255 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:44.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:44.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:44.358 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:32:44.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:44.431 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:44.486 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:44.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:44.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:44.574 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:44.593 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:44.624 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:44.678 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:44.679 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[222878]: pgmap v1327: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:44.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[222878]: failed to encode map e449 with expected crc 2023-09-23T15:32:44.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[222878]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T15:32:44.680 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[222878]: failed to encode map e449 with expected crc 2023-09-23T15:32:44.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:44.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:44.785 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:44.860 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:44.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[215646]: pgmap v1327: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:44.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[215646]: failed to encode map e449 with expected crc 2023-09-23T15:32:44.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[215646]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T15:32:44.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:44 smithi017 ceph-mon[215646]: failed to encode map e449 with expected crc 2023-09-23T15:32:44.968 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:44.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:44.974 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:44 smithi027 ceph-mon[195233]: pgmap v1327: 119 pgs: 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 114 active+clean; 3.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 719 op/s; 9491/158613 objects misplaced (5.984%); 23 MiB/s, 369 objects/s recovering 2023-09-23T15:32:44.974 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:44 smithi027 ceph-mon[195233]: failed to encode map e449 with expected crc 2023-09-23T15:32:44.974 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:44 smithi027 ceph-mon[195233]: osdmap e449: 8 total, 8 up, 7 in 2023-09-23T15:32:44.974 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:44 smithi027 ceph-mon[195233]: failed to encode map e449 with expected crc 2023-09-23T15:32:45.049 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:32:45.074 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:45.109 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:45.940 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:45.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:45.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[215646]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.977 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T15:32:45.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:45.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:45 smithi017 ceph-mon[222878]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T15:32:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:45 smithi027 ceph-mon[195233]: failed to encode map e450 with expected crc 2023-09-23T15:32:46.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:46.358 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:46.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:46.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:46.466 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:46.503 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:46.727 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:46.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:46.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:46.751 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:46 smithi027 ceph-mon[195233]: pgmap v1330: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 6757/149400 objects misplaced (4.523%); 30 MiB/s, 480 objects/s recovering 2023-09-23T15:32:46.834 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:46.881 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:46 smithi017 ceph-mon[222878]: pgmap v1330: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 6757/149400 objects misplaced (4.523%); 30 MiB/s, 480 objects/s recovering 2023-09-23T15:32:46.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:46 smithi017 ceph-mon[215646]: pgmap v1330: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 852 op/s; 6757/149400 objects misplaced (4.523%); 30 MiB/s, 480 objects/s recovering 2023-09-23T15:32:46.885 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:47.007 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:47.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:47.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:47.112 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:47.150 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:47.266 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:47.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:47.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:47.370 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:47.402 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:47.454 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:47.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:47.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:47.558 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:47.594 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:47.597 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:32:47.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T15:32:47.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.741 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1818616902' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:47.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:47.742 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:47 smithi027 ceph-mon[195233]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T15:32:47.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1818616902' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[215646]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: failed to encode map e451 with expected crc 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1818616902' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:32:47.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:47 smithi017 ceph-mon[222878]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T15:32:47.990 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:48.015 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:48.043 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:48.069 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:48.078 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:48.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:48.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:48.094 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:48.118 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:48.141 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:48.164 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.166 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:32:48.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:48.213 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:48.269 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:48.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:48.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:48.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:48.401 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:48] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T15:32:48.405 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:48.787 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:48.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:48.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:48.899 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:48.966 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: pgmap v1332: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 6757/148800 objects misplaced (4.541%); 23 MiB/s, 364 objects/s recovering 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/771220058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:49.064 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[215646]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T15:32:49.065 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: pgmap v1332: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 6757/148800 objects misplaced (4.541%); 23 MiB/s, 364 objects/s recovering 2023-09-23T15:32:49.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/771220058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:49.066 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:48 smithi017 ceph-mon[222878]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T15:32:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: pgmap v1332: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 545 op/s; 6757/148800 objects misplaced (4.541%); 23 MiB/s, 364 objects/s recovering 2023-09-23T15:32:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: failed to encode map e452 with expected crc 2023-09-23T15:32:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/771220058' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:48 smithi027 ceph-mon[195233]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T15:32:49.920 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:49.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:49.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:50.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:50.047 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[215646]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.047 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[215646]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.047 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[215646]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.048 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[222878]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.048 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[222878]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.048 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:49 smithi017 ceph-mon[222878]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:49 smithi027 ceph-mon[195233]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:49 smithi027 ceph-mon[195233]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:49 smithi027 ceph-mon[195233]: failed to encode map e453 with expected crc 2023-09-23T15:32:50.134 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:50.250 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:50.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:50.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:50.361 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:50.404 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:50.476 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:50.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:50.580 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:50.678 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 6.32456 (expected baseline 6.61168) 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 59 pgs (1.15686 -> 1.15686 * mean) 2023-09-23T15:32:50.682 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:32:50.683 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:32:50.683 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:32:50.683 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:32:50.683 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0199 2023-09-23T15:32:50.683 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1991 2023-09-23T15:32:50.691 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:32:50.780 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:50 smithi027 ceph-mon[195233]: pgmap v1335: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 6757/148800 objects misplaced (4.541%) 2023-09-23T15:32:50.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:50 smithi017 ceph-mon[215646]: pgmap v1335: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 6757/148800 objects misplaced (4.541%) 2023-09-23T15:32:50.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:50 smithi017 ceph-mon[222878]: pgmap v1335: 119 pgs: 1 active+remapped+backfill_wait, 1 peering, 3 active+remapped+backfilling, 114 active+clean; 3.0 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 46 op/s; 6757/148800 objects misplaced (4.541%) 2023-09-23T15:32:50.959 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:50.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:50.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:51.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:51.106 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:51.161 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:51.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:51.164 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:51.265 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:32:51.299 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:51.424 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:32:51.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:51.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:32:51.537 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:51.583 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:51.754 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:51.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:51.760 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:32:51.796 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:32:51.797 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T15:32:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:51 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:32:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:51 smithi027 ceph-mon[195233]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:32:51.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:51.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:51 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:51.859 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:51.899 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:52.031 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:52.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:52.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:52.074 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:32:52.136 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[215646]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[222878]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:52.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:32:52.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.196 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:51 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.223 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:52.345 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:32:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:52.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:32:52.435 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:32:52.435 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:32:52.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:167239 10.1 10:8d5de0e9:::benchmark_data_smithi017_604517_object61128:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.419473+0000", 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012247401999999999, 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012288451000000001, 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 167239 2023-09-23T15:32:52.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419473+0000", 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419473+0000", 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419471+0000", 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419480+0000", 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0900000000000005e-06 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419497+0000", 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7297999999999999e-05 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419504+0000", 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0589999999999997e-06 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419550+0000", 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6292000000000003e-05 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419652+0000", 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010170600000000001 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419730+0000", 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8257000000000006e-05 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:167240 10.1 10:833a4382:::benchmark_data_smithi017_604517_object61129:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.419518+0000", 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012202632999999999, 2023-09-23T15:32:52.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012340212, 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 167240 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419518+0000", 2023-09-23T15:32:52.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419518+0000", 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419518+0000", 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419521+0000", 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3459999999999998e-06 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419527+0000", 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1240000000000002e-06 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419531+0000", 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8979999999999998e-06 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.422284+0000", 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027525549999999998 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.422418+0000", 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013472399999999999 2023-09-23T15:32:52.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.422548+0000", 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129708 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:167243 10.1 10:83eeb99a:::benchmark_data_smithi017_604517_object61132:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.419559+0000", 2023-09-23T15:32:52.446 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.012161831999999999, 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012376949, 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 167243 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.447 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419559+0000", 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419559+0000", 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419558+0000", 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:32:52.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419564+0000", 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7960000000000001e-06 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419576+0000", 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2519e-05 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.419583+0000", 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.844e-06 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425592+0000", 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060081910000000004 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425752+0000", 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016092600000000001 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425900+0000", 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014753600000000001 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "MOSDPGPush(10.9 453/443 [PushOp(10:9c5415be:::benchmark_data_smithi017_604517_object104602:head, version: 433'6604, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(10:9c5415be:::benchmark_data_smithi017_604517_object104602:head@433'6604, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.420986+0000", 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.010734558, 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011053929000000001, 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.420986+0000", 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.420986+0000", 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.420986+0000", 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421033+0000", 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6609e-05 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421051+0000", 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8609000000000001e-05 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.454 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421065+0000", 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3275000000000001e-05 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:167246 10.1 10:8a448cb0:::benchmark_data_smithi017_604517_object61135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.421866+0000", 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0098551390000000006, 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010214253, 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.455 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "waiting for sub ops", 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 167246 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421866+0000", 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421866+0000", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421864+0000", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421875+0000", 2023-09-23T15:32:52.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0828e-05 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421893+0000", 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8334000000000001e-05 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.421902+0000", 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6470000000000004e-06 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428885+0000", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069829460000000003 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429047+0000", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000162375 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429228+0000", 2023-09-23T15:32:52.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018091200000000001 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46520.0:167239 10.1 e453/413)", 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.425699+0000", 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0060222050000000001, 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064567169999999998, 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:32:52.460 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425699+0000", 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425699+0000", 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425697+0000", 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425703+0000", 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7640000000000002e-06 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425715+0000", 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.224e-05 2023-09-23T15:32:52.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425722+0000", 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6950000000000003e-06 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46520.0:167240 10.1 e453/413)", 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.425858+0000", 2023-09-23T15:32:52.463 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0058627999999999996, 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063488720000000002, 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425858+0000", 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425858+0000", 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425858+0000", 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425861+0000", 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7340000000000002e-06 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425871+0000", 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5300000000000002e-06 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.425876+0000", 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1769999999999997e-06 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46520.0:167243 10.1 e453/413)", 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.428389+0000", 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0033318409999999999, 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038624710000000001, 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428389+0000", 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428389+0000", 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428388+0000", 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428394+0000", 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4840000000000001e-06 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428407+0000", 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.324e-05 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.428413+0000", 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1229999999999999e-06 2023-09-23T15:32:52.469 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:167245 10.3 e453/449 10:c9778109:::benchmark_data_smithi017_604517_object61134:head v 453'10598, mlcod=453'10594)", 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.429901+0000", 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 0.0018203049999999999, 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024042360000000001, 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429901+0000", 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429901+0000", 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429900+0000", 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429906+0000", 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5759999999999996e-06 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429916+0000", 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0379e-05 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429922+0000", 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0249999999999999e-06 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429968+0000", 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5626e-05 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.429992+0000", 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4241000000000001e-05 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46520.0:167239 10.1 e453/413)", 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.431924+0000", 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9997973, 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00045382700000000002, 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.474 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431924+0000", 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431924+0000", 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.475 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431922+0000", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431932+0000", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6260000000000007e-06 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431946+0000", 2023-09-23T15:32:52.476 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3749e-05 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431953+0000", 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4000000000000003e-06 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.432362+0000", 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00040850100000000002 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.477 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop_reply(client.46520.0:167240 10.1 e453/413)", 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.431970+0000", 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9997506, 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00046139100000000001, 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "queued for pg", 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431970+0000", 2023-09-23T15:32:52.478 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431970+0000", 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431970+0000", 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.479 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431973+0000", 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9529999999999999e-06 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431980+0000", 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5200000000000003e-06 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.480 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431984+0000", 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0899999999999998e-06 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:167238 10.6 e453/451 10:6f0a09c6:::benchmark_data_smithi017_604517_object61127:head v 453'10438, mlcod=453'10436)", 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:52.431988+0000", 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 4294967295.9997325, 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00048632100000000001, 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:32:52.481 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "started", 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431988+0000", 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431988+0000", 2023-09-23T15:32:52.482 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431988+0000", 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967296 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431993+0000", 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.118e-06 2023-09-23T15:32:52.483 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.431999+0000", 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1999999999999999e-06 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.432003+0000", 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8619999999999996e-06 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.484 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.432053+0000", 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9483e-05 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:52.432076+0000", 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.3229000000000001e-05 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:32:52.485 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:32:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: ], 2023-09-23T15:32:52.486 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 12 2023-09-23T15:32:52.486 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:32:52.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:52.552 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:52.635 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:52.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:52.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:52.683 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:52.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:32:52.820 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:52 smithi027 ceph-mon[195233]: pgmap v1336: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3312/141327 objects misplaced (2.344%); 33 MiB/s, 532 objects/s recovering 2023-09-23T15:32:52.820 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:52 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:32:52.820 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:52 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.820 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:52 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:52.825 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:52.857 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:53.037 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:32:53.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:53.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:32:53.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:53.181 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:53.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[215646]: pgmap v1336: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3312/141327 objects misplaced (2.344%); 33 MiB/s, 532 objects/s recovering 2023-09-23T15:32:53.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:32:53.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:53.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:53.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[222878]: pgmap v1336: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3312/141327 objects misplaced (2.344%); 33 MiB/s, 532 objects/s recovering 2023-09-23T15:32:53.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:32:53.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:53.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:52 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:32:53.288 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:32:53.508 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:53.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:53.604 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:32:53.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:53.691 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:53.911 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:53.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:53.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:53.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:53 smithi017 ceph-mon[215646]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 3312/141327 objects misplaced (2.344%); 32 MiB/s, 508 objects/s recovering 2023-09-23T15:32:53.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:53 smithi017 ceph-mon[222878]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 3312/141327 objects misplaced (2.344%); 32 MiB/s, 508 objects/s recovering 2023-09-23T15:32:54.025 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:53 smithi027 ceph-mon[195233]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 396 op/s; 3312/141327 objects misplaced (2.344%); 32 MiB/s, 508 objects/s recovering 2023-09-23T15:32:54.109 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:54.320 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:32:54.329 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:54.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:54.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:54.443 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:54.510 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:54.666 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:54.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:54.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:54.780 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:54.852 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:56.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:56 smithi017 ceph-mon[222878]: pgmap v1338: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1753/131211 objects misplaced (1.336%); 36 MiB/s, 570 objects/s recovering 2023-09-23T15:32:56.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:56 smithi017 ceph-mon[215646]: pgmap v1338: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1753/131211 objects misplaced (1.336%); 36 MiB/s, 570 objects/s recovering 2023-09-23T15:32:56.276 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:56.284 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:56.351 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:56 smithi027 ceph-mon[195233]: pgmap v1338: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 732 op/s; 1753/131211 objects misplaced (1.336%); 36 MiB/s, 570 objects/s recovering 2023-09-23T15:32:56.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:32:56.437 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:56.570 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:32:56.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:56.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:32:56.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:32:56.725 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:56.861 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:32:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:56.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:32:56.901 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:32:56.901 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:32:56.901 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:32:56.901 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:32:56.901 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:32:56.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:57.051 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:57.092 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:57 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:32:57.246 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:57 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:32:57.247 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:57 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:32:57.442 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:57.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:57.554 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:32:57.695 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:57.701 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:32:57.753 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:32:57.841 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:32:57.855 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:32:57.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:57.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:32:57.912 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:32:57.950 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:32:57.973 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:58.023 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:32:58.071 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:32:58.085 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:58.109 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:32:58.109 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:32:58.109 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 6.32456 (expected baseline 6.61168) 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 59 pgs (1.15686 -> 1.15686 * mean) 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0187 2023-09-23T15:32:58.110 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1869 2023-09-23T15:32:58.111 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:32:58.115 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.132 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.133 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.133 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:32:58.141 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:32:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:32:58] "GET /metrics HTTP/1.1" 200 36458 "" "Prometheus/2.43.0" 2023-09-23T15:32:58.318 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:32:58.533 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:32:58.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:58.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:32:58.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:32:58.642 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: pgmap v1339: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1753/130557 objects misplaced (1.343%); 30 MiB/s, 484 objects/s recovering 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2602897331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1810197417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: pgmap v1339: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1753/130557 objects misplaced (1.343%); 30 MiB/s, 484 objects/s recovering 2023-09-23T15:32:58.676 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2602897331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:58.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:32:58.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1810197417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.677 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:58.726 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:58.903 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:32:58.908 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: pgmap v1339: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 1753/130557 objects misplaced (1.343%); 30 MiB/s, 484 objects/s recovering 2023-09-23T15:32:58.908 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2602897331' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:32:58.908 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:32:58.909 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1810197417' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.909 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:32:58.909 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:32:58.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:32:58.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:32:59.010 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:32:59.040 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:32:59.069 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:32:59.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:59.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:59.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:32:59.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.713 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:59.714 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:32:59 smithi027 ceph-mon[195233]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T15:32:59.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:59.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[222878]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T15:32:59.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: from='client.46778 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:32:59.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:32:59.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:32:59.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: failed to encode map e454 with expected crc 2023-09-23T15:32:59.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:32:59.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:32:59 smithi017 ceph-mon[215646]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T15:33:00.209 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:33:00.209 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T15:33:00.375 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:00.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:00.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:00.487 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:00.536 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:00.819 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:00.827 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: pgmap v1341: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1753/130557 objects misplaced (1.343%); 29 MiB/s, 456 objects/s recovering 2023-09-23T15:33:00.827 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.827 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T15:33:00.827 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.827 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.828 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.828 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.828 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:00 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:33:00.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:00.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:00.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: pgmap v1341: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1753/130557 objects misplaced (1.343%); 29 MiB/s, 456 objects/s recovering 2023-09-23T15:33:00.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T15:33:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:33:00.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: pgmap v1341: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfilling, 115 active+clean; 2.7 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 607 op/s; 1753/130557 objects misplaced (1.343%); 29 MiB/s, 456 objects/s recovering 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: failed to encode map e455 with expected crc 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:00.947 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:00 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:33:01.021 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:01.217 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:01.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:01.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:01.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:01.324 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:01.387 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:01.505 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:01.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:01.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:01.538 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:01.539 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:01.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:01.672 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:01.733 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:01.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:01.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:01.843 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:01.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:01 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:33:01.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:01 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:33:01.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:01 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:33:01.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:01 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:33:01.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:01 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:33:01.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:01 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:33:01.935 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:02.163 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:33:02.666 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:33:02.682 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:33:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:02 smithi017 ceph-mon[215646]: pgmap v1343: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 526/121590 objects misplaced (0.433%); 20 MiB/s, 313 objects/s recovering 2023-09-23T15:33:02.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:02 smithi017 ceph-mon[215646]: 1.1b deep-scrub starts 2023-09-23T15:33:02.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:02 smithi017 ceph-mon[222878]: pgmap v1343: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 526/121590 objects misplaced (0.433%); 20 MiB/s, 313 objects/s recovering 2023-09-23T15:33:02.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:02 smithi017 ceph-mon[222878]: 1.1b deep-scrub starts 2023-09-23T15:33:03.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:02 smithi027 ceph-mon[195233]: pgmap v1343: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 526/121590 objects misplaced (0.433%); 20 MiB/s, 313 objects/s recovering 2023-09-23T15:33:03.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:02 smithi027 ceph-mon[195233]: 1.1b deep-scrub starts 2023-09-23T15:33:03.141 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:03.388 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:03.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:03.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:03.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:03.598 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:03.684 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:03.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:33:03.685 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.686 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.688 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:03 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:33:03.688 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.688 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:03 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.728 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:03.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:03.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:03.828 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:03.851 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:03.887 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:03.937 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:33:03.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:03 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:03.950 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:03.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:03.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:04.055 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:04.178 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:04.547 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:04.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:04.652 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:04.700 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:04.794 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:04.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:04.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 6.32456 (expected baseline 6.61168) 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 59 pgs (1.15686 -> 1.15686 * mean) 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:33:04.858 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:33:04.859 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:04.859 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0178 2023-09-23T15:33:04.859 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1782 2023-09-23T15:33:04.871 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: pgmap v1344: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 526/121590 objects misplaced (0.433%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: 1.17 deep-scrub starts 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:33:04.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: pgmap v1344: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 526/121590 objects misplaced (0.433%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: 1.17 deep-scrub starts 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:33:04.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:04.900 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:04.937 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: pgmap v1344: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 2.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 400 op/s; 526/121590 objects misplaced (0.433%); 8.9 MiB/s, 142 objects/s recovering 2023-09-23T15:33:04.937 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: 1.17 deep-scrub starts 2023-09-23T15:33:04.938 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:33:04.938 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:33:04.938 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:33:04.938 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:04.950 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:05.046 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:05.053 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:05.154 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:05.214 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:05.282 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:05.282 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:06.120 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:33:06.186 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:06.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:06.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:06.290 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:06.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:06.366 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: pgmap v1345: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 13 MiB/s, 205 objects/s recovering 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: 1.8 scrub starts 2023-09-23T15:33:06.367 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:06 smithi027 ceph-mon[195233]: 1.8 scrub ok 2023-09-23T15:33:06.375 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: pgmap v1345: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 13 MiB/s, 205 objects/s recovering 2023-09-23T15:33:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T15:33:06.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: 1.8 scrub starts 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[215646]: 1.8 scrub ok 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: from='client.36867 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: pgmap v1345: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 793 op/s; 13 MiB/s, 205 objects/s recovering 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: osdmap e456: 8 total, 8 up, 7 in 2023-09-23T15:33:06.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: failed to encode map e456 with expected crc 2023-09-23T15:33:06.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: 1.8 scrub starts 2023-09-23T15:33:06.445 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:06 smithi017 ceph-mon[222878]: 1.8 scrub ok 2023-09-23T15:33:06.462 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:06.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:06.575 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:06.664 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:06.671 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:07.155 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:07.162 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:07.163 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:07.178 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: failed to encode map e456 with expected crc 2023-09-23T15:33:07.178 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: 1.1 scrub starts 2023-09-23T15:33:07.178 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: 1.1 scrub ok 2023-09-23T15:33:07.178 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: 1.1b scrub starts 2023-09-23T15:33:07.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: 1.1b scrub ok 2023-09-23T15:33:07.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.179 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:07 smithi027 ceph-mon[195233]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T15:33:07.263 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:07.315 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:07.392 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:07.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:07.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:07.442 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: failed to encode map e456 with expected crc 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: 1.1 scrub starts 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: 1.1 scrub ok 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: 1.1b scrub starts 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: 1.1b scrub ok 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.443 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[215646]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: failed to encode map e456 with expected crc 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: 1.1 scrub starts 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: 1.1 scrub ok 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: 1.1b scrub starts 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: 1.1b scrub ok 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: failed to encode map e457 with expected crc 2023-09-23T15:33:07.444 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:07 smithi017 ceph-mon[222878]: osdmap e457: 8 total, 8 up, 7 in 2023-09-23T15:33:07.496 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:07.599 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:07.810 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:07.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:07.926 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:08.022 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:08.034 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:08.071 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:08.119 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:08.158 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:08.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:08.168 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:08] "GET /metrics HTTP/1.1" 200 36456 "" "Prometheus/2.43.0" 2023-09-23T15:33:08.172 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:08.218 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:08.263 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:08.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:08.322 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:08.365 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:08.370 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:08.389 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.390 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:33:08.404 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:08 smithi027 ceph-mon[195233]: 1.17 scrub starts 2023-09-23T15:33:08.405 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:08 smithi027 ceph-mon[195233]: 1.17 scrub ok 2023-09-23T15:33:08.405 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:08 smithi027 ceph-mon[195233]: pgmap v1348: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 14 MiB/s, 222 objects/s recovering 2023-09-23T15:33:08.405 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3158298014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:08.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[222878]: 1.17 scrub starts 2023-09-23T15:33:08.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[222878]: 1.17 scrub ok 2023-09-23T15:33:08.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[222878]: pgmap v1348: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 14 MiB/s, 222 objects/s recovering 2023-09-23T15:33:08.418 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3158298014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:08.421 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[215646]: 1.17 scrub starts 2023-09-23T15:33:08.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[215646]: 1.17 scrub ok 2023-09-23T15:33:08.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[215646]: pgmap v1348: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 14 MiB/s, 222 objects/s recovering 2023-09-23T15:33:08.422 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3158298014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:08.453 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:08.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:08.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:08.556 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:08.632 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:08.721 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:08.721 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:33:08.721 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:33:08.721 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:33:08.721 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:08.728 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:08.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:08.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:08.833 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:08.862 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:08.957 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:08.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:09.061 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:09.108 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:09.197 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:09.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:09.246 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:33:09.303 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:09.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3961409673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:09 smithi027 ceph-mon[195233]: 1.18 scrub starts 2023-09-23T15:33:09.359 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:09 smithi027 ceph-mon[195233]: 1.18 scrub ok 2023-09-23T15:33:09.363 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:09.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3961409673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[222878]: 1.18 scrub starts 2023-09-23T15:33:09.523 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[222878]: 1.18 scrub ok 2023-09-23T15:33:09.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3961409673' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:09.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[215646]: 1.18 scrub starts 2023-09-23T15:33:09.526 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:09 smithi017 ceph-mon[215646]: 1.18 scrub ok 2023-09-23T15:33:09.676 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:09.678 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:33:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:09.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:09.782 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:09.879 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:10.157 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:10.175 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:10.213 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:10.276 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:10.327 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:10.442 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: pgmap v1349: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-09-23T15:33:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.450 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[215646]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: pgmap v1349: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.451 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:10.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T15:33:10.452 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:10 smithi017 ceph-mon[222878]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:10.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:10.501 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: pgmap v1349: 119 pgs: 1 active+clean+remapped, 1 active+clean+scrubbing+deep, 1 active+clean+scrubbing, 116 active+clean; 2.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 448 op/s; 3.9 MiB/s, 62 objects/s recovering 2023-09-23T15:33:10.501 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:10.501 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:10.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: osdmap e458: 8 total, 8 up, 7 in 2023-09-23T15:33:10.502 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:10 smithi027 ceph-mon[195233]: failed to encode map e458 with expected crc 2023-09-23T15:33:10.553 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:10.608 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:11.000 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:11.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:11.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:11.188 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:11.293 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:11.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:11.301 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:11.402 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:11.452 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:11.712 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:11.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:11.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:11.821 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:11.846 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:33:11.877 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:12.061 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:12.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:12.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:12.157 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:33:12.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:12.220 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:12.306 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:12.307 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:12 smithi027 ceph-mon[195233]: pgmap v1351: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T15:33:12.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:12.308 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148874 8.4 8:22ed0ae9:::benchmark_data_smithi017_485823_object58737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.49085308099995, 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99896163400000004, 2023-09-23T15:33:12.322 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148874 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:33:12.323 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827872+0000", 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827871+0000", 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.324 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827874+0000", 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.311e-06 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827879+0000", 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4600000000000002e-06 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.325 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.827883+0000", 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3730000000000001e-06 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831017+0000", 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031349569999999998 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.831029+0000", 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1875000000000001e-05 2023-09-23T15:33:12.326 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782403+0000", 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95137329500000001 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782500+0000", 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7763999999999994e-05 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.327 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.782578+0000", 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7972999999999994e-05 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794469+0000", 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011890258000000001 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826626+0000", 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.032157673999999997 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.328 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826744+0000", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117747 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826793+0000", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9308000000000003e-05 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826833+0000", 2023-09-23T15:33:12.329 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0000000000000003e-05 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148906 8.4 8:29f45dea:::benchmark_data_smithi017_485823_object58769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.45017211799995, 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.95839901500000002, 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.330 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148906 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.331 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868553+0000", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868552+0000", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868555+0000", 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.266e-06 2023-09-23T15:33:12.332 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868558+0000", 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7699999999999999e-06 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868561+0000", 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.333 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868612+0000", 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0853000000000003e-05 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.868618+0000", 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4529999999999999e-06 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785580+0000", 2023-09-23T15:33:12.334 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91696206300000005 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785720+0000", 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000139677 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.785862+0000", 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014233400000000001 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.335 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799990+0000", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014128043999999999 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826695+0000", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.026704894 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826890+0000", 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019431599999999999 2023-09-23T15:33:12.336 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826928+0000", 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8148000000000001e-05 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.826952+0000", 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4049e-05 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.337 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148912 8.4 8:2ad9ecb5:::benchmark_data_smithi017_485823_object58775:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.44621855399998, 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.954622631, 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.338 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148912 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872506+0000", 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.339 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872505+0000", 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872512+0000", 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8390000000000001e-06 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.340 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872524+0000", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2469e-05 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872530+0000", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.139e-06 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872582+0000", 2023-09-23T15:33:12.341 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1983e-05 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.872592+0000", 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3710000000000006e-06 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788471+0000", 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91587959200000002 2023-09-23T15:33:12.342 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788569+0000", 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8047000000000005e-05 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.788661+0000", 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1267000000000006e-05 2023-09-23T15:33:12.343 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.799997+0000", 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011336841 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827002+0000", 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.027004980000000001 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.344 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827082+0000", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9484e-05 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827107+0000", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5216999999999999e-05 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827129+0000", 2023-09-23T15:33:12.345 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1801e-05 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148934 8.4 8:2bdf4584:::benchmark_data_smithi017_485823_object58797:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.41451028200004, 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92312028400000001, 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.346 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148934 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.347 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904215+0000", 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904213+0000", 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904224+0000", 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0525e-05 2023-09-23T15:33:12.348 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904236+0000", 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2649999999999999e-05 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904243+0000", 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.241e-06 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.349 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904339+0000", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6711000000000003e-05 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.904349+0000", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7340000000000003e-06 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791162+0000", 2023-09-23T15:33:12.350 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88681287099999995 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791285+0000", 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012337700000000001 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.791419+0000", 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134029 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.351 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804693+0000", 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013273785 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827031+0000", 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022337704999999999 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827243+0000", 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021219399999999999 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.352 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827294+0000", 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.113e-05 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827335+0000", 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0677999999999999e-05 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.353 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148942 8.4 8:2f2d7443:::benchmark_data_smithi017_485823_object58805:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.40822876200002, 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91690406000000002, 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148942 2023-09-23T15:33:12.354 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910496+0000", 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.355 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910495+0000", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910500+0000", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.57e-06 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910514+0000", 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.365e-05 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.356 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910522+0000", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7740000000000004e-06 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910590+0000", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8472000000000004e-05 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910602+0000", 2023-09-23T15:33:12.357 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1343e-05 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794512+0000", 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88391032800000002 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794644+0000", 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013237700000000001 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.358 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.794779+0000", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134709 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.804705+0000", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099256569999999992 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827184+0000", 2023-09-23T15:33:12.359 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.022478946999999999 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827355+0000", 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000171628 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827381+0000", 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5789999999999999e-05 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.360 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827400+0000", 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8995e-05 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148943 8.4 8:26834e88:::benchmark_data_smithi017_485823_object58806:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.408185344, 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.916991004, 2023-09-23T15:33:12.361 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148943 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.362 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910539+0000", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910542+0000", 2023-09-23T15:33:12.363 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7889999999999999e-06 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910554+0000", 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2216e-05 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910560+0000", 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1990000000000003e-06 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.364 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910623+0000", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3254000000000001e-05 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.910628+0000", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797471+0000", 2023-09-23T15:33:12.365 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88684291900000001 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797586+0000", 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114815 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.797686+0000", 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000100275 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.366 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810164+0000", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012477492 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827205+0000", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017040726999999999 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827444+0000", 2023-09-23T15:33:12.367 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00023902400000000001 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827492+0000", 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8707e-05 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827530+0000", 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8102000000000002e-05 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.368 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148947 8.4 8:25191517:::benchmark_data_smithi017_485823_object58810:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.40384870100002, 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91295790200000004, 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.369 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148947 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.370 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914876+0000", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914879+0000", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.683e-06 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914884+0000", 2023-09-23T15:33:12.371 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8189999999999997e-06 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914888+0000", 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.121e-06 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914950+0000", 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1359999999999995e-05 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.372 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.914961+0000", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.076e-05 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800046+0000", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88508585500000003 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800194+0000", 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014714700000000001 2023-09-23T15:33:12.373 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.800323+0000", 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000129197 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.810173+0000", 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098501830000000002 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.374 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827554+0000", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017381285 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827782+0000", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00022772600000000001 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827812+0000", 2023-09-23T15:33:12.375 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0433000000000002e-05 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827834+0000", 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1628e-05 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.45809.0:148958 8.4 8:2e39e1d6:::benchmark_data_smithi017_485823_object58821:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e327)", 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:33:12.376 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.39024004500004, 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89949832200000002, 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.45809", 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/513957407", 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 148958 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:33:12.377 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928485+0000", 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928483+0000", 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:12.378 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928493+0000", 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0638999999999999e-05 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928514+0000", 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0859000000000001e-05 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.379 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928523+0000", 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5860000000000005e-06 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928590+0000", 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7299999999999996e-05 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:26.928600+0000", 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7189999999999996e-06 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.380 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802452+0000", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.87385171800000006 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802535+0000", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3803999999999999e-05 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,4", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.802599+0000", 2023-09-23T15:33:12.381 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3928000000000002e-05 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.815177+0000", 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012578119 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827593+0000", 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012415981 2023-09-23T15:33:12.382 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827890+0000", 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00029696900000000001 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827945+0000", 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4586999999999998e-05 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.383 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:24:27.827983+0000", 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8109999999999999e-05 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83112 9.a 9:5853ad9e:::benchmark_data_smithi017_543597_object83111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 376.42669919299999, 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0903713559999999, 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.384 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83112 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.385 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892024+0000", 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.386 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892085+0000", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1613999999999997e-05 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892103+0000", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7893e-05 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892110+0000", 2023-09-23T15:33:12.387 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892224+0000", 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114508 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892371+0000", 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014686500000000001 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:33:12.388 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892552+0000", 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018046499999999999 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898624+0000", 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060722090000000003 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912313+0000", 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013688707 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.389 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912322+0000", 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4839999999999993e-06 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.970323+0000", 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0580004119999999 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:33:12.390 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971026+0000", 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070397199999999997 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982334+0000", 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011307914000000001 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982397+0000", 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2570999999999994e-05 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.391 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83117 9.a 9:51c7f339:::benchmark_data_smithi017_543597_object83116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 376.42320042099999, 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0870892780000001, 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.392 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83117 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.393 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895523+0000", 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895570+0000", 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6748e-05 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.394 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895583+0000", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3390000000000001e-05 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895590+0000", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895803+0000", 2023-09-23T15:33:12.395 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021333300000000001 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895915+0000", 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111697 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896025+0000", 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109511 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.396 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.899384+0000", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033596300000000002 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912365+0000", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012981029999999999 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912370+0000", 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5549999999999996e-06 2023-09-23T15:33:12.397 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971071+0000", 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.058701627 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.973148+0000", 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020767469999999999 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.398 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982568+0000", 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094202609999999992 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982614+0000", 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5259000000000002e-05 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.399 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83120 9.7 9:e8f3216b:::benchmark_data_smithi017_543597_object83119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 376.42062503099999, 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.062896818, 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83120 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.400 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.401 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898098+0000", 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898169+0000", 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0426000000000006e-05 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898180+0000", 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1114999999999999e-05 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.402 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898186+0000", 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0159999999999997e-06 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898238+0000", 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1777000000000002e-05 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.403 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898377+0000", 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138825 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898557+0000", 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000180058 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.901069+0000", 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025124309999999999 2023-09-23T15:33:12.404 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912790+0000", 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011721352000000001 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912795+0000", 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5430000000000004e-06 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.405 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948511+0000", 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0357156709999999 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.949214+0000", 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070288299999999996 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960946+0000", 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011732340000000001 2023-09-23T15:33:12.406 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960997+0000", 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0702999999999999e-05 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88042 9.a 9:5af6f1a8:::benchmark_data_smithi017_543597_object4251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 331.92519299700001, 2023-09-23T15:33:12.407 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9167718300000001, 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88042 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:33:12.408 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393530+0000", 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.409 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393536+0000", 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7370000000000003e-06 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393552+0000", 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5713999999999999e-05 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.410 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393560+0000", 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0800000000000006e-06 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393617+0000", 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6884e-05 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393770+0000", 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015349600000000001 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.411 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393905+0000", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013507699999999999 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400071+0000", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061654379999999996 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400083+0000", 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2109e-05 2023-09-23T15:33:12.412 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389036+0000", 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98895255699999995 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389049+0000", 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3633e-05 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.413 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303003+0000", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.913953927 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303184+0000", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000181175 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310231+0000", 2023-09-23T15:33:12.414 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007047045 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310304+0000", 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2308e-05 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88055 9.a 9:5bed7ee7:::benchmark_data_smithi017_543597_object4264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 331.87872389799998, 2023-09-23T15:33:12.415 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8698463969999999, 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88055 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.416 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440000+0000", 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.417 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440006+0000", 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5049999999999998e-06 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440018+0000", 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2359e-05 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.418 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440025+0000", 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2849999999999997e-06 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440084+0000", 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.855e-05 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440092+0000", 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.585e-06 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.419 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389071+0000", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94897856800000002 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389079+0000", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6590000000000007e-06 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303210+0000", 2023-09-23T15:33:12.420 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91413175199999996 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303344+0000", 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013339999999999999 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303490+0000", 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014616299999999999 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.421 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.307392+0000", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003901655 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309301+0000", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019098609999999999 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309677+0000", 2023-09-23T15:33:12.422 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037558600000000002 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309714+0000", 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6791000000000001e-05 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309847+0000", 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133454 2023-09-23T15:33:12.423 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88057 9.7 9:eebbf230:::benchmark_data_smithi017_543597_object4266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 331.87478609099998, 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.85708753, 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.424 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88057 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.425 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443938+0000", 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443943+0000", 2023-09-23T15:33:12.426 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5079999999999999e-06 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443955+0000", 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2754e-05 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443965+0000", 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8339999999999994e-06 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.427 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444010+0000", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5330000000000001e-05 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444019+0000", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2320000000000001e-06 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387832+0000", 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.943813609 2023-09-23T15:33:12.428 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387845+0000", 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2853e-05 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297101+0000", 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90925576399999997 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.429 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297239+0000", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013821800000000001 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297322+0000", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3220999999999995e-05 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.299762+0000", 2023-09-23T15:33:12.430 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024393729999999999 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300155+0000", 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039295999999999998 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300804+0000", 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064965300000000004 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.431 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300868+0000", 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3898000000000004e-05 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.301026+0000", 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015802500000000001 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.432 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137065 10.6 10:6a8d0ffd:::benchmark_data_smithi017_604517_object30954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 65.331818429999998, 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18030616599999999, 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137065 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.433 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.434 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986904+0000", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986913+0000", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9069999999999996e-06 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986930+0000", 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6911000000000001e-05 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.435 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986937+0000", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.092e-06 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987130+0000", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019227400000000001 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987251+0000", 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121505 2023-09-23T15:33:12.436 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987367+0000", 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011581799999999999 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995041+0000", 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076740460000000003 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.437 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002478+0000", 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074370130000000001 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167121+0000", 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.164642593 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.438 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167181+0000", 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0134999999999999e-05 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167213+0000", 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1896000000000001e-05 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137074 10.6 10:6f342ab4:::benchmark_data_smithi017_604517_object30963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:33:12.439 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 65.323507860000007, 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.172095518, 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137074 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:33:12.440 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995216+0000", 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.441 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995222+0000", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.874e-06 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995233+0000", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1024e-05 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995237+0000", 2023-09-23T15:33:12.442 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.741e-06 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995272+0000", 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4824999999999999e-05 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995354+0000", 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1582999999999995e-05 2023-09-23T15:33:12.443 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995426+0000", 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2383000000000001e-05 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002484+0000", 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070572830000000001 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.444 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.006480+0000", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039969519999999998 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167255+0000", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16077435000000001 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167287+0000", 2023-09-23T15:33:12.445 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2132000000000003e-05 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167312+0000", 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5531e-05 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.446 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137075 10.6 10:6cfb72a7:::benchmark_data_smithi017_604517_object30964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 65.321924886000005, 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17058852699999999, 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137075 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.447 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996799+0000", 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:12.448 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996805+0000", 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4989999999999999e-06 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996826+0000", 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1211000000000001e-05 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.449 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996833+0000", 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0299999999999996e-06 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.998844+0000", 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020100719999999999 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.998983+0000", 2023-09-23T15:33:12.450 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013919599999999999 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.999117+0000", 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013448099999999999 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.008974+0000", 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098565070000000005 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.451 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009751+0000", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077736600000000004 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167338+0000", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.157587325 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167366+0000", 2023-09-23T15:33:12.452 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7185000000000001e-05 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167388+0000", 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2855999999999999e-05 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137078 10.6 10:63c718f8:::benchmark_data_smithi017_604517_object30967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:33:12.453 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 65.321706270000007, 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17522478399999999, 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137078 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.454 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997018+0000", 2023-09-23T15:33:12.455 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997025+0000", 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.182e-06 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997039+0000", 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4168000000000001e-05 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.456 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997045+0000", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8579999999999996e-06 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002510+0000", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054650250000000001 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002630+0000", 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120388 2023-09-23T15:33:12.457 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002738+0000", 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107558 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009016+0000", 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062775180000000002 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.458 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009994+0000", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097875099999999993 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172153+0000", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16215827899999999 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172203+0000", 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0009000000000002e-05 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.459 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172243+0000", 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0757999999999998e-05 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:140577 10.9 10:922c9838:::benchmark_data_smithi017_604517_object34466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 60.675979632000001, 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21956315500000001, 2023-09-23T15:33:12.460 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140577 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.461 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642743+0000", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642751+0000", 2023-09-23T15:33:12.462 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1459999999999999e-06 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642767+0000", 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6663e-05 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642775+0000", 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2960000000000001e-06 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.463 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642824+0000", 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9147999999999998e-05 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642955+0000", 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131092 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2023-09-23T15:33:12.464 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.643074+0000", 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119286 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.646978+0000", 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039041100000000001 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.646997+0000", 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9102000000000001e-05 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.465 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862216+0000", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.215218992 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862267+0000", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0872999999999999e-05 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862308+0000", 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1148e-05 2023-09-23T15:33:12.466 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:140604 10.6 10:640c290b:::benchmark_data_smithi017_604517_object34493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 60.592292804000003, 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22775473700000001, 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:12.467 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140604 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:12.468 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726430+0000", 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726439+0000", 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1540000000000008e-06 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.469 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726461+0000", 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1129e-05 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726469+0000", 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1200000000000002e-06 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:12.470 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726524+0000", 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4840999999999999e-05 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726670+0000", 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146615 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726781+0000", 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011131799999999999 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.471 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.729958+0000", 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031767980000000002 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.730207+0000", 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024893199999999999 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:12.472 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954090+0000", 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22388297099999999 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954147+0000", 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7240000000000001e-05 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954187+0000", 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9335000000000002e-05 2023-09-23T15:33:12.473 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.474 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.474 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.474 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:12.474 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:12.474 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:12.474 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:33:12.474 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:33:12.474 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T15:33:12.474 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:33:12.475 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:33:12.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T15:33:12.475 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:12.520 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:12.613 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:12.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:12.622 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:12.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:12 smithi017 ceph-mon[215646]: pgmap v1351: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T15:33:12.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:12 smithi017 ceph-mon[222878]: pgmap v1351: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 638 op/s 2023-09-23T15:33:12.720 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:12.749 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:12.823 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:13.118 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:13.126 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:13.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:13.193 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:33:13.227 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:13.272 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:13.315 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:13.336 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:13.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:13.342 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:13.443 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:13.541 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:13.878 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:13.879 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:13.883 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:13.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:13.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:13.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:14.089 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:14.818 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:14.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:14.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:14 smithi027 ceph-mon[195233]: pgmap v1352: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 488 op/s 2023-09-23T15:33:14.851 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:14 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:14.924 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:14.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:14 smithi017 ceph-mon[215646]: pgmap v1352: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 488 op/s 2023-09-23T15:33:14.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:14 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:14.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:14 smithi017 ceph-mon[222878]: pgmap v1352: 119 pgs: 119 active+clean; 2.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 488 op/s 2023-09-23T15:33:14.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:14 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:14.956 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:15.119 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:15.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:15.199 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:15.199 INFO:teuthology.orchestra.run.smithi027.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T15:33:15.199 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:15.225 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:15.313 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:15.474 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:15.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:15.485 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:15.585 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:15.614 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:15.659 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:15.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:15.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:15.760 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:15.763 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:15.802 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:16.082 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:16.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:16.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:16.194 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:16.244 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:16.285 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:16.300 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:16.349 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:16.387 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:16.422 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:16.447 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:16.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:16.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:16.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:16.504 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:16.540 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:16.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.549 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.550 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.550 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.550 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:33:16.564 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:16.650 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:16.888 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:16 smithi017 ceph-mon[215646]: pgmap v1353: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T15:33:16.888 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:16 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2517590424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:16.889 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:16 smithi017 ceph-mon[222878]: pgmap v1353: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T15:33:16.889 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:16 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2517590424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:16 smithi027 ceph-mon[195233]: pgmap v1353: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 853 op/s 2023-09-23T15:33:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:16 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2517590424' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:17.383 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:17.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:17.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:17.494 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:17.560 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:17.633 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:33:17.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:17 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1766555587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:17.657 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:17 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:17.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:17 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1766555587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:17.657 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:17 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:17 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1766555587' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:17 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:18.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:18] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-23T15:33:18.901 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: pgmap v1354: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:18.902 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[222878]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: pgmap v1354: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: failed to encode map e459 with expected crc 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:18.903 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:18 smithi017 ceph-mon[215646]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T15:33:18.941 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:18.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:19.052 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: pgmap v1354: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 779 op/s 2023-09-23T15:33:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: failed to encode map e459 with expected crc 2023-09-23T15:33:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: failed to encode map e459 with expected crc 2023-09-23T15:33:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: failed to encode map e459 with expected crc 2023-09-23T15:33:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:18 smithi027 ceph-mon[195233]: osdmap e459: 8 total, 8 up, 7 in 2023-09-23T15:33:19.122 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:19.341 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:19.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:19.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:19.450 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:19.536 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:19.663 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:19.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:19.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:19.761 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:19.761 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:19.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:19.834 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:19.966 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:19.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:19.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:20.081 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:20.123 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:20.217 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:20.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:20.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:20.329 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:20.383 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:20.458 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:20.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:20.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:20.565 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:20.640 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:20 smithi027 ceph-mon[195233]: pgmap v1356: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T15:33:20.646 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:20.676 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:33:20.774 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:33:20.799 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:20.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:20.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:20.913 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:20.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:33:20.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:20 smithi017 ceph-mon[222878]: pgmap v1356: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T15:33:20.946 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:20 smithi017 ceph-mon[215646]: pgmap v1356: 119 pgs: 119 active+clean; 1.8 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 680 op/s 2023-09-23T15:33:21.084 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:21.101 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:33:21.101 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:33:21.101 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T15:33:21.101 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T15:33:21.249 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:21.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:21.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:21.368 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:21.489 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:21.638 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:21.659 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:21.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:21.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:21.749 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:21.779 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:21.900 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:22.007 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:22.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:22.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:22.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:22.198 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:22.328 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:22.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:22.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:22.434 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:22.481 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:22.565 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:22.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:22.674 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:22.745 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:22.852 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:22 smithi027 ceph-mon[195233]: pgmap v1357: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:22.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:22 smithi017 ceph-mon[215646]: pgmap v1357: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:22.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:22 smithi017 ceph-mon[222878]: pgmap v1357: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:23.037 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:23.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:23.140 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:23.186 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:23.213 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:23.213 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:23.259 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:23.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:23.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:23.365 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:23.421 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:24.149 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:24.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:24.167 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:24.268 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:24.385 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:24.624 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:24.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:24.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:24.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:24.854 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:24.898 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:24 smithi017 ceph-mon[215646]: pgmap v1358: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:24.899 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:24 smithi017 ceph-mon[222878]: pgmap v1358: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:24 smithi027 ceph-mon[195233]: pgmap v1358: 119 pgs: 119 active+clean; 1.5 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 824 op/s 2023-09-23T15:33:25.458 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:33:25.458 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:33:25.458 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:33:25.458 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:33:25.458 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:33:25.465 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:25.465 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:25.493 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:25.536 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:25.592 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:25.631 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:25.677 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:25.715 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:25.728 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.728 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.728 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:33:25.729 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:33:26.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:25 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1849071829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:26.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:25 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1849071829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:26.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:25 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1849071829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:26 smithi027 ceph-mon[195233]: pgmap v1359: 119 pgs: 119 active+clean; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:33:27.137 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:26 smithi017 ceph-mon[215646]: pgmap v1359: 119 pgs: 119 active+clean; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:33:27.138 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:26 smithi017 ceph-mon[222878]: pgmap v1359: 119 pgs: 119 active+clean; 1.3 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 801 op/s 2023-09-23T15:33:27.771 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:27.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:27.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:27.881 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:27.925 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:27.954 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:33:27.983 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:27.991 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:27.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:28.092 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:28.184 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:28] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.43.0" 2023-09-23T15:33:28.192 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:28.480 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:28.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:28.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:28.589 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:28.626 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:28.687 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:28.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:28.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:28.797 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:28.831 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:28.881 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:28.887 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:28 smithi027 ceph-mon[195233]: pgmap v1360: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T15:33:28.888 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:28.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:28.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:28.906 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:28.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:28 smithi017 ceph-mon[215646]: pgmap v1360: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T15:33:28.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:28.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:28 smithi017 ceph-mon[222878]: pgmap v1360: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 802 op/s 2023-09-23T15:33:28.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:28.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:29.082 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:29.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:29.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:29.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:29.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:29.433 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:29.520 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:29.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:29.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:29.628 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:29.678 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:29.761 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:29.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:29.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:29.767 INFO:teuthology.orchestra.run.smithi017.stdout:moved 5 / 357 (1.40056%) 2023-09-23T15:33:29.767 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.32456 -> 6.48074 (expected baseline 6.61168) 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 36 pgs (0.72549 -> 0.705882 * mean) 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 59 -> 58 pgs (1.15686 -> 1.13725 * mean) 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0125 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0137 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 1.0000 -> 0.9500 2023-09-23T15:33:29.768 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 weight 1.0000 -> 0.9500 2023-09-23T15:33:29.774 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:33:29.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:29.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1973723650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:29.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:29.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-09-23T15:33:29.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2023-09-23T15:33:29.937 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]: dispatch 2023-09-23T15:33:29.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1973723650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:29.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:29.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-09-23T15:33:29.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2023-09-23T15:33:29.938 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:29 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]: dispatch 2023-09-23T15:33:29.943 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:29.999 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:33:30.040 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:30.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:30.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:29 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1973723650' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:29 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:33:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]: dispatch 2023-09-23T15:33:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]: dispatch 2023-09-23T15:33:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:29 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]: dispatch 2023-09-23T15:33:30.150 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:30.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:30.234 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:30.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:30.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:30.365 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:30.411 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:30.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:30.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:30.545 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:30.587 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:30.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:30.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:30.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:30.721 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:30.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: pgmap v1361: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: from='client.56671 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: failed to encode map e460 with expected crc 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: failed to encode map e460 with expected crc 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]': finished 2023-09-23T15:33:30.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:30 smithi027 ceph-mon[195233]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: pgmap v1361: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: from='client.56671 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: failed to encode map e460 with expected crc 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: failed to encode map e460 with expected crc 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:33:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]': finished 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[215646]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: pgmap v1361: 119 pgs: 119 active+clean; 1.3 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 703 op/s 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: from='client.56671 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: failed to encode map e460 with expected crc 2023-09-23T15:33:31.194 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: failed to encode map e460 with expected crc 2023-09-23T15:33:31.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:33:31.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 7]}]': finished 2023-09-23T15:33:31.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 2]}]': finished 2023-09-23T15:33:31.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','4':'62260'}"}]': finished 2023-09-23T15:33:31.195 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:30 smithi017 ceph-mon[222878]: osdmap e460: 8 total, 8 up, 7 in 2023-09-23T15:33:31.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: 1.a scrub starts 2023-09-23T15:33:31.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: 1.a scrub ok 2023-09-23T15:33:31.761 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[215646]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: 1.a scrub starts 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: 1.a scrub ok 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.762 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:31 smithi017 ceph-mon[222878]: failed to encode map e461 with expected crc 2023-09-23T15:33:31.764 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T15:33:31.764 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T15:33:31.773 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:31.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:31.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:31.877 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:31.917 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:31.998 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:32.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:32.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:32.104 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:32.106 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: 1.a scrub starts 2023-09-23T15:33:32.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: 1.a scrub ok 2023-09-23T15:33:32.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: failed to encode map e461 with expected crc 2023-09-23T15:33:32.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: osdmap e461: 8 total, 8 up, 7 in 2023-09-23T15:33:32.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: failed to encode map e461 with expected crc 2023-09-23T15:33:32.107 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:31 smithi027 ceph-mon[195233]: failed to encode map e461 with expected crc 2023-09-23T15:33:32.152 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:32.191 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:32.242 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:32.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:32.352 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:32.415 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:32.468 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83106 9.e 9:7e487db8:::benchmark_data_smithi017_543597_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 396.54566823800002, 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1221425209999998, 2023-09-23T15:33:32.469 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83106 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.470 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.471 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916522+0000", 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916551+0000", 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9121000000000001e-05 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.472 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916561+0000", 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4059999999999994e-06 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916570+0000", 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5149999999999995e-06 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918608+0000", 2023-09-23T15:33:32.473 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020381800000000001 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918619+0000", 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1222e-05 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913525+0000", 2023-09-23T15:33:32.474 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490589500000004 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913537+0000", 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1749000000000001e-05 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.028480+0000", 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1149425470000001 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.475 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.029082+0000", 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060205899999999995 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038593+0000", 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095110260000000005 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.476 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038666+0000", 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3088000000000002e-05 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83114 9.d 9:bfb40ed3:::benchmark_data_smithi017_543597_object83113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 396.54561564300002, 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024477069, 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.477 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83114 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.478 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916575+0000", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916600+0000", 2023-09-23T15:33:32.479 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4606000000000001e-05 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916607+0000", 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916613+0000", 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2219999999999999e-06 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.480 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916703+0000", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9252000000000005e-05 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916711+0000", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9039999999999998e-06 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.931237+0000", 2023-09-23T15:33:32.481 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0145257830000001 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933189+0000", 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001951745 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933360+0000", 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017091799999999999 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.482 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940451+0000", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070909079999999999 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940940+0000", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048871099999999996 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940977+0000", 2023-09-23T15:33:32.483 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7722999999999998e-05 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.941053+0000", 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5729000000000006e-05 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83118 9.e 9:7615419c:::benchmark_data_smithi017_543597_object83117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:33:32.484 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 396.54556565600001, 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1299481519999999, 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83118 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.485 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.486 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916625+0000", 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916651+0000", 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5911e-05 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916659+0000", 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2023-09-23T15:33:32.487 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916665+0000", 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9460000000000003e-06 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918662+0000", 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019976350000000002 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.488 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918669+0000", 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2910000000000002e-06 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913570+0000", 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490066300000002 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913580+0000", 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.489 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.030629+0000", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170488599999999 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032411+0000", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001782063 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032533+0000", 2023-09-23T15:33:32.490 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012223299999999999 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038742+0000", 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006208613 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038809+0000", 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6854999999999996e-05 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.491 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045301+0000", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064921479999999997 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045369+0000", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8693000000000001e-05 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046574+0000", 2023-09-23T15:33:32.492 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012047169999999999 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83121 9.e 9:71f6c9a6:::benchmark_data_smithi017_543597_object83120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 396.545435908, 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1305874610000002, 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.493 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83121 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.494 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916755+0000", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916784+0000", 2023-09-23T15:33:32.495 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9522e-05 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916793+0000", 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1000000000000004e-06 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916799+0000", 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7530000000000004e-06 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.496 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918708+0000", 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001908734 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918717+0000", 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6629999999999999e-06 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914427+0000", 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99571021100000001 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.497 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914437+0000", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0592e-05 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034594+0000", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12015687 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034760+0000", 2023-09-23T15:33:32.498 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001661 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034983+0000", 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022256399999999999 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038750+0000", 2023-09-23T15:33:32.499 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037670210000000002 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038859+0000", 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010865 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.501 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046631+0000", 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077723879999999999 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046672+0000", 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0759000000000001e-05 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047343+0000", 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067126700000000002 2023-09-23T15:33:32.502 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83124 9.3 9:c55b3f52:::benchmark_data_smithi017_543597_object83123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 396.54319038300002, 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0418616510000001, 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.503 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83124 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.504 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919000+0000", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919028+0000", 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8895e-05 2023-09-23T15:33:32.505 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919041+0000", 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2996000000000001e-05 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919051+0000", 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5729999999999997e-06 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.506 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919101+0000", 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9864999999999998e-05 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919108+0000", 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2760000000000003e-06 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.944052+0000", 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024943395 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.507 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946310+0000", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022579900000000001 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946472+0000", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016254900000000001 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.958448+0000", 2023-09-23T15:33:32.508 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975497999999999 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960747+0000", 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002299592 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960794+0000", 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6409000000000002e-05 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.509 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960863+0000", 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9227999999999995e-05 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83137 9.e 9:7d999fd1:::benchmark_data_smithi017_543597_object83136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 395.47059421400002, 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056604109, 2023-09-23T15:33:32.510 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83137 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.511 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991596+0000", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991621+0000", 2023-09-23T15:33:32.512 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4907000000000001e-05 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991634+0000", 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2556e-05 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991642+0000", 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1340000000000004e-06 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.513 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991689+0000", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7781000000000002e-05 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991696+0000", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5239999999999997e-06 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038894+0000", 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.047198039 2023-09-23T15:33:32.514 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039062+0000", 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016794100000000001 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039246+0000", 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018399900000000001 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.515 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045231+0000", 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059852339999999999 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046596+0000", 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001365125 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047421+0000", 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082434100000000003 2023-09-23T15:33:32.516 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047483+0000", 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2686000000000003e-05 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048201+0000", 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071811100000000001 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.517 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83228 9.e 9:7f83ce07:::benchmark_data_smithi017_543597_object83227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 395.30604499899999, 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89863112300000003, 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83228 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.518 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.519 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156145+0000", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156194+0000", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8671000000000002e-05 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156210+0000", 2023-09-23T15:33:32.520 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6716999999999999e-05 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156219+0000", 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156278+0000", 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8897999999999998e-05 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.521 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156292+0000", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042118+0000", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88582658800000003 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042277+0000", 2023-09-23T15:33:32.522 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015909299999999999 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042470+0000", 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193022 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048224+0000", 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057530630000000001 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.523 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048291+0000", 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7751999999999998e-05 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.053961+0000", 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056697969999999999 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.524 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054020+0000", 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9419999999999997e-05 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054778+0000", 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075720099999999999 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83232 9.e 9:72608dfb:::benchmark_data_smithi017_543597_object83231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:33:32.525 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 395.29418336100002, 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88775156300000002, 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83232 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.526 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168006+0000", 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.527 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168035+0000", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8824e-05 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168053+0000", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8185e-05 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168065+0000", 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1391e-05 2023-09-23T15:33:32.528 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168131+0000", 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168140+0000", 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7129999999999994e-06 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.529 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048345+0000", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88020536000000005 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048556+0000", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021095900000000001 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048793+0000", 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023668699999999999 2023-09-23T15:33:32.530 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054856+0000", 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060630129999999999 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054871+0000", 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851000000000001e-05 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.531 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054915+0000", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4753999999999999e-05 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054975+0000", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9562999999999999e-05 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078479899999999998 2023-09-23T15:33:32.532 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83259 9.4 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 394.40648640400002, 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.144342432, 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.533 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83259 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:33:32.534 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055704+0000", 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055749+0000", 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5618000000000003e-05 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.535 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012000000000001e-05 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055769+0000", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055829+0000", 2023-09-23T15:33:32.536 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0362000000000002e-05 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056020+0000", 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190998 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056238+0000", 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021735600000000001 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.537 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061804+0000", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055659680000000001 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065167+0000", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033630359999999998 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197822+0000", 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132655089 2023-09-23T15:33:32.538 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197887+0000", 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200048+0000", 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002161035 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.539 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83260 9.4 9:295d7846:::benchmark_data_smithi017_543597_object83259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 394.40640687299998, 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.146144756, 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83260 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.540 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.541 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055784+0000", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055805+0000", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055812+0000", 2023-09-23T15:33:32.542 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4960000000000001e-06 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055819+0000", 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.058865+0000", 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030458220000000001 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.543 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059023+0000", 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015745099999999999 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059189+0000", 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166126 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065176+0000", 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005987132 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.544 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068875+0000", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003699698 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200125+0000", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131249117999999 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200200+0000", 2023-09-23T15:33:32.545 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4951999999999997e-05 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.201929+0000", 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001729897 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83261 9.4 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:33:32.546 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 394.405581563, 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.147188163999999, 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83261 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.547 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056609+0000", 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:33:32.548 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056631+0000", 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1569999999999998e-05 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056646+0000", 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4924e-05 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.549 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056654+0000", 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0160000000000008e-06 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061842+0000", 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051878740000000003 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062037+0000", 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019499900000000001 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.550 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062246+0000", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020959400000000001 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065285+0000", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030387909999999999 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069749+0000", 2023-09-23T15:33:32.551 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044636470000000003 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202002+0000", 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132253035 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202066+0000", 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4215999999999996e-05 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.552 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203798+0000", 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017322940000000001 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83265 9.4 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 394.39663829, 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.139692201999999, 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.553 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83265 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.554 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065572+0000", 2023-09-23T15:33:32.555 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8663000000000001e-05 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065576+0000", 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7530000000000001e-06 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065580+0000", 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.326e-06 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.556 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065625+0000", 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5077999999999998e-05 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065774+0000", 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014929099999999999 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065946+0000", 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017193699999999999 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.557 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068958+0000", 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030115939999999998 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069898+0000", 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093998699999999998 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.558 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203861+0000", 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.133963208000001 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203918+0000", 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6950999999999997e-05 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.205245+0000", 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013277479999999999 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.559 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88039 9.d 9:b338c46d:::benchmark_data_smithi017_543597_object4248:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05141161300003, 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90200742499999997, 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.560 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88039 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.561 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410779+0000", 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410784+0000", 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5480000000000001e-06 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.562 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410796+0000", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1653e-05 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410807+0000", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1181e-05 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410861+0000", 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3981999999999999e-05 2023-09-23T15:33:32.563 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410880+0000", 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8553999999999999e-05 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420827+0000", 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0099472469999999993 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.564 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420840+0000", 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2969999999999999e-05 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.302886+0000", 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88204606500000005 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303061+0000", 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175076 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.565 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312708+0000", 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096470840000000002 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312787+0000", 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9296999999999997e-05 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.566 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88041 9.d 9:b6e8eb25:::benchmark_data_smithi017_543597_object4250:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05137059600003, 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90221110000000004, 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88041 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.567 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.568 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410820+0000", 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410824+0000", 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3720000000000001e-06 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410831+0000", 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.569 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410837+0000", 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.19e-06 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410931+0000", 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3634000000000005e-05 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:33:32.570 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410938+0000", 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422325+0000", 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011386744000000001 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422336+0000", 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1028e-05 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.571 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303400+0000", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88106453799999995 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303535+0000", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013442700000000001 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303696+0000", 2023-09-23T15:33:32.572 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016155799999999999 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309737+0000", 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060407810000000003 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309784+0000", 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7237000000000002e-05 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.573 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312835+0000", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030510619999999998 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312900+0000", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4393999999999994e-05 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313032+0000", 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013220300000000001 2023-09-23T15:33:32.574 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88044 9.d 9:bfc8f2ba:::benchmark_data_smithi017_543597_object4253:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05135017700002, 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90561986000000005, 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.575 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88044 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.576 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410843+0000", 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2079999999999999e-06 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.577 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410849+0000", 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6489999999999996e-06 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410854+0000", 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9810000000000003e-06 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410963+0000", 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109166 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.578 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410970+0000", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5649999999999998e-06 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422375+0000", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01140536 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422395+0000", 2023-09-23T15:33:32.579 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9913000000000001e-05 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306676+0000", 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88428063000000001 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306827+0000", 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015183300000000001 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.580 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306965+0000", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137427 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313079+0000", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061140869999999998 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313118+0000", 2023-09-23T15:33:32.581 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316281+0000", 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031639160000000001 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316328+0000", 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6897000000000002e-05 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.582 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316461+0000", 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013290900000000001 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88045 9.d 9:b617d99b:::benchmark_data_smithi017_543597_object4254:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05133327499999, 2023-09-23T15:33:32.583 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91186458299999995, 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88045 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.584 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.585 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410860+0000", 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5600000000000001e-06 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410866+0000", 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6860000000000003e-06 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.586 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410871+0000", 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9980000000000003e-06 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410993+0000", 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122359 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411000+0000", 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0950000000000001e-06 2023-09-23T15:33:32.587 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422421+0000", 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011421093 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422430+0000", 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.588 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309805+0000", 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88737535099999998 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309933+0000", 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012809900000000001 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310023+0000", 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9247999999999997e-05 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.589 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313160+0000", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031370759999999999 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316471+0000", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003311278 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322477+0000", 2023-09-23T15:33:32.590 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060065279999999997 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322536+0000", 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8792000000000002e-05 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322723+0000", 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018669199999999999 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.591 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88046 9.3 9:c0e328f8:::benchmark_data_smithi017_543597_object4255:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05131649499998, 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89982923699999995, 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88046 2023-09-23T15:33:32.592 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.593 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410877+0000", 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4669999999999998e-06 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.594 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410885+0000", 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.841e-06 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410890+0000", 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411148+0000", 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025797 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.595 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411157+0000", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5380000000000002e-06 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413770+0000", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026130089999999999 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413782+0000", 2023-09-23T15:33:32.596 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.289e-05 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303621+0000", 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88983906499999998 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303734+0000", 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011293099999999999 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.597 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303859+0000", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012465600000000001 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310319+0000", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006459752 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310340+0000", 2023-09-23T15:33:32.598 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1526999999999999e-05 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310470+0000", 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012976200000000001 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310519+0000", 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8906999999999998e-05 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.599 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310704+0000", 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018528199999999999 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88047 9.3 9:c39718c3:::benchmark_data_smithi017_543597_object4256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 351.05128899499999, 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90486523799999996, 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.600 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88047 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.601 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410902+0000", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410905+0000", 2023-09-23T15:33:32.602 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6520000000000002e-06 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410911+0000", 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410917+0000", 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5759999999999996e-06 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.603 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411181+0000", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026399799999999999 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411187+0000", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6200000000000001e-06 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413808+0000", 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026202399999999998 2023-09-23T15:33:32.604 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413816+0000", 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2530000000000004e-06 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307084+0000", 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89326843099999997 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.605 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307210+0000", 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125623 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307340+0000", 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013041499999999999 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.311327+0000", 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039864119999999999 2023-09-23T15:33:32.606 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313609+0000", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002282315 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315515+0000", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019060640000000001 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315579+0000", 2023-09-23T15:33:32.607 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3367999999999994e-05 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315768+0000", 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018929 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18484 10.2 10:46869aa5:::benchmark_data_smithi017_604517_object18483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:33:32.608 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 208.39195848599999, 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40252255100000001, 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18484 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.609 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070231+0000", 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:32.610 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070257+0000", 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6016e-05 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070274+0000", 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6682999999999999e-05 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.611 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070283+0000", 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0580000000000004e-06 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070333+0000", 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0442999999999998e-05 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070465+0000", 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013140100000000001 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.612 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070591+0000", 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126329 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072125+0000", 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001533855 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.613 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077175+0000", 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050495369999999998 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472642+0000", 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39546752299999999 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.614 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472704+0000", 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2088999999999995e-05 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472756+0000", 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1493999999999999e-05 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18486 10.2 10:43afa650:::benchmark_data_smithi017_604517_object18485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:33:32.615 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 208.39153159899999, 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40280343299999999, 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18486 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:33:32.616 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070659+0000", 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.617 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070685+0000", 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5811000000000001e-05 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070697+0000", 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1596999999999999e-05 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:33:32.618 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070705+0000", 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1359999999999997e-06 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071435+0000", 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072993699999999995 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071545+0000", 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011006999999999999 2023-09-23T15:33:32.619 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071635+0000", 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0004000000000001e-05 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072295+0000", 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00066030600000000004 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.620 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077270+0000", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049749349999999998 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473366+0000", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39609567099999998 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473429+0000", 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3528999999999994e-05 2023-09-23T15:33:32.621 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473463+0000", 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4245999999999997e-05 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:33:32.622 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:33:32.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:32.623 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:32.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:32.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: pgmap v1364: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: failed to encode map e462 with expected crc 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: failed to encode map e462 with expected crc 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: failed to encode map e462 with expected crc 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: failed to encode map e462 with expected crc 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 459'11918 2023-09-23T15:33:32.765 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[215646]: failed to encode map e462 with expected crc 2023-09-23T15:33:32.814 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: pgmap v1364: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:33:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 459'11918 2023-09-23T15:33:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:32 smithi027 ceph-mon[195233]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: pgmap v1364: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 993 op/s 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: osdmap e462: 8 total, 8 up, 7 in 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: 10.7 starting backfill to osd.4 from (0'0,0'0] MAX to 459'11918 2023-09-23T15:33:33.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:32 smithi017 ceph-mon[222878]: failed to encode map e462 with expected crc 2023-09-23T15:33:33.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:33:33.648 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:33:33.696 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:33.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:33.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:33.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:33.837 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:34.206 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:34.261 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:34.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:34.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:34.369 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:34.460 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:34.476 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:34.543 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:34.583 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:34.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:34.692 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:34.774 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:34 smithi027 ceph-mon[195233]: pgmap v1366: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T15:33:34.779 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:34.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:34 smithi017 ceph-mon[215646]: pgmap v1366: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T15:33:34.866 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:34 smithi017 ceph-mon[222878]: pgmap v1366: 119 pgs: 119 active+clean; 1.0 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 662 op/s 2023-09-23T15:33:35.198 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:35.200 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:35.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:35.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:35.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:35.272 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:35.309 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:35.313 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:35.346 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:35.361 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:35.379 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:35.418 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:35.450 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:35.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:35.461 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:35.485 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.485 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.486 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:33:35.557 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:35.606 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:35.694 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:35.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:35.698 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:35.798 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:35.842 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:35.907 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:35.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:35.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:36.011 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:36.035 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3123701541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:36.038 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:36.083 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:36.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:36.187 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:36.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3123701541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:36.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3123701541' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:36.215 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:36.258 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:36.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:36.261 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:36.362 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:36.393 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:36.593 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:36.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:36.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:36.698 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:36.726 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: pgmap v1367: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 914 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 242/43731 objects misplaced (0.553%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T15:33:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:36 smithi027 ceph-mon[195233]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: pgmap v1367: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 914 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 242/43731 objects misplaced (0.553%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:37.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T15:33:37.135 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.136 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[222878]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: pgmap v1367: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 914 MiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.09k op/s; 242/43731 objects misplaced (0.553%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: osdmap e463: 8 total, 8 up, 7 in 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.141 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:36 smithi017 ceph-mon[215646]: failed to encode map e463 with expected crc 2023-09-23T15:33:37.190 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:37.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:37.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:37.299 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:37.342 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:37.417 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:37.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:37.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:37.521 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:37.573 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:37.644 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:37.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:37.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[222878]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[222878]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[222878]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[222878]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[215646]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.818 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[215646]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[215646]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T15:33:37.819 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:37 smithi017 ceph-mon[215646]: failed to encode map e464 with expected crc 2023-09-23T15:33:37.821 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:37.945 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:37.950 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:38.051 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:33:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:37 smithi027 ceph-mon[195233]: failed to encode map e464 with expected crc 2023-09-23T15:33:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:37 smithi027 ceph-mon[195233]: failed to encode map e464 with expected crc 2023-09-23T15:33:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:37 smithi027 ceph-mon[195233]: osdmap e464: 8 total, 8 up, 7 in 2023-09-23T15:33:38.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:37 smithi027 ceph-mon[195233]: failed to encode map e464 with expected crc 2023-09-23T15:33:38.113 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:38] "GET /metrics HTTP/1.1" 200 36416 "" "Prometheus/2.43.0" 2023-09-23T15:33:38.123 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:38.299 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:33:38.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:38.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:33:38.414 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:38.480 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:38.577 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:33:38.577 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.09449 -> 6.09449 (expected baseline 6.61168) 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 58 -> 58 pgs (1.13725 -> 1.13725 * mean) 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0106 2023-09-23T15:33:38.578 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.1060 2023-09-23T15:33:38.590 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:33:38.663 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:38.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:38.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:38.742 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:38.742 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:38.767 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:38.863 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:38.863 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:33:38.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[222878]: pgmap v1370: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:38.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/296139472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:38.890 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:38.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[215646]: pgmap v1370: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:38.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/296139472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:38.891 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:38 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:38.960 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:38.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:39.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:38 smithi027 ceph-mon[195233]: pgmap v1370: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/296139472' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:39.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:38 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:39.126 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:39.207 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:39.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:39.316 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:39.358 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:33:39.361 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:39.431 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:39.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:39.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:39.537 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:39.634 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:39.765 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: from='client.56683 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:39 smithi027 ceph-mon[195233]: pgmap v1372: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:40.142 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: from='client.56683 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[215646]: pgmap v1372: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: from='client.56683 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:40.143 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: osdmap e465: 8 total, 8 up, 7 in 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: failed to encode map e465 with expected crc 2023-09-23T15:33:40.144 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:39 smithi017 ceph-mon[222878]: pgmap v1372: 119 pgs: 1 active+remapped+backfilling, 118 active+clean; 899 MiB data, 6.6 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 242/43041 objects misplaced (0.562%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T15:33:40.170 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:33:40.170 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T15:33:40.319 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:40.321 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:40.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:40.435 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:40.478 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:40.608 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:40.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:40.662 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:40.718 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:40.765 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:41.017 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:41.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:41.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:41.124 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:41.205 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:41.232 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:41.275 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:41.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:41.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:41.378 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:41.438 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:41.712 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:41.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:41.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:41.826 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:41.890 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:42.189 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:42 smithi017 ceph-mon[215646]: pgmap v1373: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 2.4 MiB/s, 39 objects/s recovering 2023-09-23T15:33:42.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:42 smithi017 ceph-mon[222878]: pgmap v1373: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 2.4 MiB/s, 39 objects/s recovering 2023-09-23T15:33:42.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:42.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:42.302 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:42.324 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:42.324 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:42.348 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:42 smithi027 ceph-mon[195233]: pgmap v1373: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 634 op/s; 2.4 MiB/s, 39 objects/s recovering 2023-09-23T15:33:42.366 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:42.480 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:33:42.633 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:42.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:42.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:42.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:42.805 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:42.899 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:42.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:42.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:43.006 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:43.108 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:43.196 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:33:43.378 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:43.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:43.383 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:43.484 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:43.582 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:43.716 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:43.769 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:43.795 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:43.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:43.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:43.903 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:43.979 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:43.980 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:44.037 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:44.096 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:44.135 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:44.159 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:44.166 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:44.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:44.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:44.216 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:44.256 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:44.270 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:44.301 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.307 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.308 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.308 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:33:44.327 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:44.393 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:44.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:44.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:44.405 INFO:teuthology.orchestra.run.smithi017.stdout:moved 0 / 357 (0%) 2023-09-23T15:33:44.405 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:44.405 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.09449 -> 6.09449 (expected baseline 6.61168) 2023-09-23T15:33:44.405 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 37 pgs (0.72549 -> 0.72549 * mean) 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 58 -> 58 pgs (1.13725 -> 1.13725 * mean) 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 3 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0096 2023-09-23T15:33:44.406 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0962 2023-09-23T15:33:44.408 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:33:44.499 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:44.533 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:44 smithi027 ceph-mon[195233]: pgmap v1374: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:33:44.533 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:44.533 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:44.533 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/139010989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:44.538 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:44.613 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:44.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:44.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[215646]: pgmap v1374: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:33:44.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:44.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:44.632 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/139010989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:44.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[222878]: pgmap v1374: 119 pgs: 119 active+clean; 676 MiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 525 op/s; 2.0 MiB/s, 32 objects/s recovering 2023-09-23T15:33:44.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:44.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:44.635 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/139010989' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:44.720 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:44.765 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:45.439 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:45 smithi027 ceph-mon[195233]: from='client.56695 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:45.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:45 smithi017 ceph-mon[215646]: from='client.56695 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:45.695 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:45 smithi017 ceph-mon[222878]: from='client.56695 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:45.751 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83113 9.1 9:84c8439c:::benchmark_data_smithi017_543597_object83112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 409.85553755900003, 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067041744, 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:45.752 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83113 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892814+0000", 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892915+0000", 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101517 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892934+0000", 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8862999999999999e-05 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892943+0000", 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:33:45.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892994+0000", 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1109000000000001e-05 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893124+0000", 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013067299999999999 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893274+0000", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149534 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896402+0000", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031281579999999998 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906247+0000", 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098445379999999999 2023-09-23T15:33:45.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906253+0000", 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7000000000000002e-06 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.947718+0000", 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414645709999999 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948393+0000", 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00067537000000000001 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959773+0000", 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01137928 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959857+0000", 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4637999999999998e-05 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18474 10.d 10:b2d26cad:::benchmark_data_smithi017_604517_object18473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 221.67988943, 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40456172899999998, 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.760 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18474 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068463+0000", 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068810+0000", 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034735199999999999 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068821+0000", 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0264e-05 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068827+0000", 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.443e-06 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069014+0000", 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018699400000000001 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069120+0000", 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010654000000000001 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.764 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069251+0000", 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013087999999999999 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072104+0000", 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028525260000000002 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077080+0000", 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049762950000000004 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472922+0000", 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39584164300000002 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472980+0000", 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8023000000000003e-05 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473025+0000", 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5479000000000003e-05 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.767 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18476 10.d 10:b8c42a96:::benchmark_data_smithi017_604517_object18475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 221.67951285800001, 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40577544799999998, 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.768 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18476 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068872+0000", 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2694000000000002e-05 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068881+0000", 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.371e-06 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068885+0000", 2023-09-23T15:33:45.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3229999999999999e-06 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070067+0000", 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011816400000000001 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070195+0000", 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012812699999999999 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070349+0000", 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015427600000000001 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072111+0000", 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017617259999999999 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077110+0000", 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004998794 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474512+0000", 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39740209999999998 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474569+0000", 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7841999999999999e-05 2023-09-23T15:33:45.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474616+0000", 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6134000000000002e-05 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18477 10.a 10:55f810cd:::benchmark_data_smithi017_604517_object18476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 221.679463164, 2023-09-23T15:33:45.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.403843171, 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18477 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068889+0000", 2023-09-23T15:33:45.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068920+0000", 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0133000000000001e-05 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068927+0000", 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5780000000000001e-06 2023-09-23T15:33:45.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068932+0000", 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7840000000000003e-06 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072819+0000", 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003887129 2023-09-23T15:33:45.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072970+0000", 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015059700000000001 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.073147+0000", 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017778 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.075739+0000", 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025920600000000002 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077221+0000", 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014813630000000001 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472627+0000", 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39540625000000001 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472672+0000", 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5337999999999999e-05 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472733+0000", 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0619999999999999e-05 2023-09-23T15:33:45.782 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195331 10.7 10:e31d95f4:::benchmark_data_smithi017_604517_object89220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.010624541999999, 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1400244129999999, 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.783 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195331 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737727+0000", 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737734+0000", 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9539999999999998e-06 2023-09-23T15:33:45.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737745+0000", 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.132e-05 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737750+0000", 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3329999999999998e-06 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737809+0000", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9676000000000003e-05 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737892+0000", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2812999999999995e-05 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737966+0000", 2023-09-23T15:33:45.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4040000000000003e-05 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.742332+0000", 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043653770000000001 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.750884+0000", 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085524480000000007 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758674+0000", 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077902680000000004 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758686+0000", 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1745e-05 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759654+0000", 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0009683540000001 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759672+0000", 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7527e-05 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.877695+0000", 2023-09-23T15:33:45.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.118023293 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.877753+0000", 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7645999999999997e-05 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195336 10.7 10:e0c5d133:::benchmark_data_smithi017_604517_object89225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T15:33:45.791 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 16.004906160000001, 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1346623220000001, 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195336 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743446+0000", 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743451+0000", 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8010000000000003e-06 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743468+0000", 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6775000000000001e-05 2023-09-23T15:33:45.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743473+0000", 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743503+0000", 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0386e-05 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743624+0000", 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120671 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743739+0000", 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115364 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.750339+0000", 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066004480000000001 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.753689+0000", 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033490400000000002 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758831+0000", 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051428940000000003 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758840+0000", 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2239999999999995e-06 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759707+0000", 2023-09-23T15:33:45.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0008677020000001 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759712+0000", 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9830000000000004e-06 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878063+0000", 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1183501629999999 2023-09-23T15:33:45.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878109+0000", 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6693999999999997e-05 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195402 10.7 10:e8a13b56:::benchmark_data_smithi017_604517_object89291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:33:45.800 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.916832956, 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0597443800000002, 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195402 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831518+0000", 2023-09-23T15:33:45.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831527+0000", 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2549999999999997e-06 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831544+0000", 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7051e-05 2023-09-23T15:33:45.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831552+0000", 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0600000000000008e-06 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831615+0000", 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3020000000000003e-05 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831626+0000", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1319e-05 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759726+0000", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92809969400000003 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759733+0000", 2023-09-23T15:33:45.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7889999999999997e-06 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878161+0000", 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1184288579999999 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878312+0000", 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000150459 2023-09-23T15:33:45.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878488+0000", 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017655899999999999 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884902+0000", 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064132140000000004 2023-09-23T15:33:45.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884913+0000", 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1184e-05 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885002+0000", 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9128000000000005e-05 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891051+0000", 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060486730000000001 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891118+0000", 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7461999999999999e-05 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891264+0000", 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014637299999999999 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195408 10.7 10:e55c1ca3:::benchmark_data_smithi017_604517_object89297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.811 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.914805504, 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0579925349999999, 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195408 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:33:45.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833547+0000", 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833549+0000", 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9970000000000001e-06 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833554+0000", 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1050000000000003e-06 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833558+0000", 2023-09-23T15:33:45.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4790000000000001e-06 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833601+0000", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3269999999999997e-05 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833610+0000", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2490000000000007e-06 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759748+0000", 2023-09-23T15:33:45.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92613742399999999 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759755+0000", 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.148e-06 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881083+0000", 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1213284610000001 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881184+0000", 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010103300000000001 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881595+0000", 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041013399999999999 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884953+0000", 2023-09-23T15:33:45.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033583160000000001 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888041+0000", 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030881319999999999 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.890997+0000", 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029561489999999999 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891415+0000", 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000418102 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891454+0000", 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8875999999999998e-05 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891540+0000", 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5929999999999999e-05 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195418 10.7 10:eae79fec:::benchmark_data_smithi017_604517_object89307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.902815832, 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0517118569999999, 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195418 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.821 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.822 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845540+0000", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3699999999999999e-06 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845547+0000", 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3669999999999999e-06 2023-09-23T15:33:45.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845552+0000", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7520000000000004e-06 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845607+0000", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.524e-05 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845614+0000", 2023-09-23T15:33:45.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3149999999999997e-06 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759809+0000", 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91419547400000001 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759818+0000", 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5359999999999992e-06 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885028+0000", 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1252105480000001 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885118+0000", 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9884999999999997e-05 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885252+0000", 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013337099999999999 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891282+0000", 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060303270000000003 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891357+0000", 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5458999999999997e-05 2023-09-23T15:33:45.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891396+0000", 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9125000000000002e-05 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897023+0000", 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056263069999999997 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897087+0000", 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4036999999999997e-05 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897249+0000", 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016222400000000001 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195467 10.7 10:eb064bcf:::benchmark_data_smithi017_604517_object89356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.853318069, 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0025557250000001, 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195467 2023-09-23T15:33:45.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:33:45.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895034+0000", 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895039+0000", 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6689999999999997e-06 2023-09-23T15:33:45.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895048+0000", 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8470000000000002e-06 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895053+0000", 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3600000000000004e-06 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895104+0000", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1248000000000001e-05 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895116+0000", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1756000000000001e-05 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759849+0000", 2023-09-23T15:33:45.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86473266500000001 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759854+0000", 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1200000000000001e-06 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888086+0000", 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1282324240000001 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888206+0000", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120252 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888298+0000", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1280999999999996e-05 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891292+0000", 2023-09-23T15:33:45.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002994118 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891563+0000", 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027159800000000001 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891584+0000", 2023-09-23T15:33:45.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0157e-05 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897377+0000", 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057933710000000003 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897440+0000", 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3437999999999998e-05 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897591+0000", 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015026699999999999 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195469 10.7 10:e6a6a234:::benchmark_data_smithi017_604517_object89358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:33:45.839 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.850879695, 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0175167780000001, 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195469 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:33:45.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897472+0000", 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897478+0000", 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2070000000000003e-06 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897493+0000", 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4875000000000001e-05 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897499+0000", 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4740000000000002e-06 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897555+0000", 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5829999999999999e-05 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897562+0000", 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8290000000000002e-06 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759879+0000", 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86231730600000001 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759883+0000", 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4379999999999997e-06 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891597+0000", 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.131713433 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891688+0000", 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1167000000000003e-05 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891777+0000", 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8654999999999997e-05 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.896977+0000", 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052007900000000003 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897336+0000", 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035870999999999999 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907873+0000", 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010536581 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914774+0000", 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069013700000000004 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914840+0000", 2023-09-23T15:33:45.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5710999999999995e-05 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914990+0000", 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015026699999999999 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195470 10.7 10:e10e854e:::benchmark_data_smithi017_604517_object89359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.848 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.850842026, 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0178075889999998, 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195470 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:33:45.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897513+0000", 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6280000000000001e-06 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897523+0000", 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3109999999999995e-06 2023-09-23T15:33:45.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897527+0000", 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3950000000000003e-06 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897595+0000", 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8285000000000001e-05 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.852 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897605+0000", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3160000000000003e-06 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759906+0000", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86230155399999997 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759910+0000", 2023-09-23T15:33:45.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.177e-06 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894172+0000", 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1342621820000001 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894290+0000", 2023-09-23T15:33:45.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117631 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894379+0000", 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9316999999999999e-05 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897322+0000", 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029428710000000001 2023-09-23T15:33:45.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.900796+0000", 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034740050000000001 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915068+0000", 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014271855999999999 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915113+0000", 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4650000000000001e-05 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915181+0000", 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7730000000000004e-05 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915319+0000", 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013808700000000001 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195472 10.7 10:e3ee2323:::benchmark_data_smithi017_604517_object89361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.850777857000001, 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.01809517, 2023-09-23T15:33:45.858 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195472 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:33:45.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897578+0000", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9160000000000001e-06 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897586+0000", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0099999999999995e-06 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897590+0000", 2023-09-23T15:33:45.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7450000000000003e-06 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897626+0000", 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5605000000000002e-05 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897630+0000", 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3059999999999999e-06 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759930+0000", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86229939200000005 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759934+0000", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1810000000000003e-06 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897641+0000", 2023-09-23T15:33:45.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1377075400000001 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897743+0000", 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102106 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897827+0000", 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3660999999999997e-05 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907809+0000", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099816060000000005 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915336+0000", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075277720000000003 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915439+0000", 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102908 2023-09-23T15:33:45.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915482+0000", 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2416999999999997e-05 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915538+0000", 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6577999999999999e-05 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915670+0000", 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013201599999999999 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195497 10.7 10:ec423fb3:::benchmark_data_smithi017_604517_object89386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.819268787, 2023-09-23T15:33:45.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9868509519999999, 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195497 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929083+0000", 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.869 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929089+0000", 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3180000000000003e-06 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929102+0000", 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3128e-05 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929109+0000", 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3690000000000003e-06 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929174+0000", 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4955000000000004e-05 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929187+0000", 2023-09-23T15:33:45.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3665e-05 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759962+0000", 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83077460400000003 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759970+0000", 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8480000000000001e-06 2023-09-23T15:33:45.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.900848+0000", 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.140877777 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.901004+0000", 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000156088 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.901181+0000", 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000177646 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907842+0000", 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066605249999999996 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915348+0000", 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075065030000000003 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915697+0000", 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034851400000000002 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915793+0000", 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6348000000000007e-05 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915847+0000", 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3866999999999998e-05 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915935+0000", 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8084999999999994e-05 2023-09-23T15:33:45.876 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195557 10.7 10:ebbb3f94:::benchmark_data_smithi017_604517_object89446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.738903623000001, 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9068306000000002, 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.877 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195557 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009454+0000", 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9219999999999996e-06 2023-09-23T15:33:45.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009465+0000", 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1234e-05 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009471+0000", 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9429999999999997e-06 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.011481+0000", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020101619999999998 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.011487+0000", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6270000000000002e-06 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759994+0000", 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74850652900000003 2023-09-23T15:33:45.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759998+0000", 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4290000000000003e-06 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904399+0000", 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1444002740000001 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904540+0000", 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014094500000000001 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904696+0000", 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015625900000000001 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907907+0000", 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032108760000000001 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915360+0000", 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074533530000000002 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915989+0000", 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00062900200000000003 2023-09-23T15:33:45.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916072+0000", 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3102999999999994e-05 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916127+0000", 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5126e-05 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.885 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916280+0000", 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015262300000000001 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195584 10.7 10:eb9b89c9:::benchmark_data_smithi017_604517_object89473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.690381507, 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8769677090000001, 2023-09-23T15:33:45.886 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195584 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.887 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057969+0000", 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.888 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057981+0000", 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.146e-05 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058014+0000", 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3043000000000002e-05 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058027+0000", 2023-09-23T15:33:45.889 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3183000000000001e-05 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058163+0000", 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013621299999999999 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058190+0000", 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6449e-05 2023-09-23T15:33:45.890 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760021+0000", 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70183132199999998 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760026+0000", 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6330000000000004e-06 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.891 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907929+0000", 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1479031609999999 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.908041+0000", 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011233 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.892 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.908178+0000", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013667 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915372+0000", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071940249999999997 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915745+0000", 2023-09-23T15:33:45.893 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037352199999999999 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916331+0000", 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058567700000000005 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934743+0000", 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018411927000000002 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.894 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934801+0000", 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7664999999999998e-05 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934939+0000", 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138589 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.895 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195610 10.7 10:efb219c1:::benchmark_data_smithi017_604517_object89499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.63898753, 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8259069960000001, 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.896 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195610 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.897 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109364+0000", 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109371+0000", 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2230000000000001e-06 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.898 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109389+0000", 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7427000000000001e-05 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109395+0000", 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8730000000000004e-06 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.899 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109446+0000", 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0974999999999999e-05 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109456+0000", 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0292000000000001e-05 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760047+0000", 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.65059110499999995 2023-09-23T15:33:45.900 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760051+0000", 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0589999999999996e-06 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911246+0000", 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1511953669999999 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.901 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911420+0000", 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017322099999999999 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911571+0000", 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015136699999999999 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.902 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915383+0000", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038122500000000001 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916028+0000", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064446299999999996 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935004+0000", 2023-09-23T15:33:45.903 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01897654 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935042+0000", 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8118999999999999e-05 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935087+0000", 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4762999999999998e-05 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.904 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935273+0000", 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018532100000000001 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195653 10.7 10:e4163779:::benchmark_data_smithi017_604517_object89542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:33:45.905 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.556532992999999, 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7437647190000001, 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195653 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.906 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.907 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191818+0000", 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191827+0000", 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7690000000000002e-06 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.908 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191846+0000", 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8978999999999999e-05 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191856+0000", 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7820000000000005e-06 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191919+0000", 2023-09-23T15:33:45.909 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2704000000000001e-05 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191929+0000", 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.081e-05 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760070+0000", 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56814065000000002 2023-09-23T15:33:45.910 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760074+0000", 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0910000000000003e-06 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916357+0000", 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1562823070000001 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.911 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916461+0000", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104323 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916550+0000", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9177000000000004e-05 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.922092+0000", 2023-09-23T15:33:45.912 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055422589999999999 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934700+0000", 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012608199 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935356+0000", 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00065563000000000004 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.913 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935390+0000", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3699999999999999e-05 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935443+0000", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3597000000000002e-05 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935585+0000", 2023-09-23T15:33:45.914 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141349 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195677 10.7 10:e92f236a:::benchmark_data_smithi017_604517_object89566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.508944156, 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6964536799999999, 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.915 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195677 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.916 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239408+0000", 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.917 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239413+0000", 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7550000000000002e-06 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239423+0000", 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8660000000000001e-06 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.918 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239427+0000", 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.262e-06 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239483+0000", 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5686000000000002e-05 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239492+0000", 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3719999999999994e-06 2023-09-23T15:33:45.919 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760096+0000", 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.52060388099999999 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760100+0000", 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0749999999999999e-06 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.920 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919203+0000", 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1591034630000001 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919314+0000", 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110086 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919411+0000", 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7120999999999994e-05 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.921 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.922156+0000", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027455940000000001 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934951+0000", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012794244999999999 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935636+0000", 2023-09-23T15:33:45.922 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068527400000000004 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935675+0000", 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9638000000000002e-05 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935724+0000", 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8804999999999998e-05 2023-09-23T15:33:45.923 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935863+0000", 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013832199999999999 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195695 10.7 10:e3e7c11c:::benchmark_data_smithi017_604517_object89584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:33:45.924 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 15.375351065, 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5632770379999998, 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195695 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.925 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.926 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373001+0000", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373006+0000", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6999999999999999e-06 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373016+0000", 2023-09-23T15:33:45.927 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9699999999999994e-06 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373020+0000", 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1790000000000001e-06 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373064+0000", 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3591000000000003e-05 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.928 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373073+0000", 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1689999999999998e-06 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760112+0000", 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38703970300000001 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:33:45.929 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760118+0000", 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2089999999999996e-06 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923633+0000", 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1635158809999999 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923768+0000", 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013468099999999999 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.930 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923910+0000", 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141428 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934966+0000", 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011056015000000001 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.931 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935321+0000", 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035499899999999999 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936057+0000", 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073604400000000004 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936096+0000", 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8983999999999998e-05 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.932 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936147+0000", 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1069000000000002e-05 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936279+0000", 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013231199999999999 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.933 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:33:45.934 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:33:45.934 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:45.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:45.935 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:45.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:46.036 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:46.066 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:46.120 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:46.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:46.223 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:46.269 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:46.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:46 smithi027 ceph-mon[195233]: pgmap v1375: 119 pgs: 119 active+clean; 439 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:33:46.616 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:46 smithi017 ceph-mon[215646]: pgmap v1375: 119 pgs: 119 active+clean; 439 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:33:46.620 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:46 smithi017 ceph-mon[222878]: pgmap v1375: 119 pgs: 119 active+clean; 439 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 893 op/s; 1.8 MiB/s, 28 objects/s recovering 2023-09-23T15:33:46.638 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:33:46.641 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:33:46.641 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T15:33:46.682 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:46.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:46.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:46.791 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:46.841 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:46.935 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:46.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:47.043 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:47.138 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:47.184 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:47.304 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:47.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:47.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:47.416 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:47.462 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:47.547 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:47.548 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:47.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:47.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:47.652 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:47.742 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:47.898 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:47.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:48.007 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:48.036 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:48.080 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:48.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:48.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:48.183 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:48.220 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:48.306 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:48] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T15:33:48.414 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:33:48.508 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:48.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:48.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:48.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[215646]: pgmap v1376: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:48.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/619066109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:48.668 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:48.696 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:33:48.719 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:48.831 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:48.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:48.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:48 smithi027 ceph-mon[195233]: pgmap v1376: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:48.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/619066109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:48.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:48 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:48.935 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[222878]: pgmap v1376: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/619066109' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:48.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:48 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:33:49.027 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:49.158 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:49.158 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:33:49.158 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:33:49.158 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:49.854 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:49.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:49.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T15:33:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: failed to encode map e466 with expected crc 2023-09-23T15:33:49.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: failed to encode map e466 with expected crc 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: failed to encode map e466 with expected crc 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: failed to encode map e466 with expected crc 2023-09-23T15:33:49.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:49 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:49.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:50.056 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:50.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:33:50.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: osdmap e466: 8 total, 8 up, 7 in 2023-09-23T15:33:50.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: failed to encode map e466 with expected crc 2023-09-23T15:33:50.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: failed to encode map e466 with expected crc 2023-09-23T15:33:50.057 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:49 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:50.071 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:33:50.077 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:50.173 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:50.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:50.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:50.280 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:50.352 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:50.363 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:50.453 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:50.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:50.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:50.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:50.592 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:50.659 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:50.762 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:50.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:50.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:50.873 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:50.911 INFO:teuthology.orchestra.run.smithi017.stdout:moved 7 / 357 (1.96078%) 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.09449 -> 6.98979 (expected baseline 6.61168) 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 36 pgs (0.72549 -> 0.705882 * mean) 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 58 -> 60 pgs (1.13725 -> 1.17647 * mean) 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:oload 110 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 0.05 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0084 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0093 2023-09-23T15:33:50.912 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0 weight 0.9500 -> 0.9000 2023-09-23T15:33:50.913 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2 weight 0.9500 -> 0.9000 2023-09-23T15:33:50.913 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T15:33:50.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:50 smithi017 ceph-mon[222878]: pgmap v1378: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:50.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:50 smithi017 ceph-mon[215646]: pgmap v1378: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:50.914 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:50 smithi017 conmon[202315]: 2023-09-23T15:33:50.909+0000 7fdcb8de7700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T15:33:50.915 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:33:50.926 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:51.040 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:50 smithi027 ceph-mon[195233]: pgmap v1378: 119 pgs: 119 active+clean; 424 MiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 1.5 MiB/s, 23 objects/s recovering 2023-09-23T15:33:51.171 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:51.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:51.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:51.276 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:51.337 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:51.414 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:51.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:51.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:51.521 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:51.552 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:52.027 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:33:52.588 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:52.598 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:33:52.599 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:33:52.599 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:33:52.599 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:33:52.599 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:33:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:52.602 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:52.703 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:52.710 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T15:33:52.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 1.0 2023-09-23T15:33:52.837 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:52.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:52 smithi017 ceph-mon[215646]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:52.860 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:52 smithi017 ceph-mon[215646]: pgmap v1379: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:52.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:52 smithi017 ceph-mon[222878]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:52.860 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:52 smithi017 ceph-mon[222878]: pgmap v1379: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:52.979 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:52.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:52.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:53.018 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:53.085 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:52 smithi027 ceph-mon[195233]: from='client.56704 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:52 smithi027 ceph-mon[195233]: pgmap v1379: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:53.115 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:53.184 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:53.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:53.296 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:53.329 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:53.434 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:33:53.446 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Removed 106108 objects 2023-09-23T15:33:53.446 INFO:tasks.radosbench.radosbench.0.smithi017.stdout:Clean up completed and total clean up time :146.112 2023-09-23T15:33:53.460 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:33:53.483 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:33:53.507 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:33:53.526 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:33:53.545 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:33:53.564 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:33:53.587 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.589 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:33:53.618 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:53.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:53.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:33:53.624 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:53.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:53.753 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:53.798 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:53.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:53.907 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:53.933 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:53 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1419318725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:53.935 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:53.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:53 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1419318725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:53.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:53 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1419318725' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:33:53.985 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:53.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:53.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:54.034 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:33:54.091 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:54.117 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:54.161 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:54.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:54.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:54.265 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:54.293 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:54.338 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:54.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:54.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:54.442 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:54.468 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:54.670 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:54.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:54.673 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:54.774 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:54.793 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:54 smithi017 ceph-mon[215646]: pgmap v1380: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:54.793 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:54 smithi017 ceph-mon[222878]: pgmap v1380: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:54.801 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:54.846 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:54.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:54.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:54.890 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:54.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:54.981 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:55.027 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:55.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:55.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:54 smithi027 ceph-mon[195233]: pgmap v1380: 119 pgs: 119 active+clean; 186 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 782 op/s 2023-09-23T15:33:55.131 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:33:55.159 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:55.209 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:33:55.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:55.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:33:55.270 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T15:33:55.271 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T15:33:55.315 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:55.343 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:55.544 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:33:55.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:55.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:33:55.611 INFO:teuthology.orchestra.run.smithi027.stdout:ERROR: (22) Invalid argument 2023-09-23T15:33:55.611 INFO:teuthology.orchestra.run.smithi027.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:33:55.648 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:55.677 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:55.877 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:55.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:55.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:55.980 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:56.011 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:56.277 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:56.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:56.339 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T15:33:56.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:56.412 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:56.444 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:56.471 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:56.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:56.475 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:56.576 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:56.607 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:56.694 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:33:56.749 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:56.752 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:56.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:56.853 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:33:56.883 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:56.929 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:33:56.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:56.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:33:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:56 smithi017 ceph-mon[215646]: pgmap v1381: 119 pgs: 119 active+clean; 12 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:33:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:56 smithi017 ceph-mon[222878]: pgmap v1381: 119 pgs: 119 active+clean; 12 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:33:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:56.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:33:57.036 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:57.066 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:56 smithi027 ceph-mon[195233]: pgmap v1381: 119 pgs: 119 active+clean; 12 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 682 op/s 2023-09-23T15:33:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:33:57.516 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:57.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:57.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:57.620 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:57.649 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[215646]: failed to encode map e467 with expected crc 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[215646]: failed to encode map e467 with expected crc 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[215646]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[222878]: failed to encode map e467 with expected crc 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[222878]: failed to encode map e467 with expected crc 2023-09-23T15:33:57.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:57.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:57 smithi017 ceph-mon[222878]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T15:33:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:57 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:33:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:57 smithi027 ceph-mon[195233]: failed to encode map e467 with expected crc 2023-09-23T15:33:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:57 smithi027 ceph-mon[195233]: failed to encode map e467 with expected crc 2023-09-23T15:33:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1106753527' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:33:58.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:57 smithi027 ceph-mon[195233]: osdmap e467: 8 total, 8 up, 7 in 2023-09-23T15:33:58.103 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:58.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:58.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:58.207 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:33:58.244 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:58.285 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:58.327 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:33:58.359 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:33:58.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:58.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:33:58.365 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:33:58] "GET /metrics HTTP/1.1" 200 36472 "" "Prometheus/2.43.0" 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:moved 2 / 357 (0.560224%) 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:avg 51 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:stddev 6.09449 -> 6.76123 (expected baseline 6.61168) 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:min osd.2 with 37 -> 36 pgs (0.72549 -> 0.705882 * mean) 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout:max osd.4 with 58 -> 60 pgs (1.13725 -> 1.17647 * mean) 2023-09-23T15:33:58.370 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:oload 1000 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:max_change 1 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:max_change_osds 4 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:average_utilization 0.0064 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:overload_utilization 0.0638 2023-09-23T15:33:58.371 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4 weight 0.9500 -> 1.0000 2023-09-23T15:33:58.373 INFO:teuthology.orchestra.run.smithi017.stderr:no change 2023-09-23T15:33:58.466 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:33:58.497 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:58.693 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:33:58 smithi017 conmon[202315]: 2023-09-23T15:33:58.369+0000 7fdcb8de7700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[215646]: pgmap v1383: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[215646]: failed to encode map e467 with expected crc 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[222878]: pgmap v1383: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[222878]: failed to encode map e467 with expected crc 2023-09-23T15:33:59.078 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:59.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:59.083 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:33:59.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:59.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:33:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:58 smithi027 ceph-mon[195233]: pgmap v1383: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 798 op/s 2023-09-23T15:33:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:58 smithi027 ceph-mon[195233]: failed to encode map e467 with expected crc 2023-09-23T15:33:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:33:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:33:59.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:33:59.221 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:59.272 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:33:59.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:59.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:33:59.377 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:33:59.406 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:59.452 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:33:59.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:33:59.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:33:59.559 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:33:59.590 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:33:59.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:33:59 smithi017 ceph-mon[222878]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:33:59.965 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:33:59 smithi017 ceph-mon[215646]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:34:00.042 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:34:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:33:59 smithi027 ceph-mon[195233]: from='client.46856 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T15:34:00.294 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:00.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:00.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:00.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:00.427 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:00.745 INFO:teuthology.orchestra.run.smithi017.stderr:pool 'unique_pool_5' removed 2023-09-23T15:34:00.814 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:00.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:00.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:00.921 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:00.948 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:00.988 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:00 smithi017 ceph-mon[215646]: pgmap v1384: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T15:34:00.988 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:34:00.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:00 smithi017 ceph-mon[222878]: pgmap v1384: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T15:34:00.988 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:34:00.991 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:00.997 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:00 smithi027 ceph-mon[195233]: pgmap v1384: 119 pgs: 119 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 670 op/s 2023-09-23T15:34:00.997 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T15:34:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:00.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:01.099 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:01.126 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:01.170 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:01.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:01.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:01.200 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:34:01.201 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:34:01.278 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:01.309 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:01.812 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:01.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:01.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:01.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:01.943 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:01 smithi027 ceph-mon[195233]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:01 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:34:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:01 smithi027 ceph-mon[195233]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T15:34:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:01 smithi027 ceph-mon[195233]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:01 smithi027 ceph-mon[195233]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[215646]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[215646]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[215646]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[215646]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[222878]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4113917685' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[222878]: osdmap e468: 8 total, 8 up, 7 in 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[222878]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:01 smithi017 ceph-mon[222878]: failed to encode map e468 with expected crc 2023-09-23T15:34:02.339 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T15:34:02.339 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T15:34:02.339 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T15:34:02.339 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T15:34:02.340 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T15:34:02.340 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:02.391 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:02.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:02.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:34:02.421 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T15:34:02.421 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:02.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T15:34:02.476 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:02.495 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:02.503 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:02.504 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T15:34:02.528 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:02.528 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:34:02.547 INFO:tasks.workunit:timeout=3h 2023-09-23T15:34:02.547 INFO:tasks.workunit:cleanup=True 2023-09-23T15:34:02.547 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T15:34:02.552 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:34:02.571 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:34:02.579 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:02.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:02.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:02.591 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:34:02.610 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:34:02.613 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T15:34:02.630 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:34:02.650 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:34:02.669 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.671 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:34:02.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:02.713 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:02.760 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:02.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:02.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:02.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:02 smithi027 ceph-mon[195233]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:02.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:02 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2422235061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:02.868 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:02.895 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:02.939 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:02.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:02.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:02.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:02 smithi017 ceph-mon[215646]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:02.955 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:02 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2422235061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:02.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:02 smithi017 ceph-mon[222878]: pgmap v1386: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:02.955 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:02 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2422235061' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:03.045 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:03.074 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:03.983 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:03.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:03.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:04.087 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:04.119 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:04.151 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:04.178 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:04.183 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:04.283 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [] live_osds: [0, 2, 3, 4, 5, 1, 6, 7] 2023-09-23T15:34:04.283 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:34:04.284 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:04.316 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:04.368 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:04.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:04.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:04.475 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:04.511 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:04.624 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:04.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:04.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:04.735 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:04.763 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:04.812 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:04.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:04.922 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:04.948 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:04 smithi027 ceph-mon[195233]: pgmap v1387: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:04 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:34:05.153 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:05.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:05.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:04 smithi017 ceph-mon[215646]: pgmap v1387: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:05.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:04 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:34:05.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:04 smithi017 ceph-mon[222878]: pgmap v1387: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:05.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:04 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:34:05.258 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:05.285 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:05.382 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:05.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:05.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:05.404 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:05.491 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:05.527 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:05.581 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:05.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:05.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:05.685 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:05.721 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:05.753 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:34:05.823 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:34:05.823 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:34:05.823 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:34:05.823 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:34:05.823 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:34:05.949 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:34:05.977 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:34:05.978 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:34:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:34:07.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: pgmap v1388: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[222878]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.026 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: pgmap v1388: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.027 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:06 smithi017 ceph-mon[215646]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: pgmap v1388: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3642523834' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: osdmap e469: 8 total, 8 up, 7 in 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:06 smithi027 ceph-mon[195233]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.389 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:07.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:07.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:07.494 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:07.523 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:07.575 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:07.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:07.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:07.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:07.712 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:07 smithi027 ceph-mon[195233]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:07 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:07.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:07 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:07.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[215646]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:07.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[222878]: failed to encode map e469 with expected crc 2023-09-23T15:34:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:07 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:08.275 INFO:tasks.workunit.client.0.smithi017.stderr:Updating files: 87% (10067/11502) Updating files: 88% (10122/11502) Updating files: 89% (10237/11502) Updating files: 90% (10352/11502) Updating files: 91% (10467/11502) Updating files: 92% (10582/11502) Updating files: 93% (10697/11502) Updating files: 94% (10812/11502) Updating files: 95% (10927/11502) Updating files: 96% (11042/11502) Updating files: 97% (11157/11502) Updating files: 98% (11272/11502) Updating files: 99% (11387/11502) Updating files: 100% (11502/11502) Updating files: 100% (11502/11502), done. 2023-09-23T15:34:08.291 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T15:34:08.350 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T15:34:08.352 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T15:34:08.352 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T15:34:08.442 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:08] "GET /metrics HTTP/1.1" 200 36448 "" "Prometheus/2.43.0" 2023-09-23T15:34:08.680 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T15:34:08.761 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T15:34:08.852 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T15:34:08.853 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T15:34:08.854 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T15:34:09.002 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T15:34:09.083 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:08 smithi027 ceph-mon[195233]: pgmap v1390: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:09.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:08 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.084 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:08 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[215646]: pgmap v1390: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[222878]: pgmap v1390: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:08 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:09.453 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T15:34:09.453 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T15:34:09.745 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T15:34:09.747 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T15:34:09.747 DEBUG:teuthology.orchestra.run.smithi017:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T15:34:09.824 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:09.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:09.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:09.929 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:09.937 INFO:tasks.workunit.client.0.smithi017.stdout:Running main() from gmock_main.cc 2023-09-23T15:34:09.937 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T15:34:09.938 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment set-up. 2023-09-23T15:34:09.938 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T15:34:09.964 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:10.012 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T15:34:10.036 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:10.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:10.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:10.078 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:10.142 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:10.182 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:10.519 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:10.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:10.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:10.658 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:10.706 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:10.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:10.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:10.750 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:10 smithi017 ceph-mon[215646]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:10.811 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:10.842 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:10 smithi027 ceph-mon[195233]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:11.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:10 smithi017 ceph-mon[222878]: pgmap v1391: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:11.642 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:11.678 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:11.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:11.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:11.772 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T15:34:11.775 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:34:11.786 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:11.817 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:11.842 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:34:11.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:11.910 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:11.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:11.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: failed to encode map e470 with expected crc 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: failed to encode map e470 with expected crc 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3337115689' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4116966301' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:11.945 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: failed to encode map e470 with expected crc 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: failed to encode map e470 with expected crc 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3337115689' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4116966301' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:11.946 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:11.947 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_all_features (174 ms) 2023-09-23T15:34:11.947 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T15:34:12.011 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:12.050 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: failed to encode map e470 with expected crc 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: failed to encode map e470 with expected crc 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: osdmap e470: 8 total, 8 up, 7 in 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3337115689' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4116966301' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:34:12.103 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:12.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:12.142 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.copyup (195 ms) 2023-09-23T15:34:12.142 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T15:34:12.160 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparse_copyup (18 ms) 2023-09-23T15:34:12.160 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T15:34:12.167 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_and_set_id (8 ms) 2023-09-23T15:34:12.167 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T15:34:12.180 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2023-09-23T15:34:12.180 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T15:34:12.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:12.215 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.directory_methods (36 ms) 2023-09-23T15:34:12.215 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.create 2023-09-23T15:34:12.236 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.create (21 ms) 2023-09-23T15:34:12.236 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T15:34:12.237 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:12.238 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2023-09-23T15:34:12.238 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T15:34:12.240 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_object_prefix (3 ms) 2023-09-23T15:34:12.240 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T15:34:12.242 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_create_timestamp (1 ms) 2023-09-23T15:34:12.242 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T15:34:12.244 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_access_timestamp (3 ms) 2023-09-23T15:34:12.244 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T15:34:12.246 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-09-23T15:34:12.246 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T15:34:12.250 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_data_pool (4 ms) 2023-09-23T15:34:12.250 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T15:34:12.254 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.get_size (5 ms) 2023-09-23T15:34:12.254 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T15:34:12.260 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2023-09-23T15:34:12.260 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T15:34:12.278 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2023-09-23T15:34:12.279 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T15:34:12.286 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:12.287 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshot_limits (9 ms) 2023-09-23T15:34:12.287 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T15:34:12.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:12.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:12.338 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v1 (50 ms) 2023-09-23T15:34:12.338 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T15:34:12.367 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.parents_v2 (30 ms) 2023-09-23T15:34:12.368 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T15:34:12.389 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapshots (22 ms) 2023-09-23T15:34:12.390 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T15:34:12.392 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:12.395 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.snapid_race (5 ms) 2023-09-23T15:34:12.395 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T15:34:12.411 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.stripingv2 (16 ms) 2023-09-23T15:34:12.411 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T15:34:12.412 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_save (2 ms) 2023-09-23T15:34:12.412 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T15:34:12.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_resize (9 ms) 2023-09-23T15:34:12.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T15:34:12.427 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_update (5 ms) 2023-09-23T15:34:12.427 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T15:34:12.427 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T15:34:12.429 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T15:34:12.429 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:12.433 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2023-09-23T15:34:12.433 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T15:34:12.438 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.object_map_snap_remove (4 ms) 2023-09-23T15:34:12.438 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T15:34:12.443 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T15:34:12.444 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T15:34:12.468 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.metadata (25 ms) 2023-09-23T15:34:12.468 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T15:34:12.475 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.set_features (6 ms) 2023-09-23T15:34:12.475 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T15:34:12.569 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror (94 ms) 2023-09-23T15:34:12.569 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T15:34:12.584 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image (16 ms) 2023-09-23T15:34:12.584 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T15:34:13.022 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: pgmap v1393: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: pgmap v1393: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:13.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.026 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:12 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:13.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: pgmap v1393: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi017-662893-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: osdmap e471: 8 total, 8 up, 7 in 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: failed to encode map e471 with expected crc 2023-09-23T15:34:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:12 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:13.128 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:13.164 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:13.960 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:13.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:13.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.041 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.042 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:14.065 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: osdmap e472: 8 total, 8 up, 7 in 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: failed to encode map e472 with expected crc 2023-09-23T15:34:14.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:14.098 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:14.114 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:34:14.151 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:14.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:14.256 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:14.281 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:14.285 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:14.331 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:14.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:14.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:14.378 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:34:14.407 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:34:14.435 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:34:14.437 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:14.464 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:14.468 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:34:14.497 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:34:14.508 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:14.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:14.522 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:34:14.551 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:34:14.578 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:34:14.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.580 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.581 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:34:14.614 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:14.641 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:14.684 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:14.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:14.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:14.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:14.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:14 smithi017 ceph-mon[215646]: pgmap v1396: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:14.802 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:14 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1715501968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:14.802 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:14 smithi017 ceph-mon[222878]: pgmap v1396: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:14.803 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:14 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1715501968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:14.817 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:14.860 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:14.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:14.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:14.967 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:14.994 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:15.034 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:34:15.034 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:34:15.037 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:15.042 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:14 smithi027 ceph-mon[195233]: pgmap v1396: 135 pgs: 5 creating+peering, 27 unknown, 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:15.042 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:14 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1715501968' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:15.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:15.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:15.144 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:15.170 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:15.214 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:15.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:15.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:15.319 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:15.359 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:16.131 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_status (3546 ms) 2023-09-23T15:34:16.131 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T15:34:16.324 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_image_map (193 ms) 2023-09-23T15:34:16.324 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T15:34:16.388 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_instances (65 ms) 2023-09-23T15:34:16.388 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T15:34:16.402 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.mirror_snapshot (14 ms) 2023-09-23T15:34:16.402 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T15:34:16.405 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-23T15:34:16.405 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T15:34:16.407 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2023-09-23T15:34:16.407 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T15:34:16.411 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T15:34:16.411 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T15:34:16.418 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_rename (7 ms) 2023-09-23T15:34:16.418 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T15:34:16.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-09-23T15:34:16.421 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T15:34:16.425 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T15:34:16.425 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T15:34:16.428 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_add (3 ms) 2023-09-23T15:34:16.428 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T15:34:16.432 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T15:34:16.432 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T15:34:16.435 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_list (4 ms) 2023-09-23T15:34:16.435 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T15:34:16.440 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-09-23T15:34:16.440 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T15:34:16.442 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-09-23T15:34:16.442 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T15:34:16.446 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_remove (4 ms) 2023-09-23T15:34:16.446 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T15:34:16.449 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2023-09-23T15:34:16.449 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T15:34:16.450 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T15:34:16.450 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T15:34:16.452 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T15:34:16.452 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T15:34:16.455 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (2 ms) 2023-09-23T15:34:16.455 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T15:34:16.458 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T15:34:16.458 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T15:34:16.460 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_set (3 ms) 2023-09-23T15:34:16.460 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T15:34:16.463 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T15:34:16.463 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T15:34:16.481 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2023-09-23T15:34:16.481 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T15:34:16.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T15:34:16.687 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (206 ms) 2023-09-23T15:34:16.687 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T15:34:16.691 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T15:34:16.691 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T15:34:16.698 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (6 ms) 2023-09-23T15:34:16.698 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T15:34:16.705 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.trash_methods (8 ms) 2023-09-23T15:34:16.705 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T15:34:16.712 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.op_features (7 ms) 2023-09-23T15:34:16.713 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T15:34:16.723 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:16.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:16.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:16.827 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:16.847 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent (135 ms) 2023-09-23T15:34:16.847 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T15:34:16.856 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:16.857 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_parent_ns (9 ms) 2023-09-23T15:34:16.857 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T15:34:16.870 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.clone_child (13 ms) 2023-09-23T15:34:16.870 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T15:34:16.877 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.namespace_methods (7 ms) 2023-09-23T15:34:16.877 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T15:34:16.887 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T15:34:16.887 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T15:34:16.897 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.migration_v1 (10 ms) 2023-09-23T15:34:16.897 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T15:34:16.906 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:16.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:16.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:16.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:16 smithi017 ceph-mon[215646]: pgmap v1397: 135 pgs: 5 creating+peering, 130 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.8 MiB/s rd, 973 KiB/s wr, 760 op/s 2023-09-23T15:34:16.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:16 smithi017 ceph-mon[222878]: pgmap v1397: 135 pgs: 5 creating+peering, 130 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.8 MiB/s rd, 973 KiB/s wr, 760 op/s 2023-09-23T15:34:17.013 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:17.041 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:16 smithi027 ceph-mon[195233]: pgmap v1397: 135 pgs: 5 creating+peering, 130 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.8 MiB/s rd, 973 KiB/s wr, 760 op/s 2023-09-23T15:34:17.573 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:17.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:17.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:17.677 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:17.707 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:17.755 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:17.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:17.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:17.861 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:17.893 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:17.945 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:17.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:17.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:17.982 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:17.983 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:18.049 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:18.085 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:18.136 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:18.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:18.140 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:18.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:18] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T15:34:18.241 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:18.270 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:18.373 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:18.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:18.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:18.478 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:18.512 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:18.787 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1889 ms) 2023-09-23T15:34:18.787 INFO:tasks.workunit.client.0.smithi017.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T15:34:18.814 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: pgmap v1398: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 935 KiB/s wr, 776 op/s 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.815 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/115593219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: pgmap v1398: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 935 KiB/s wr, 776 op/s 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: failed to encode map e473 with expected crc 2023-09-23T15:34:18.816 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T15:34:18.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:34:18.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:34:18.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:18.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/115593219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:18.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:18 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:18.821 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:34:18.852 INFO:tasks.workunit.client.0.smithi017.stdout:[ OK ] TestClsRbd.sparsify (66 ms) 2023-09-23T15:34:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: pgmap v1398: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 935 KiB/s wr, 776 op/s 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: failed to encode map e473 with expected crc 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: failed to encode map e473 with expected crc 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: failed to encode map e473 with expected crc 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: osdmap e473: 8 total, 8 up, 7 in 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/115593219' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:18 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:19.617 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:19.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:19.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:34:19.721 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:19.749 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:19.805 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] 70 tests from TestClsRbd (7080 ms total) 2023-09-23T15:34:19.805 INFO:tasks.workunit.client.0.smithi017.stdout: 2023-09-23T15:34:19.805 INFO:tasks.workunit.client.0.smithi017.stdout:[----------] Global test environment tear-down 2023-09-23T15:34:19.805 INFO:tasks.workunit.client.0.smithi017.stdout:[==========] 70 tests from 1 test suite ran. (9867 ms total) 2023-09-23T15:34:19.805 INFO:tasks.workunit.client.0.smithi017.stdout:[ PASSED ] 70 tests. 2023-09-23T15:34:19.810 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T15:34:19.810 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T15:34:19.862 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:19.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:19.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[215646]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.883 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.884 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:19 smithi017 ceph-mon[222878]: failed to encode map e474 with expected crc 2023-09-23T15:34:19.885 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T15:34:19.885 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T15:34:19.967 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:19.996 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: failed to encode map e474 with expected crc 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: failed to encode map e474 with expected crc 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: osdmap e474: 8 total, 8 up, 7 in 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: failed to encode map e474 with expected crc 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: failed to encode map e474 with expected crc 2023-09-23T15:34:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:19 smithi027 ceph-mon[195233]: failed to encode map e474 with expected crc 2023-09-23T15:34:20.197 DEBUG:teuthology.parallel:result is None 2023-09-23T15:34:20.198 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:20.268 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:20.268 DEBUG:teuthology.orchestra.run.smithi017:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:20.327 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:20.327 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T15:34:20.328 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T15:34:20.328 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T15:34:20.328 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T15:34:20.328 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T15:34:20.329 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:34:20.385 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T15:34:20.386 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:20.417 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:20.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:20.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:20.437 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:20.437 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T15:34:20.495 INFO:tasks.workunit:timeout=3h 2023-09-23T15:34:20.496 INFO:tasks.workunit:cleanup=True 2023-09-23T15:34:20.496 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T15:34:20.523 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:20.590 INFO:tasks.workunit.client.0.smithi017.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T15:34:20.602 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:20.919 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:20.923 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:20.951 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:21.024 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:21.054 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:34:21.054 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:34:21.054 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: pgmap v1401: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 936 KiB/s wr, 776 op/s 2023-09-23T15:34:21.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T15:34:21.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.055 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:20 smithi027 ceph-mon[195233]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.057 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: pgmap v1401: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 936 KiB/s wr, 776 op/s 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[215646]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: pgmap v1401: 135 pgs: 135 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.7 MiB/s rd, 936 KiB/s wr, 776 op/s 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: osdmap e475: 8 total, 8 up, 7 in 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.081 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:20 smithi017 ceph-mon[222878]: failed to encode map e475 with expected crc 2023-09-23T15:34:21.109 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:21.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:21.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:21.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:21.251 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:21.302 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:21.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:21.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:21.406 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:21.434 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:21.803 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:34:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/469843699' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:21 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/469843699' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:21 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/469843699' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:21 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:34:22.532 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T15:34:22.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T15:34:22.887 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:22.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:22.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:22.991 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:23.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: pgmap v1403: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[215646]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: pgmap v1403: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:34:23.015 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:22 smithi017 ceph-mon[222878]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T15:34:23.032 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:23.038 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:34:23.083 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:23.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:23.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: pgmap v1403: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: failed to encode map e476 with expected crc 2023-09-23T15:34:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:34:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:22 smithi027 ceph-mon[195233]: osdmap e476: 8 total, 8 up, 7 in 2023-09-23T15:34:23.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:23.219 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:23.271 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:23.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:23.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:23.375 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:23.408 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:23.459 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:23.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:23.566 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T15:34:23.567 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:23.584 INFO:tasks.workunit.client.0.smithi017.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T15:34:23.586 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T15:34:23.586 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T15:34:23.594 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:24.188 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T15:34:24.394 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T15:34:24.444 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T15:34:24.446 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T15:34:24.446 INFO:tasks.workunit.client.0.smithi017.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T15:34:24.731 INFO:tasks.workunit.client.0.smithi017.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T15:34:24.735 DEBUG:teuthology.orchestra.run.smithi017:> set -ex 2023-09-23T15:34:24.735 DEBUG:teuthology.orchestra.run.smithi017:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T15:34:24.795 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T15:34:24.796 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T15:34:24.797 DEBUG:teuthology.orchestra.run.smithi017:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T15:34:24.859 INFO:tasks.workunit.client.0.smithi017.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T15:34:24.859 INFO:tasks.workunit.client.0.smithi017.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T15:34:24.860 INFO:tasks.workunit.client.0.smithi017.stderr:+ TMPDIR=/tmp/rbd_import_export_667636 2023-09-23T15:34:24.860 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_667636 2023-09-23T15:34:24.860 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir /tmp/rbd_import_export_667636 2023-09-23T15:34:24.861 INFO:tasks.workunit.client.0.smithi017.stderr:+ trap 'rm -rf /tmp/rbd_import_export_667636' INT TERM EXIT 2023-09-23T15:34:24.861 INFO:tasks.workunit.client.0.smithi017.stderr:+ mkdir foo.667636 2023-09-23T15:34:24.862 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import foo.667636 foo.dir 2023-09-23T15:34:25.054 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: cannot import a directory 2023-09-23T15:34:25.054 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 0% complete...failed. 2023-09-23T15:34:25.054 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: import failed: (21) Is a directory 2023-09-23T15:34:25.057 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:25.057 INFO:tasks.workunit.client.0.smithi017.stderr:+ rmdir foo.667636 2023-09-23T15:34:25.058 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_667636/img bs=1k count=1 seek=10 2023-09-23T15:34:25.058 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T15:34:25.059 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T15:34:25.059 INFO:tasks.workunit.client.0.smithi017.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 5.2149e-05 s, 19.6 MB/s 2023-09-23T15:34:25.059 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_667636/img bs=1k count=10 seek=100 2023-09-23T15:34:25.059 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2023-09-23T15:34:25.059 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2023-09-23T15:34:25.060 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.2623e-05 s, 164 MB/s 2023-09-23T15:34:25.060 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_667636/img bs=1k count=100 seek=1000 2023-09-23T15:34:25.060 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records in 2023-09-23T15:34:25.060 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records out 2023-09-23T15:34:25.060 INFO:tasks.workunit.client.0.smithi017.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000210716 s, 342 MB/s 2023-09-23T15:34:25.061 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_667636/img bs=1k seek=10000 2023-09-23T15:34:25.061 INFO:tasks.workunit.client.0.smithi017.stderr:139+1 records in 2023-09-23T15:34:25.061 INFO:tasks.workunit.client.0.smithi017.stderr:139+1 records out 2023-09-23T15:34:25.062 INFO:tasks.workunit.client.0.smithi017.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000427352 s, 335 MB/s 2023-09-23T15:34:25.062 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_667636/img bs=1k seek=100000 2023-09-23T15:34:25.062 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records in 2023-09-23T15:34:25.062 INFO:tasks.workunit.client.0.smithi017.stderr:70+1 records out 2023-09-23T15:34:25.063 INFO:tasks.workunit.client.0.smithi017.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000237764 s, 304 MB/s 2023-09-23T15:34:25.063 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_667636/img bs=1k seek=1000000 2023-09-23T15:34:25.064 INFO:tasks.workunit.client.0.smithi017.stderr:189+1 records in 2023-09-23T15:34:25.064 INFO:tasks.workunit.client.0.smithi017.stderr:189+1 records out 2023-09-23T15:34:25.064 INFO:tasks.workunit.client.0.smithi017.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000539925 s, 359 MB/s 2023-09-23T15:34:25.064 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T15:34:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:24 smithi027 ceph-mon[195233]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:25.102 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T15:34:25.102 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T15:34:25.103 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:25.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:24 smithi017 ceph-mon[215646]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:25.105 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:25.105 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:24 smithi017 ceph-mon[222878]: pgmap v1405: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:25.106 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:25.106 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:25.106 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:25.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:25.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:25.208 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:25.241 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:25.277 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:25.292 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:25.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:25.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:25.331 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:25.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:25.431 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:25.440 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:25.573 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:25.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:25.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:25.677 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:25.715 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:25.755 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T15:34:25.757 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:25.759 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_667636/img2 2023-09-23T15:34:25.891 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:25.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:25.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:25.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:26.029 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:26.057 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:26.062 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2023-09-23T15:34:26.081 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:26.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:26.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:26.182 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:34:26.189 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:26.205 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:34:26.227 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:26.233 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:34:26.260 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:34:26.285 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:34:26.308 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:34:26.333 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:34:26.356 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.358 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.359 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:34:26.419 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:26.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:26.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:26.524 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:26.620 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:26.853 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:26.857 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T15:34:26.972 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T15:34:26.974 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img2 2023-09-23T15:34:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:26 smithi027 ceph-mon[195233]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:26 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3319486047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:27.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:26 smithi017 ceph-mon[215646]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:27.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:26 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3319486047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:27.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:26 smithi017 ceph-mon[222878]: pgmap v1406: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:27.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:26 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3319486047' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:27.211 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:27.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:27.215 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:27.316 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:27.576 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:27.818 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img3 2023-09-23T15:34:27.859 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:27.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:27.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:27.963 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:27.991 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:28.041 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:28.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:28.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:28.145 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:28.180 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:28.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:28] "GET /metrics HTTP/1.1" 200 36397 "" "Prometheus/2.43.0" 2023-09-23T15:34:28.234 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:28.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:28.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:28.346 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img2 /tmp/rbd_import_export_667636/img3 2023-09-23T15:34:28.368 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:28.415 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:28.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:28.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:28.522 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:28.550 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:28 smithi027 ceph-mon[195233]: pgmap v1407: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:29.153 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:29.155 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[222878]: pgmap v1407: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:29.155 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:29.156 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:29.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[215646]: pgmap v1407: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:34:29.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:29.156 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:29.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:29.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:29.258 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:29.290 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:29.340 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:29.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:29.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:29.444 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:29.473 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:29.521 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:29.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:29.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:29.624 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:29.657 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:29.708 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:29.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:29.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:29.816 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:29.844 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:29.891 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:29.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:29.996 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:30.024 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.068 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:30.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:30.173 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:30.200 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.244 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:30.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:30.349 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:30.375 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.420 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:30.527 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:30.560 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.613 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:30.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:30.717 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:30.745 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.790 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:30.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.796 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:30.897 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:30.932 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:30.981 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:30.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:30.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:31.085 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:30 smithi027 ceph-mon[195233]: pgmap v1408: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 40 KiB/s rd, 54 KiB/s wr, 11 op/s 2023-09-23T15:34:31.112 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:30 smithi017 ceph-mon[215646]: pgmap v1408: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 40 KiB/s rd, 54 KiB/s wr, 11 op/s 2023-09-23T15:34:31.112 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:30 smithi017 ceph-mon[222878]: pgmap v1408: 103 pgs: 103 active+clean; 10 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 40 KiB/s rd, 54 KiB/s wr, 11 op/s 2023-09-23T15:34:31.114 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:31.162 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:31.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:31.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:31.266 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:31.296 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:31.341 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:31.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:31.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:31.445 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:31.478 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:31.530 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:31.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:31.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:31.634 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:31.663 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:31.710 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:31.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:31.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:31.815 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:31.848 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:31.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:31.928 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:31.928 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:32.029 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:32.056 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:32.102 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:32.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:32.108 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:32.209 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:32.235 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:32.280 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:32.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:32.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:32.385 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:32.413 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:32.456 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:32.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:32.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:32.560 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:32.594 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:32.647 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:32.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:32.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:32.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:32.782 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:32.833 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:32.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:32.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:32.937 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:32.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:32 smithi017 ceph-mon[215646]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T15:34:32.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:32 smithi017 ceph-mon[222878]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T15:34:32.965 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.011 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:33.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:32 smithi027 ceph-mon[195233]: pgmap v1409: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 119 KiB/s rd, 51 KiB/s wr, 30 op/s 2023-09-23T15:34:33.119 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:33.146 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.189 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:33.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:33.296 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:33.330 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.382 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:33.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:33.486 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:33.515 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.559 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:33.668 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:33.702 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.754 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:33.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:33.858 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:33.890 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:33.943 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:33.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:33.988 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - testimg 2023-09-23T15:34:34.048 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:34.080 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:34.134 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:34.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:34.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:34.239 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:34.271 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:34.322 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:34.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:34.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:34.395 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:34.395 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:34.398 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg /tmp/rbd_import_export_667636/img2 2023-09-23T15:34:34.427 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:34.454 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:34.711 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:34.714 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg - 2023-09-23T15:34:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:34 smithi017 ceph-mon[215646]: pgmap v1410: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 106 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-09-23T15:34:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:34 smithi017 ceph-mon[222878]: pgmap v1410: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 106 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-09-23T15:34:35.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:34 smithi027 ceph-mon[195233]: pgmap v1410: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 106 KiB/s rd, 45 KiB/s wr, 27 op/s 2023-09-23T15:34:35.404 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:35.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:35.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:35.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:35.535 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:35.538 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T15:34:35.540 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:35.590 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:35.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:35.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:35.635 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T15:34:35.638 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img2 2023-09-23T15:34:35.694 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:35.727 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:35.777 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:35.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:35.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:35.915 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:36.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:35 smithi027 ceph-mon[195233]: pgmap v1411: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 128 KiB/s rd, 69 KiB/s wr, 33 op/s 2023-09-23T15:34:36.113 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:36.115 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:35 smithi017 ceph-mon[215646]: pgmap v1411: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 128 KiB/s rd, 69 KiB/s wr, 33 op/s 2023-09-23T15:34:36.115 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:35 smithi017 ceph-mon[222878]: pgmap v1411: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 128 KiB/s rd, 69 KiB/s wr, 33 op/s 2023-09-23T15:34:36.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:36.116 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:36.217 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:36.245 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:36.295 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:36.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:36.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:36.325 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img3 2023-09-23T15:34:36.398 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:36.426 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:36.474 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:36.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:36.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:36.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:36.606 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:36.655 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:36.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:36.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:36.759 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:36.792 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:36.842 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:36.846 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img2 /tmp/rbd_import_export_667636/img3 2023-09-23T15:34:36.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:36.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:36.949 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:36.979 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.026 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:37.133 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:37.161 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.205 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:37.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:37.313 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:37.345 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.397 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:37.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.400 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:37.501 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:37.531 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.580 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:37.685 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:37.713 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.758 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:37.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:37.867 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:37.900 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:37.952 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:37.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:38.056 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:38.085 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:38.131 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:38.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:38.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:38.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:38] "GET /metrics HTTP/1.1" 200 34030 "" "Prometheus/2.43.0" 2023-09-23T15:34:38.238 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:38.272 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:38.325 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:38.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:38.328 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:38.428 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:38.459 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:38.509 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:38.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:38.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:38.613 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:38.642 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:38.691 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:38.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:38.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:38.794 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:38.826 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:38.875 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:38.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:38.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:38.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:38 smithi017 ceph-mon[215646]: pgmap v1412: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 184 KiB/s rd, 86 KiB/s wr, 62 op/s 2023-09-23T15:34:38.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:38 smithi017 ceph-mon[222878]: pgmap v1412: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 184 KiB/s rd, 86 KiB/s wr, 62 op/s 2023-09-23T15:34:38.984 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:39.012 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.058 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:39.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:38 smithi027 ceph-mon[195233]: pgmap v1412: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 184 KiB/s rd, 86 KiB/s wr, 62 op/s 2023-09-23T15:34:39.162 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:39.190 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.236 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:39.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:39.339 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:39.372 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.424 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:39.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:39.530 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:39.561 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.609 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:39.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:39.713 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:39.741 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.788 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:39.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:39.891 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:39.918 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:39.963 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:39.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:39.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:40.066 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:40.099 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:40.151 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:40.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:40.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:40.259 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:40.287 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:40.333 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:40.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:40.439 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:40.465 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:40.509 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:40.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:40.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:40.615 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:40.642 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:40.686 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:40.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:40.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:40.792 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:40.816 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:40 smithi027 ceph-mon[195233]: pgmap v1413: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 152 KiB/s rd, 44 KiB/s wr, 53 op/s 2023-09-23T15:34:40.818 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:40.863 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:40.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:40.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:40.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:40 smithi017 ceph-mon[215646]: pgmap v1413: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 152 KiB/s rd, 44 KiB/s wr, 53 op/s 2023-09-23T15:34:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:40 smithi017 ceph-mon[222878]: pgmap v1413: 103 pgs: 103 active+clean; 9.2 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 152 KiB/s rd, 44 KiB/s wr, 53 op/s 2023-09-23T15:34:40.969 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:41.003 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.056 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:41.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:41.159 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:41.187 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.231 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:41.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:41.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:41.365 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.409 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:41.515 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:41.542 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.569 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd help export 2023-09-23T15:34:41.569 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -q export-format 2023-09-23T15:34:41.586 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:41.590 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_667636/img bs=1k count=10 seek=100 2023-09-23T15:34:41.591 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records in 2023-09-23T15:34:41.591 INFO:tasks.workunit.client.0.smithi017.stderr:10+0 records out 2023-09-23T15:34:41.591 INFO:tasks.workunit.client.0.smithi017.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2571e-05 s, 141 MB/s 2023-09-23T15:34:41.591 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:41.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:41.651 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:41.651 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:41.654 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap 2023-09-23T15:34:41.695 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:41.722 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.767 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:41.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:41.871 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:41.899 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:41.943 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:41.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:41.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:42.048 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:42.074 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:42.118 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:42.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:42.123 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:42.223 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:42.250 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:42.293 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:42.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:42.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:42.431 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:42.661 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T15:34:42.668 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T15:34:42.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:42 smithi017 ceph-mon[215646]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 180 KiB/s rd, 45 KiB/s wr, 57 op/s 2023-09-23T15:34:42.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:42 smithi017 ceph-mon[222878]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 180 KiB/s rd, 45 KiB/s wr, 57 op/s 2023-09-23T15:34:42.717 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg 2023-09-23T15:34:42.757 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T15:34:42.757 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T15:34:42.758 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T15:34:42.758 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2023-09-23T15:34:42.758 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:42.782 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:42.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:42.798 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:42.803 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667636/img_v2 testimg_import 2023-09-23T15:34:42.886 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:42.916 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:43.037 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:34:43.037 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:34:43.037 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:34:43.037 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:34:43.037 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:34:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:42 smithi027 ceph-mon[195233]: pgmap v1414: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 180 KiB/s rd, 45 KiB/s wr, 57 op/s 2023-09-23T15:34:43.222 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:43.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:43.327 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:43.357 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:43.405 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:43.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:43.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:43.511 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:43.540 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:43.585 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:43.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:43.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:43.660 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:34:43.661 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/281486321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:43.664 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:34:43.666 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is set 2023-09-23T15:34:43.676 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T15:34:43.680 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T15:34:43.690 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout: id: b79e7937c666 2023-09-23T15:34:43.716 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.b79e7937c666 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.717 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.717 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/281486321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:43.718 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: osdmap e477: 8 total, 8 up, 7 in 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: failed to encode map e477 with expected crc 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/281486321' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:43.719 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:43 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:43.720 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap 2023-09-23T15:34:43.720 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout:rbd image 'testimg_import': 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: size 110 KiB in 1 objects 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: order 22 (4 MiB objects) 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: snapshot_count: 1 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: id: b79e7937c666 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: block_name_prefix: rbd_data.b79e7937c666 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: format: 2 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: op_features: 2023-09-23T15:34:43.757 INFO:tasks.workunit.client.0.smithi017.stdout: flags: 2023-09-23T15:34:43.758 INFO:tasks.workunit.client.0.smithi017.stdout: create_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.758 INFO:tasks.workunit.client.0.smithi017.stdout: access_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.758 INFO:tasks.workunit.client.0.smithi017.stdout: modify_timestamp: Sat Sep 23 15:34:42 2023 2023-09-23T15:34:43.758 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2023-09-23T15:34:43.760 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd image-meta list testimg_import 2023-09-23T15:34:43.798 INFO:tasks.workunit.client.0.smithi017.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T15:34:43.798 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T15:34:43.798 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr: 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:Key Value 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:key1 value1' ']' 2023-09-23T15:34:43.799 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import /tmp/rbd_import_export_667636/img_import 2023-09-23T15:34:43.872 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T15:34:43.876 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img_import 2023-09-23T15:34:43.876 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/img /tmp/rbd_import_export_667636/img_import 2023-09-23T15:34:43.877 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/img --format %b 2023-09-23T15:34:43.878 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=24 2023-09-23T15:34:43.878 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/img_import --format %b 2023-09-23T15:34:43.879 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2023-09-23T15:34:43.879 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2023-09-23T15:34:43.879 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=200 2023-09-23T15:34:43.880 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T15:34:43.880 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_667636/img_snap 2023-09-23T15:34:43.913 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T15:34:43.916 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_667636/img_snap_import 2023-09-23T15:34:43.958 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T15:34:43.961 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/img_snap /tmp/rbd_import_export_667636/img_snap_import 2023-09-23T15:34:43.962 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/img_snap /tmp/rbd_import_export_667636/img_snap_import 2023-09-23T15:34:43.962 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/img_snap --format %b 2023-09-23T15:34:43.964 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=224 2023-09-23T15:34:43.964 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/img_snap_import --format %b 2023-09-23T15:34:43.965 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=224 2023-09-23T15:34:43.965 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:43.966 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:43.966 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T15:34:43.966 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:43.966 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_import 2023-09-23T15:34:43.967 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_snap 2023-09-23T15:34:43.967 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_snap_import 2023-09-23T15:34:43.968 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T15:34:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:43 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T15:34:44.148 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T15:34:44.502 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T15:34:44.622 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:44.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:44.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:44.675 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2023-09-23T15:34:44.688 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T15:34:44.726 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:44.759 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:44.780 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:44.783 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap rm testimg@snap 2023-09-23T15:34:44.812 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:44.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:44.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:44.915 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:44.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T15:34:44.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:44.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[215646]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:34:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:34:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T15:34:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:44 smithi017 ceph-mon[222878]: failed to encode map e478 with expected crc 2023-09-23T15:34:44.946 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: pgmap v1416: 103 pgs: 103 active+clean; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 134 KiB/s rd, 53 KiB/s wr, 49 op/s 2023-09-23T15:34:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: failed to encode map e478 with expected crc 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/873351797' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: osdmap e478: 8 total, 8 up, 7 in 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: failed to encode map e478 with expected crc 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: failed to encode map e478 with expected crc 2023-09-23T15:34:45.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:44 smithi027 ceph-mon[195233]: failed to encode map e478 with expected crc 2023-09-23T15:34:45.187 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:34:45.233 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:45.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:45.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:45.338 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:45.356 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:45.371 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:45.375 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:45.422 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:45.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:45.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:45.528 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:45.556 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:45.603 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:45.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:45.676 INFO:tasks.workunit.client.0.smithi017.stderr: Removing snap: 100% complete...done. 2023-09-23T15:34:45.686 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm testimg 2023-09-23T15:34:45.711 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:45.745 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:45.793 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:45.796 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --order 20 /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:45.853 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:45.854 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:45.856 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:45.908 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:45.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667636/img_v2 testimg_import 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[215646]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[215646]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[215646]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[215646]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[215646]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.913 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[222878]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[222878]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T15:34:45.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[222878]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[222878]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.914 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:45 smithi017 ceph-mon[222878]: failed to encode map e479 with expected crc 2023-09-23T15:34:45.974 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T15:34:45.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T15:34:45.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep order 2023-09-23T15:34:45.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $2}' 2023-09-23T15:34:45.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 20 2023-09-23T15:34:46.015 INFO:tasks.workunit.client.0.smithi017.stdout:20 2023-09-23T15:34:46.015 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:46.015 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T15:34:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:45 smithi027 ceph-mon[195233]: failed to encode map e479 with expected crc 2023-09-23T15:34:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:45 smithi027 ceph-mon[195233]: osdmap e479: 8 total, 8 up, 7 in 2023-09-23T15:34:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:45 smithi027 ceph-mon[195233]: failed to encode map e479 with expected crc 2023-09-23T15:34:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:45 smithi027 ceph-mon[195233]: failed to encode map e479 with expected crc 2023-09-23T15:34:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:45 smithi027 ceph-mon[195233]: failed to encode map e479 with expected crc 2023-09-23T15:34:46.104 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:46.107 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T15:34:46.192 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:46.195 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:46.240 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:46.243 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg 2023-09-23T15:34:46.243 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2023-09-23T15:34:46.309 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:46.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:46.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:46.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:46.413 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T15:34:46.413 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2023-09-23T15:34:46.413 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:46.443 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:46.452 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:46.455 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667636/img_v2 testimg_import 2023-09-23T15:34:46.512 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T15:34:46.516 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info testimg_import 2023-09-23T15:34:46.516 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep features 2023-09-23T15:34:46.550 INFO:tasks.workunit.client.0.smithi017.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T15:34:46.550 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' 2023-09-23T15:34:46.551 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' ' features: layering 2023-09-23T15:34:46.551 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' '!=' ' features: layering 2023-09-23T15:34:46.551 INFO:tasks.workunit.client.0.smithi017.stderr: op_features: ' ']' 2023-09-23T15:34:46.551 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:46.551 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T15:34:46.616 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:46.619 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T15:34:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[222878]: pgmap v1419: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2023-09-23T15:34:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[222878]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[222878]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[222878]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T15:34:46.666 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[222878]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.690 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:46.693 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:46.754 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:46.757 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:46.795 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:46.798 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667636/img_v2 testimg_import 2023-09-23T15:34:46.891 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T15:34:46.894 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T15:34:46.894 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe unit' 2023-09-23T15:34:46.894 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T15:34:46.929 INFO:tasks.workunit.client.0.smithi017.stdout: stripe unit: 4 KiB 2023-09-23T15:34:46.930 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import 2023-09-23T15:34:46.930 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'stripe count' 2023-09-23T15:34:46.930 INFO:tasks.workunit.client.0.smithi017.stderr:+ awk '{print $3}' 2023-09-23T15:34:46.930 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 1000 2023-09-23T15:34:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[215646]: pgmap v1419: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2023-09-23T15:34:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[215646]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[215646]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[215646]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T15:34:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:46 smithi017 ceph-mon[215646]: failed to encode map e480 with expected crc 2023-09-23T15:34:46.958 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T15:34:46.966 INFO:tasks.workunit.client.0.smithi017.stdout:1000 2023-09-23T15:34:46.966 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/img_v2 2023-09-23T15:34:46.967 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T15:34:47.061 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T15:34:47.063 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T15:34:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:46 smithi027 ceph-mon[195233]: pgmap v1419: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 8.5 KiB/s wr, 109 op/s 2023-09-23T15:34:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:46 smithi027 ceph-mon[195233]: failed to encode map e480 with expected crc 2023-09-23T15:34:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:46 smithi027 ceph-mon[195233]: failed to encode map e480 with expected crc 2023-09-23T15:34:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:46 smithi027 ceph-mon[195233]: osdmap e480: 8 total, 8 up, 7 in 2023-09-23T15:34:47.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:46 smithi027 ceph-mon[195233]: failed to encode map e480 with expected crc 2023-09-23T15:34:47.155 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T15:34:47.159 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_667636/img testimg 2023-09-23T15:34:47.217 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:47.220 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap1 2023-09-23T15:34:47.303 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:47.316 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:47.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:47.422 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:47.451 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:47.510 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:47.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:47.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:47.617 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:47.649 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:47.693 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T15:34:47.700 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:47.700 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap create testimg@snap2 2023-09-23T15:34:47.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:47.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:47.807 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:47.838 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:47.947 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:47.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:47.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:48.047 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:34:48.050 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:48.061 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:48.066 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:34:48.079 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:48.088 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:34:48.112 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:34:48.126 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:48.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:48.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:48.131 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:34:48.150 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:34:48.161 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:34:48.161 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:34:48.162 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:48] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T15:34:48.169 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:34:48.188 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:34:48.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.189 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.190 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T15:34:48.231 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:48.264 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:48.691 INFO:tasks.workunit.client.0.smithi017.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T15:34:48.698 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap protect testimg@snap2 2023-09-23T15:34:48.738 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_667636/snap_protect 2023-09-23T15:34:48.780 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:48.783 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_667636/snap_protect testimg_import 2023-09-23T15:34:48.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: pgmap v1421: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3858916636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: pgmap v1421: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-09-23T15:34:48.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T15:34:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: failed to encode map e481 with expected crc 2023-09-23T15:34:48.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3858916636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: pgmap v1421: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 138 KiB/s rd, 12 KiB/s wr, 171 op/s 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: failed to encode map e481 with expected crc 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: osdmap e481: 8 total, 8 up, 7 in 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: failed to encode map e481 with expected crc 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: failed to encode map e481 with expected crc 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: failed to encode map e481 with expected crc 2023-09-23T15:34:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3858916636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:49.682 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[215646]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.871 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:49.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:49.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:49.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T15:34:49.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[215646]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[215646]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T15:34:49.938 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[215646]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[222878]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[222878]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[222878]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T15:34:49.939 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:49 smithi017 ceph-mon[222878]: failed to encode map e482 with expected crc 2023-09-23T15:34:49.975 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:50.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:49 smithi027 ceph-mon[195233]: failed to encode map e482 with expected crc 2023-09-23T15:34:50.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:49 smithi027 ceph-mon[195233]: failed to encode map e482 with expected crc 2023-09-23T15:34:50.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:49 smithi027 ceph-mon[195233]: osdmap e482: 8 total, 8 up, 7 in 2023-09-23T15:34:50.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:49 smithi027 ceph-mon[195233]: failed to encode map e482 with expected crc 2023-09-23T15:34:50.008 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:50.059 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:50.063 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:50.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:50.165 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:50.194 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:50.242 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:50.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:50.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:50.349 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:50.378 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:50.475 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:50.477 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:50.578 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:50.606 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:50.777 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T15:34:50.781 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap1 2023-09-23T15:34:50.781 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: False' 2023-09-23T15:34:50.820 INFO:tasks.workunit.client.0.smithi017.stdout: protected: False 2023-09-23T15:34:50.820 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd info testimg_import@snap2 2023-09-23T15:34:50.820 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep 'protected: True' 2023-09-23T15:34:50.859 INFO:tasks.workunit.client.0.smithi017.stdout: protected: True 2023-09-23T15:34:50.859 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/snap_protect 2023-09-23T15:34:50.860 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T15:34:50.904 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T15:34:50.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[215646]: pgmap v1424: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 135 KiB/s rd, 11 KiB/s wr, 164 op/s 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[215646]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[215646]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[215646]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[215646]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[222878]: pgmap v1424: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 135 KiB/s rd, 11 KiB/s wr, 164 op/s 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[222878]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[222878]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[222878]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T15:34:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:50 smithi017 ceph-mon[222878]: failed to encode map e483 with expected crc 2023-09-23T15:34:50.957 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg 2023-09-23T15:34:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:50 smithi027 ceph-mon[195233]: pgmap v1424: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 135 KiB/s rd, 11 KiB/s wr, 164 op/s 2023-09-23T15:34:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:50 smithi027 ceph-mon[195233]: failed to encode map e483 with expected crc 2023-09-23T15:34:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:50 smithi027 ceph-mon[195233]: failed to encode map e483 with expected crc 2023-09-23T15:34:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:50 smithi027 ceph-mon[195233]: osdmap e483: 8 total, 8 up, 7 in 2023-09-23T15:34:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:50 smithi027 ceph-mon[195233]: failed to encode map e483 with expected crc 2023-09-23T15:34:51.206 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:51.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:51.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:51.238 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:51.311 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:51.345 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:51.397 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:51.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:51.404 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:51.504 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T15:34:51.534 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:51.684 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0 found: no such container 2023-09-23T15:34:51.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:51.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T15:34:51.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.694 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T15:34:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3393438523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:51.695 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:51.705 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:34:51.790 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:51.821 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:51.870 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:51.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:51.957 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T15:34:51.978 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: failed to encode map e484 with expected crc 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3393438523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:51.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:52.004 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: failed to encode map e484 with expected crc 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: failed to encode map e484 with expected crc 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: failed to encode map e484 with expected crc 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: osdmap e484: 8 total, 8 up, 7 in 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: failed to encode map e484 with expected crc 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3393438523' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:52.005 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:34:52.007 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.052 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:52.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:52.160 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:52.187 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.231 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:52.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:52.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:52.337 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:52.364 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.408 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:52.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:52.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:52.513 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:52.546 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.590 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:52.595 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:52.653 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is set 2023-09-23T15:34:52.654 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T15:34:52.666 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd snap purge testimg_import 2023-09-23T15:34:52.695 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:52.722 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.767 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:52.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:52.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:52.871 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:52.900 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: pgmap v1427: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1389001354' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[215646]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: pgmap v1427: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: failed to encode map e485 with expected crc 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1389001354' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:52.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:52 smithi017 ceph-mon[222878]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T15:34:52.980 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:34:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: pgmap v1427: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: failed to encode map e485 with expected crc 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: failed to encode map e485 with expected crc 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: failed to encode map e485 with expected crc 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: osdmap e485: 8 total, 8 up, 7 in 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: failed to encode map e485 with expected crc 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1389001354' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:34:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T15:34:53.101 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:52 smithi027 ceph-mon[195233]: osdmap e486: 8 total, 8 up, 7 in 2023-09-23T15:34:53.299 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:53.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:53.302 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:53.403 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:53.430 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:53.728 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:53.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:53.731 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:53.832 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:53.866 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:53.917 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:53.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:53.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[215646]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[215646]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[215646]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[215646]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[222878]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[222878]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[222878]: failed to encode map e486 with expected crc 2023-09-23T15:34:53.949 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:53 smithi017 ceph-mon[222878]: failed to encode map e486 with expected crc 2023-09-23T15:34:54.023 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:54.053 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:53 smithi027 ceph-mon[195233]: failed to encode map e486 with expected crc 2023-09-23T15:34:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:53 smithi027 ceph-mon[195233]: failed to encode map e486 with expected crc 2023-09-23T15:34:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:53 smithi027 ceph-mon[195233]: failed to encode map e486 with expected crc 2023-09-23T15:34:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:53 smithi027 ceph-mon[195233]: failed to encode map e486 with expected crc 2023-09-23T15:34:54.100 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:54.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:54.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:54.162 INFO:teuthology.orchestra.run.smithi017.stdout:ERROR: (22) Invalid argument 2023-09-23T15:34:54.162 INFO:teuthology.orchestra.run.smithi017.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T15:34:54.166 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:54.204 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:54.236 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:54.287 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:54.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:54.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:54.393 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:54.413 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:54.426 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:54.575 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:54.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:54.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:54.679 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:54.711 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:54.711 INFO:tasks.workunit.client.0.smithi017.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T15:34:54.720 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg 2023-09-23T15:34:54.812 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:54.815 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd remove testimg_import 2023-09-23T15:34:54.909 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:54.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ tiered=0 2023-09-23T15:34:54.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ ceph osd dump 2023-09-23T15:34:54.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep '^pool' 2023-09-23T15:34:54.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep ''\''rbd'\''' 2023-09-23T15:34:54.912 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep tier 2023-09-23T15:34:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:54 smithi027 ceph-mon[195233]: pgmap v1430: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:54 smithi027 ceph-mon[195233]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:54 smithi027 ceph-mon[195233]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:54 smithi027 ceph-mon[195233]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T15:34:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:54 smithi027 ceph-mon[195233]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[215646]: pgmap v1430: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[215646]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[215646]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[215646]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[215646]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[222878]: pgmap v1430: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 446 KiB/s rd, 52 KiB/s wr, 597 op/s 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[222878]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[222878]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[222878]: osdmap e487: 8 total, 8 up, 7 in 2023-09-23T15:34:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:54 smithi017 ceph-mon[222878]: failed to encode map e487 with expected crc 2023-09-23T15:34:55.290 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T15:34:55.313 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:55.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:55.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:55.417 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:55.450 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:55.500 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:55.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:55.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:55.607 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:55.635 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:55.675 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_667636/sparse1 2023-09-23T15:34:55.681 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T15:34:55.681 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T15:34:55.681 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.0050404 s, 208 MB/s 2023-09-23T15:34:55.681 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_667636/sparse2 2023-09-23T15:34:55.682 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:55.687 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T15:34:55.687 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T15:34:55.687 INFO:tasks.workunit.client.0.smithi017.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00509085 s, 206 MB/s 2023-09-23T15:34:55.687 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_667636/sparse2 -s 2M 2023-09-23T15:34:55.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:55.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:55.688 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T15:34:55.726 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T15:34:55.726 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T15:34:55.729 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:55.729 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667636/sparse1 2023-09-23T15:34:55.788 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T15:34:55.788 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T15:34:55.789 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:55.789 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:55.794 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T15:34:55.794 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2023-09-23T15:34:55.794 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T15:34:55.816 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.817 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: failed to encode map e488 with expected crc 2023-09-23T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:55.818 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:55 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:55.820 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:55.830 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 2 MiB 2 2023-09-23T15:34:55.830 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2023-09-23T15:34:55.830 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2023-09-23T15:34:55.831 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2023-09-23T15:34:55.831 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:55.831 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:55.877 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.92164db4c782 2023-09-23T15:34:55.878 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.92164db4c782 2023-09-23T15:34:55.878 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2023-09-23T15:34:55.878 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2023-09-23T15:34:55.878 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.92164db4c782\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:55.878 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:55.879 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2023-09-23T15:34:55.879 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:55.880 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:55.919 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:55.919 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:55.919 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:55.919 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:55.919 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:56.021 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:34:56.065 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T15:34:56.065 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T15:34:56.065 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T15:34:56.065 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: failed to encode map e488 with expected crc 2023-09-23T15:34:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: failed to encode map e488 with expected crc 2023-09-23T15:34:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T15:34:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: failed to encode map e488 with expected crc 2023-09-23T15:34:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: failed to encode map e488 with expected crc 2023-09-23T15:34:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:55 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/92656495' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T15:34:56.103 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:56.107 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.108 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.110 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse1 --format %b 2023-09-23T15:34:56.111 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T15:34:56.111 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse1.out --format %b 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.113 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T15:34:56.193 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2 found: no such container 2023-09-23T15:34:56.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:56.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T15:34:56.218 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T15:34:56.221 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T15:34:56.260 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 0% complete...failed. 2023-09-23T15:34:56.260 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T15:34:56.263 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:56.264 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667636/sparse2 2023-09-23T15:34:56.298 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:56.325 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:56.338 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T15:34:56.339 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:56.339 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:56.343 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T15:34:56.343 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2023-09-23T15:34:56.343 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T15:34:56.369 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:56.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:56.385 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 2 MiB 2 2023-09-23T15:34:56.385 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2023-09-23T15:34:56.385 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2023-09-23T15:34:56.386 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2023-09-23T15:34:56.386 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:56.386 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:56.424 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.9246544bf69c 2023-09-23T15:34:56.424 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.9246544bf69c 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.9246544bf69c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:56.425 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:56.460 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:56.460 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:56.461 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:56.461 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:56.461 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:56.476 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:56.495 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2023-09-23T15:34:56.495 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2023-09-23T15:34:56.496 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T15:34:56.496 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:56.503 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:56.532 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:56.535 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:56.536 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:56.538 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse2 --format %b 2023-09-23T15:34:56.539 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T15:34:56.539 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse2.out --format %b 2023-09-23T15:34:56.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T15:34:56.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T15:34:56.540 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:56.541 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T15:34:56.546 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:56.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:56.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:56.626 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T15:34:56.629 INFO:tasks.workunit.client.0.smithi017.stderr:+ truncate /tmp/rbd_import_export_667636/sparse1 -s 10M 2023-09-23T15:34:56.630 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T15:34:56.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:56.680 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:56.717 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:56.717 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:56.718 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:56.720 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T15:34:56.720 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse1 2023-09-23T15:34:56.720 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T15:34:56.728 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:56.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:56.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:56.757 INFO:tasks.workunit.client.0.smithi017.stdout:sparse1 10 MiB 2 2023-09-23T15:34:56.758 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse1 2023-09-23T15:34:56.758 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse1 2023-09-23T15:34:56.758 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse1 2023-09-23T15:34:56.759 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:56.759 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:56.794 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.b93337faeb38 2023-09-23T15:34:56.795 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.b93337faeb38 2023-09-23T15:34:56.795 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse1 2023-09-23T15:34:56.795 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse1 2023-09-23T15:34:56.795 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.b93337faeb38\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:56.795 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:56.796 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse1 2023-09-23T15:34:56.796 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:56.796 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:56.832 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:56.832 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:56.832 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:56.832 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:56.832 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:56.836 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:56.865 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:56 smithi017 ceph-mon[215646]: pgmap v1433: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 6.5 KiB/s rd, 1.7 KiB/s wr, 14 op/s 2023-09-23T15:34:56.865 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:56 smithi017 ceph-mon[222878]: pgmap v1433: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 6.5 KiB/s rd, 1.7 KiB/s wr, 14 op/s 2023-09-23T15:34:56.866 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T15:34:56.866 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T15:34:56.866 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T15:34:56.866 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.868 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:56.910 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:56.914 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.914 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/sparse1 /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.923 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse1 --format %b 2023-09-23T15:34:56.924 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=2048 2023-09-23T15:34:56.924 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse1.out --format %b 2023-09-23T15:34:56.925 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=2048 2023-09-23T15:34:56.925 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.925 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:56.925 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T15:34:56.925 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse1.out 2023-09-23T15:34:56.926 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse1 2023-09-23T15:34:56.972 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:34:56.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:56.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:34:57.027 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T15:34:57.031 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_667636/sparse2 oflag=append conv=notrunc 2023-09-23T15:34:57.042 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T15:34:57.042 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T15:34:57.042 INFO:tasks.workunit.client.0.smithi017.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00998252 s, 210 MB/s 2023-09-23T15:34:57.042 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T15:34:57.078 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:56 smithi027 ceph-mon[195233]: pgmap v1433: 103 pgs: 103 active+clean; 4.8 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 6.5 KiB/s rd, 1.7 KiB/s wr, 14 op/s 2023-09-23T15:34:57.110 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:57.121 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:57.122 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:57.122 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:57.125 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd ls -l 2023-09-23T15:34:57.125 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep sparse2 2023-09-23T15:34:57.125 INFO:tasks.workunit.client.0.smithi017.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T15:34:57.164 INFO:tasks.workunit.client.0.smithi017.stdout:sparse2 4 MiB 2 2023-09-23T15:34:57.165 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse2 2023-09-23T15:34:57.165 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse2 2023-09-23T15:34:57.165 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse2 2023-09-23T15:34:57.165 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:57.165 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:57.204 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.929d6f18b176 2023-09-23T15:34:57.205 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.929d6f18b176 2023-09-23T15:34:57.205 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse2 2023-09-23T15:34:57.205 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse2 2023-09-23T15:34:57.205 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.929d6f18b176\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:57.205 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:57.206 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse2 2023-09-23T15:34:57.206 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:57.206 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:57.266 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:57.266 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:57.266 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:57.266 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:57.266 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects='0 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:2 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:3' 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2 3 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T15:34:57.314 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:57.321 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:57.329 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:57.336 INFO:teuthology.orchestra.run.smithi017.stdout:osd.0: {} 2023-09-23T15:34:57.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:57.338 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:57.366 INFO:teuthology.orchestra.run.smithi017.stdout:osd.1: {} 2023-09-23T15:34:57.372 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T15:34:57.375 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:34:57.376 INFO:tasks.workunit.client.0.smithi017.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_667636/sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:57.376 INFO:tasks.workunit.client.0.smithi017.stderr:+ cmp -l /tmp/rbd_import_export_667636/sparse2 /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:57.380 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse2 --format %b 2023-09-23T15:34:57.381 INFO:tasks.workunit.client.0.smithi017.stderr:+ origsize=6144 2023-09-23T15:34:57.381 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse2.out --format %b 2023-09-23T15:34:57.382 INFO:tasks.workunit.client.0.smithi017.stderr:+ exportsize=6144 2023-09-23T15:34:57.382 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:57.382 INFO:tasks.workunit.client.0.smithi017.stderr:+ difference=0 2023-09-23T15:34:57.382 INFO:tasks.workunit.client.0.smithi017.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T15:34:57.382 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse2.out 2023-09-23T15:34:57.383 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse2 2023-09-23T15:34:57.390 INFO:teuthology.orchestra.run.smithi017.stdout:osd.2: {} 2023-09-23T15:34:57.413 INFO:teuthology.orchestra.run.smithi017.stdout:osd.3: {} 2023-09-23T15:34:57.439 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:57.440 INFO:teuthology.orchestra.run.smithi017.stdout:osd.4: {} 2023-09-23T15:34:57.465 INFO:teuthology.orchestra.run.smithi017.stdout:osd.5: {} 2023-09-23T15:34:57.480 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:57.492 INFO:teuthology.orchestra.run.smithi017.stdout:osd.6: {} 2023-09-23T15:34:57.500 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T15:34:57.505 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T15:34:57.505 INFO:tasks.workunit.client.0.smithi017.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T15:34:57.506 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_667636/sparse1 sparse 2023-09-23T15:34:57.519 INFO:teuthology.orchestra.run.smithi017.stdout:osd.7: {} 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.522 INFO:teuthology.orchestra.run.smithi017.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T15:34:57.583 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T15:34:57.583 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:57.583 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T15:34:57.586 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T15:34:57.586 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T15:34:57.587 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T15:34:57.587 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:57.587 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:57.623 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.b993a483be8 2023-09-23T15:34:57.624 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.b993a483be8 2023-09-23T15:34:57.624 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.b993a483be8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:57.624 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T15:34:57.624 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T15:34:57.624 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:57.625 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T15:34:57.625 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:57.625 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:57.668 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:57.668 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:57.668 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:57.669 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:57.669 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:57.680 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:57.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:57.683 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:57.703 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=1 2023-09-23T15:34:57.703 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 1 2023-09-23T15:34:57.703 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T15:34:57.704 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T15:34:57.784 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T15:34:57.804 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T15:34:57.808 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T15:34:57.808 INFO:tasks.workunit.client.0.smithi017.stdout:zeros import through stdin to sparse image 2023-09-23T15:34:57.808 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T15:34:57.809 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd import --new-format - sparse 2023-09-23T15:34:57.811 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:57.856 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7 found: no such container 2023-09-23T15:34:57.856 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records in 2023-09-23T15:34:57.857 INFO:tasks.workunit.client.0.smithi017.stderr:4+0 records out 2023-09-23T15:34:57.857 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0481118 s, 87.2 MB/s 2023-09-23T15:34:57.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:57.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T15:34:57.867 INFO:tasks.workunit.client.0.smithi017.stderr: Importing image: 100% complete...done. 2023-09-23T15:34:57.868 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:57.871 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T15:34:57.871 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T15:34:57.872 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T15:34:57.872 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:57.872 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:57.873 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2098908768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:57.874 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2098908768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:57.907 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.92f77d7f5619 2023-09-23T15:34:57.908 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.92f77d7f5619 2023-09-23T15:34:57.908 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.92f77d7f5619\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:57.908 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T15:34:57.908 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T15:34:57.908 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:57.910 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T15:34:57.910 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:57.911 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:57.945 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:57.945 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:57.945 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:57.945 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:57.945 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:57.962 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T15:34:57.978 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects= 2023-09-23T15:34:57.978 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 2023-09-23T15:34:57.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T15:34:57.978 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T15:34:57.986 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2098908768' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T15:34:57.988 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:58.035 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5 found: no such container 2023-09-23T15:34:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:58.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T15:34:58.141 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T15:34:58.171 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:34:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:34:58] "GET /metrics HTTP/1.1" 200 34032 "" "Prometheus/2.43.0" 2023-09-23T15:34:58.175 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:58.185 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:58.188 INFO:tasks.workunit.client.0.smithi017.stdout:zeros export to sparse file 2023-09-23T15:34:58.189 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo 'zeros export to sparse file' 2023-09-23T15:34:58.189 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T15:34:58.230 INFO:tasks.workunit.client.0.smithi017.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T15:34:58.233 INFO:tasks.workunit.client.0.smithi017.stderr:++ rbd info sparse 2023-09-23T15:34:58.233 INFO:tasks.workunit.client.0.smithi017.stderr:++ grep block_name_prefix 2023-09-23T15:34:58.233 INFO:tasks.workunit.client.0.smithi017.stderr:++ awk '{print $NF;}' 2023-09-23T15:34:58.269 INFO:tasks.workunit.client.0.smithi017.stderr:+ prefix=rbd_data.b9db26dda880 2023-09-23T15:34:58.269 INFO:tasks.workunit.client.0.smithi017.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T15:34:58.270 INFO:tasks.workunit.client.0.smithi017.stderr:++ get_image_data_pool sparse 2023-09-23T15:34:58.270 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T15:34:58.270 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T15:34:58.270 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep 'data_pool: ' 2023-09-23T15:34:58.271 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T15:34:58.309 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool= 2023-09-23T15:34:58.310 INFO:tasks.workunit.client.0.smithi017.stderr:++ '[' -z ']' 2023-09-23T15:34:58.310 INFO:tasks.workunit.client.0.smithi017.stderr:++ data_pool=rbd 2023-09-23T15:34:58.310 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo rbd 2023-09-23T15:34:58.310 INFO:tasks.workunit.client.0.smithi017.stderr:+ rados -p rbd put rbd_data.b9db26dda880.000000000000 - 2023-09-23T15:34:58.339 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records in 2023-09-23T15:34:58.339 INFO:tasks.workunit.client.0.smithi017.stderr:1+0 records out 2023-09-23T15:34:58.340 INFO:tasks.workunit.client.0.smithi017.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0692986 s, 60.5 MB/s 2023-09-23T15:34:58.355 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects sparse 2023-09-23T15:34:58.356 INFO:tasks.workunit.client.0.smithi017.stderr:++ image=sparse 2023-09-23T15:34:58.356 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rbd info sparse 2023-09-23T15:34:58.356 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep block_name_prefix 2023-09-23T15:34:58.356 INFO:tasks.workunit.client.0.smithi017.stderr:+++ awk '{print $NF;}' 2023-09-23T15:34:58.392 INFO:tasks.workunit.client.0.smithi017.stderr:++ prefix=rbd_data.b9db26dda880 2023-09-23T15:34:58.393 INFO:tasks.workunit.client.0.smithi017.stderr:+++ grep rbd_data.b9db26dda880 2023-09-23T15:34:58.393 INFO:tasks.workunit.client.0.smithi017.stderr:++++ get_image_data_pool sparse 2023-09-23T15:34:58.393 INFO:tasks.workunit.client.0.smithi017.stderr:++++ image=sparse 2023-09-23T15:34:58.393 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sed -e 's/rbd_data.b9db26dda880\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T15:34:58.393 INFO:tasks.workunit.client.0.smithi017.stderr:+++ sort -u 2023-09-23T15:34:58.394 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ rbd info sparse 2023-09-23T15:34:58.394 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ grep 'data_pool: ' 2023-09-23T15:34:58.394 INFO:tasks.workunit.client.0.smithi017.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T15:34:58.429 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool= 2023-09-23T15:34:58.429 INFO:tasks.workunit.client.0.smithi017.stderr:++++ '[' -z ']' 2023-09-23T15:34:58.429 INFO:tasks.workunit.client.0.smithi017.stderr:++++ data_pool=rbd 2023-09-23T15:34:58.429 INFO:tasks.workunit.client.0.smithi017.stderr:++++ echo rbd 2023-09-23T15:34:58.429 INFO:tasks.workunit.client.0.smithi017.stderr:+++ rados ls -p rbd 2023-09-23T15:34:58.462 INFO:tasks.workunit.client.0.smithi017.stderr:++ objects=0 2023-09-23T15:34:58.462 INFO:tasks.workunit.client.0.smithi017.stderr:++ echo 0 2023-09-23T15:34:58.462 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T15:34:58.462 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse 2023-09-23T15:34:58.463 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_667636/sparse': No such file or directory 2023-09-23T15:34:58.463 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:58.463 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd export sparse /tmp/rbd_import_export_667636/sparse 2023-09-23T15:34:58.498 INFO:tasks.workunit.client.0.smithi017.stderr: Exporting image: 100% complete...done. 2023-09-23T15:34:58.501 INFO:tasks.workunit.client.0.smithi017.stderr:++ stat /tmp/rbd_import_export_667636/sparse --format=%b 2023-09-23T15:34:58.502 INFO:tasks.workunit.client.0.smithi017.stderr:+ '[' 0 = 0 ']' 2023-09-23T15:34:58.502 INFO:tasks.workunit.client.0.smithi017.stderr:+ rbd rm sparse 2023-09-23T15:34:58.583 INFO:tasks.workunit.client.0.smithi017.stderr: Removing image: 100% complete...done. 2023-09-23T15:34:58.585 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3 found: no such container 2023-09-23T15:34:58.586 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm /tmp/rbd_import_export_667636/sparse /tmp/rbd_import_export_667636/sparse1 /tmp/rbd_import_export_667636/sparse2 /tmp/rbd_import_export_667636/sparse3 2023-09-23T15:34:58.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:58.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T15:34:58.589 INFO:tasks.workunit.client.0.smithi017.stderr:rm: cannot remove '/tmp/rbd_import_export_667636/sparse3': No such file or directory 2023-09-23T15:34:58.589 INFO:tasks.workunit.client.0.smithi017.stderr:+ true 2023-09-23T15:34:58.590 INFO:tasks.workunit.client.0.smithi017.stderr:+ echo OK 2023-09-23T15:34:58.590 INFO:tasks.workunit.client.0.smithi017.stdout:OK 2023-09-23T15:34:58.591 INFO:tasks.workunit.client.0.smithi017.stderr:+ rm -rf /tmp/rbd_import_export_667636 2023-09-23T15:34:58.591 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T15:34:58.591 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T15:34:58.667 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T15:34:58.667 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T15:34:58.689 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T15:34:58.718 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:58.728 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:34:58.761 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4 found: no such container 2023-09-23T15:34:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:58.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T15:34:58.855 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[215646]: pgmap v1434: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 73 KiB/s rd, 4.8 KiB/s wr, 97 op/s 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[222878]: pgmap v1434: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 73 KiB/s rd, 4.8 KiB/s wr, 97 op/s 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:58.856 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:58.870 DEBUG:teuthology.orchestra.run.smithi027:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T15:34:58.896 INFO:teuthology.orchestra.run.smithi027.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:58.941 INFO:teuthology.orchestra.run.smithi027.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6 found: no such container 2023-09-23T15:34:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:34:58.946 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi027 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T15:34:59.047 DEBUG:teuthology.orchestra.run.smithi017:> sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T15:34:59.075 INFO:teuthology.orchestra.run.smithi017.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T15:34:59.082 DEBUG:teuthology.parallel:result is None 2023-09-23T15:34:59.082 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:58 smithi027 ceph-mon[195233]: pgmap v1434: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 73 KiB/s rd, 4.8 KiB/s wr, 97 op/s 2023-09-23T15:34:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:34:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:34:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:34:59.149 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:59.150 DEBUG:teuthology.orchestra.run.smithi017:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:59.206 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:34:59.207 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T15:34:59.207 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T15:34:59.207 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T15:34:59.207 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T15:34:59.207 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T15:34:59.208 DEBUG:teuthology.orchestra.run.smithi017:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:34:59.264 INFO:teuthology.orchestra.run.smithi017.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T15:34:59.264 DEBUG:teuthology.orchestra.run.smithi017:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:59.321 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T15:34:59.321 DEBUG:teuthology.orchestra.run.smithi017:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T15:34:59.379 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T15:35:00.076 INFO:teuthology.orchestra.run.smithi017.stderr:Error: no container with name or ID ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1 found: no such container 2023-09-23T15:35:00.078 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[215646]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[215646]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[215646]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[222878]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[222878]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:35:00.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:34:59 smithi017 ceph-mon[222878]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T15:35:00.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T15:35:00.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi017 with status 125: 'sudo docker kill -s 1 ceph-546d6668-5a20-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T15:35:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:59 smithi027 ceph-mon[195233]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:59 smithi027 ceph-mon[195233]: failed to encode map e489 with expected crc 2023-09-23T15:35:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:59 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2350431887' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:35:00.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:34:59 smithi027 ceph-mon[195233]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T15:35:00.751 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:35:00.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:00 smithi017 ceph-mon[215646]: pgmap v1436: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s rd, 4.3 KiB/s wr, 87 op/s 2023-09-23T15:35:00.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:00 smithi017 ceph-mon[222878]: pgmap v1436: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s rd, 4.3 KiB/s wr, 87 op/s 2023-09-23T15:35:01.033 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:01.033 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:35:01.033 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:35:01.033 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:35:01.034 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:01.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:00 smithi027 ceph-mon[195233]: pgmap v1436: 103 pgs: 103 active+clean; 4.7 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 65 KiB/s rd, 4.3 KiB/s wr, 87 op/s 2023-09-23T15:35:01.735 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:35:01.997 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:01 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/769386526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:01.997 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:01 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:01.997 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:01 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/769386526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:01.997 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:01 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:01 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/769386526' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:01 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:02.206 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T15:35:02.665 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:02.716 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:02.871 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: pgmap v1437: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 392 op/s 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: failed to encode map e490 with expected crc 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: failed to encode map e490 with expected crc 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[215646]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T15:35:02.872 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: pgmap v1437: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 392 op/s 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: failed to encode map e490 with expected crc 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: failed to encode map e490 with expected crc 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T15:35:02.873 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:02 smithi017 ceph-mon[222878]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T15:35:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: pgmap v1437: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.1 MiB/s rd, 1.5 MiB/s wr, 392 op/s 2023-09-23T15:35:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T15:35:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: failed to encode map e490 with expected crc 2023-09-23T15:35:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: failed to encode map e490 with expected crc 2023-09-23T15:35:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T15:35:03.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:02 smithi027 ceph-mon[195233]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T15:35:03.791 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:03 smithi017 ceph-mon[215646]: failed to encode map e491 with expected crc 2023-09-23T15:35:03.792 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:03 smithi017 ceph-mon[215646]: failed to encode map e491 with expected crc 2023-09-23T15:35:03.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:03 smithi017 ceph-mon[222878]: failed to encode map e491 with expected crc 2023-09-23T15:35:03.792 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:03 smithi017 ceph-mon[222878]: failed to encode map e491 with expected crc 2023-09-23T15:35:04.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:03 smithi027 ceph-mon[195233]: failed to encode map e491 with expected crc 2023-09-23T15:35:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:03 smithi027 ceph-mon[195233]: failed to encode map e491 with expected crc 2023-09-23T15:35:04.510 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T15:35:04.874 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:04 smithi017 ceph-mon[215646]: pgmap v1440: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 394 op/s 2023-09-23T15:35:04.875 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:04 smithi017 ceph-mon[215646]: failed to encode map e491 with expected crc 2023-09-23T15:35:04.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:04 smithi017 ceph-mon[222878]: pgmap v1440: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 394 op/s 2023-09-23T15:35:04.875 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:04 smithi017 ceph-mon[222878]: failed to encode map e491 with expected crc 2023-09-23T15:35:05.075 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:04 smithi027 ceph-mon[195233]: pgmap v1440: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 394 op/s 2023-09-23T15:35:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:04 smithi027 ceph-mon[195233]: failed to encode map e491 with expected crc 2023-09-23T15:35:05.110 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83104 9.c 9:3b5b5686:::benchmark_data_smithi017_543597_object83103:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=3 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79843659599999, 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.450679148000001, 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.111 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83104 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.112 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309516+0000", 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309514+0000", 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.113 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309542+0000", 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7407e-05 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309556+0000", 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4404e-05 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309570+0000", 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3918e-05 2023-09-23T15:35:05.114 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313812+0000", 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042423749999999996 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313824+0000", 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.166e-05 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.115 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.754362+0000", 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.440538513 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.760195+0000", 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0058326949999999997 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.116 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83557 9.c 9:3ca96c17:::benchmark_data_smithi017_543597_object83556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79836975400002, 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.541859289, 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83557 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.117 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309583+0000", 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.118 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309582+0000", 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309604+0000", 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.207e-05 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309614+0000", 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.679e-06 2023-09-23T15:35:05.119 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309621+0000", 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9110000000000004e-06 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313846+0000", 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042255110000000004 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.120 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313853+0000", 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9179999999999997e-06 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.774722+0000", 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.460868532999999 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.781443+0000", 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0067207079999999997 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.121 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851313+0000", 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.069870627000000005 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851442+0000", 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000128901 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.122 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83560 9.c 9:3d5597f0:::benchmark_data_smithi017_543597_object83559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79832431099999, 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.543202913, 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83560 2023-09-23T15:35:05.123 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.124 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309628+0000", 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309648+0000", 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0092999999999999e-05 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.125 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309656+0000", 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.8730000000000007e-06 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309662+0000", 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0800000000000002e-06 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313873+0000", 2023-09-23T15:35:05.126 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042114159999999999 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313880+0000", 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9659999999999999e-06 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.782319+0000", 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.468439010999999 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.127 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.789923+0000", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0076032809999999999 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.801626+0000", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011703523 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.836893+0000", 2023-09-23T15:35:05.128 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.035267251999999999 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851529+0000", 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014635815 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851566+0000", 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7289000000000002e-05 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.129 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.851601+0000", 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5088e-05 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.852831+0000", 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001229759 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.130 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83591 9.c 9:3687f2ca:::benchmark_data_smithi017_543597_object83590:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79828543899998, 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.545316396, 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83591 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.131 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.132 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309667+0000", 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309686+0000", 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9743999999999999e-05 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309694+0000", 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.729e-06 2023-09-23T15:35:05.133 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309700+0000", 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9669999999999998e-06 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313901+0000", 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0042010720000000001 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.134 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313908+0000", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8669999999999996e-06 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.809499+0000", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.495590667 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.816644+0000", 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0071449620000000004 2023-09-23T15:35:05.135 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.828154+0000", 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011510423000000001 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.840204+0000", 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012049908 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.136 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.852892+0000", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012687617 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.853952+0000", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0010600189999999999 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.853989+0000", 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7109e-05 2023-09-23T15:35:05.137 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.854984+0000", 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00099478700000000001 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83628 9.c 9:3ed93163:::benchmark_data_smithi017_543597_object83627:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:35:05.138 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.798246533, 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.55767582, 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83628 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.139 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309706+0000", 2023-09-23T15:35:05.140 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309730+0000", 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4318e-05 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309739+0000", 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.141 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309746+0000", 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1019999999999999e-06 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313930+0000", 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041835470000000001 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.142 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313937+0000", 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8249999999999999e-06 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855030+0000", 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.541093047 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855155+0000", 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000124931 2023-09-23T15:35:05.143 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.855264+0000", 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010964299999999999 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864529+0000", 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0092652810000000002 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.144 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864584+0000", 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5031000000000002e-05 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864715+0000", 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000130536 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.145 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864771+0000", 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5627000000000001e-05 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867382+0000", 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002611262 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.146 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83637 9.c 9:3ca6bb70:::benchmark_data_smithi017_543597_object83636:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79820098800002, 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.564530451, 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83637 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.147 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309752+0000", 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.148 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309751+0000", 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309779+0000", 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.7466000000000001e-05 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309787+0000", 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9789999999999993e-06 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.149 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309793+0000", 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2539999999999998e-06 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313957+0000", 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041637480000000001 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.150 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313963+0000", 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.781e-06 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857515+0000", 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.543551269 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857634+0000", 2023-09-23T15:35:05.151 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011945 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.857744+0000", 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010961199999999999 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864649+0000", 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0069050509999999997 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.152 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864661+0000", 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.183e-05 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872189+0000", 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007528814 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.153 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872246+0000", 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6660999999999998e-05 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874282+0000", 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020360729999999998 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.154 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83643 9.c 9:37b8a565:::benchmark_data_smithi017_543597_object83642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79815458899998, 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.574886889, 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83643 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.155 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309798+0000", 2023-09-23T15:35:05.156 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309817+0000", 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9907000000000001e-05 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309825+0000", 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9619999999999993e-06 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.157 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309831+0000", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6400000000000002e-06 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313986+0000", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041548380000000001 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.313993+0000", 2023-09-23T15:35:05.158 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7379999999999997e-06 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.859721+0000", 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.545728274 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.859860+0000", 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013950799999999999 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.159 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.860015+0000", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015417 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.864666+0000", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0046517329999999999 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869847+0000", 2023-09-23T15:35:05.160 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051809680000000002 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882639+0000", 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012791497000000001 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882710+0000", 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1630000000000004e-05 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.161 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884685+0000", 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001974515 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83670 9.c 9:3f0b0e67:::benchmark_data_smithi017_543597_object83669:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79811657200003, 2023-09-23T15:35:05.162 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.576675178, 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83670 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:35:05.163 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309836+0000", 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.164 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309887+0000", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1570999999999998e-05 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309896+0000", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5480000000000002e-06 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309902+0000", 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2110000000000003e-06 2023-09-23T15:35:05.165 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314013+0000", 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041110369999999997 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314020+0000", 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5479999999999998e-06 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.166 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862483+0000", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.548463865 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862614+0000", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013095899999999999 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.862718+0000", 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000103745 2023-09-23T15:35:05.167 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869859+0000", 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007141077 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869905+0000", 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5868000000000002e-05 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.168 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884831+0000", 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.014925905 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884902+0000", 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1285000000000006e-05 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886511+0000", 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001608979 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.169 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83675 9.c 9:34036259:::benchmark_data_smithi017_543597_object83674:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79804503600002, 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.57811843, 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.170 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83675 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309908+0000", 2023-09-23T15:35:05.171 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309907+0000", 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309953+0000", 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5658000000000002e-05 2023-09-23T15:35:05.172 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309961+0000", 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9570000000000002e-06 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309968+0000", 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9099999999999999e-06 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.173 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314039+0000", 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040712120000000003 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314046+0000", 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5000000000000002e-06 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867440+0000", 2023-09-23T15:35:05.174 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.553393381999999 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867560+0000", 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000120005 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.867672+0000", 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011218 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.175 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.872157+0000", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044847680000000001 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874297+0000", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021403350000000002 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886622+0000", 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.012324701 2023-09-23T15:35:05.176 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886696+0000", 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4108000000000003e-05 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888026+0000", 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013302500000000001 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.177 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83705 9.c 9:344ed245:::benchmark_data_smithi017_543597_object83704:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79797887199999, 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.579326626, 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.178 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83705 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309974+0000", 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.179 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.309973+0000", 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310021+0000", 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7636000000000003e-05 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.180 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310030+0000", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0229999999999998e-06 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310036+0000", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.1680000000000001e-06 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314066+0000", 2023-09-23T15:35:05.181 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0040297309999999999 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314073+0000", 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7900000000000002e-06 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.869943+0000", 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.555870074 2023-09-23T15:35:05.182 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.870056+0000", 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000113826 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.870161+0000", 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000104693 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.183 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877402+0000", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072409420000000002 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877418+0000", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5412000000000001e-05 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888089+0000", 2023-09-23T15:35:05.184 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010670999 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.888159+0000", 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0282000000000002e-05 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889300+0000", 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0011415609999999999 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.185 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83714 9.c 9:3ecd0e01:::benchmark_data_smithi017_543597_object83713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79791158699999, 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.583786938999999, 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.186 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83714 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:35:05.187 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310041+0000", 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310082+0000", 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0963000000000001e-05 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.188 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310090+0000", 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2069999999999995e-06 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310095+0000", 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.755e-06 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.189 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314094+0000", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003998896 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314101+0000", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7120000000000003e-06 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874330+0000", 2023-09-23T15:35:05.190 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.560229039999999 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874474+0000", 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014405 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.874604+0000", 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012965900000000001 2023-09-23T15:35:05.191 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.882580+0000", 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0079760430000000004 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884707+0000", 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0021275420000000001 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.192 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.892134+0000", 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0074264400000000003 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.892199+0000", 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4642999999999998e-05 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893828+0000", 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00162948 2023-09-23T15:35:05.193 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83736 9.c 9:31e1d931:::benchmark_data_smithi017_543597_object83735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79785177399998, 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.585146602, 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.194 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83736 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.195 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310101+0000", 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310100+0000", 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310145+0000", 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4842999999999997e-05 2023-09-23T15:35:05.196 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310154+0000", 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.3890000000000005e-06 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310159+0000", 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8950000000000003e-06 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.197 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314121+0000", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039618409999999998 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314129+0000", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9820000000000008e-06 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877437+0000", 2023-09-23T15:35:05.198 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.563308244 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877563+0000", 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012571699999999999 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.877684+0000", 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012044699999999999 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.199 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884719+0000", 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070358269999999997 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.884772+0000", 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.2868000000000003e-05 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.200 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893901+0000", 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0091287179999999992 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.893958+0000", 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.7123999999999997e-05 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895247+0000", 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012892579999999999 2023-09-23T15:35:05.201 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83741 9.c 9:327e7b25:::benchmark_data_smithi017_543597_object83740:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79778832800002, 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.586280486, 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.202 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83741 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.203 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310164+0000", 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310208+0000", 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3733999999999998e-05 2023-09-23T15:35:05.204 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310216+0000", 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.303e-06 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310222+0000", 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0059999999999998e-06 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.205 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314151+0000", 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039289360000000001 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314157+0000", 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6100000000000002e-06 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.879806+0000", 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.565648801 2023-09-23T15:35:05.206 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.880006+0000", 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019943000000000001 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.880426+0000", 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000419978 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.207 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886532+0000", 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061066209999999996 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.886567+0000", 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.5091e-05 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895321+0000", 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0087539499999999999 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.208 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.895407+0000", 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5644999999999997e-05 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896445+0000", 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001037865 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.209 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83768 9.c 9:351643c6:::benchmark_data_smithi017_543597_object83767:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79772581899999, 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.587392286, 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83768 2023-09-23T15:35:05.210 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310227+0000", 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.211 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310226+0000", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310270+0000", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.3220000000000003e-05 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310278+0000", 2023-09-23T15:35:05.212 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4339999999999992e-06 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310284+0000", 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2099999999999998e-06 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314177+0000", 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003892543 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.213 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314183+0000", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4470000000000003e-06 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889341+0000", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.575158296 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889546+0000", 2023-09-23T15:35:05.214 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000204611 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.889767+0000", 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000221153 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896466+0000", 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0066984519999999997 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.215 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896498+0000", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.2681000000000001e-05 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896558+0000", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9395999999999998e-05 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.896623+0000", 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5241999999999996e-05 2023-09-23T15:35:05.216 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897619+0000", 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00099607899999999997 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83784 9.c 9:3b4fc178:::benchmark_data_smithi017_543597_object83783:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79766314599999, 2023-09-23T15:35:05.217 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.600940644, 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83784 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.218 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310289+0000", 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.219 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310337+0000", 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.795e-05 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310346+0000", 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6170000000000007e-06 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.220 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310352+0000", 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3069999999999999e-06 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314206+0000", 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038536709999999999 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314214+0000", 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.4409999999999993e-06 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.221 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897667+0000", 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.583452796 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897794+0000", 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012709700000000001 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.222 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.897922+0000", 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012811800000000001 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909243+0000", 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01132096 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909259+0000", 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6317e-05 2023-09-23T15:35:05.223 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909284+0000", 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5222e-05 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.909338+0000", 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.4014000000000002e-05 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.224 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911230+0000", 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0018916580000000001 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83788 9.c 9:3675609c:::benchmark_data_smithi017_543597_object83787:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 460.79759511499998, 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.602168643000001, 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.225 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83788 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.226 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310358+0000", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310357+0000", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310410+0000", 2023-09-23T15:35:05.227 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3470000000000001e-05 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310420+0000", 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1230000000000006e-06 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.310426+0000", 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.55e-06 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.228 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314234+0000", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038076410000000001 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:24.314241+0000", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8789999999999997e-06 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900244+0000", 2023-09-23T15:35:05.229 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.586003567000001 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900448+0000", 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020388399999999999 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.900672+0000", 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000224396 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.230 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911301+0000", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.010628687 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911316+0000", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.5136e-05 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911348+0000", 2023-09-23T15:35:05.231 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.1900000000000003e-05 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.911404+0000", 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5878000000000003e-05 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:34.912526+0000", 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001122086 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.232 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88021 9.c 9:32f7ad96:::benchmark_data_smithi017_543597_object4230:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 444.75940583300002, 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0658238880000002, 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88021 2023-09-23T15:35:05.233 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348547+0000", 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.234 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348546+0000", 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348551+0000", 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3070000000000002e-06 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.235 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348561+0000", 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0054000000000001e-05 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.348569+0000", 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.345e-06 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.352466+0000", 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0038970519999999998 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.236 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.352516+0000", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0056000000000003e-05 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381306+0000", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.028790685 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381435+0000", 2023-09-23T15:35:05.237 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012852800000000001 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.381595+0000", 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015973300000000001 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.388797+0000", 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.007202623 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.238 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.392930+0000", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0041327489999999998 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415879+0000", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.022948867000000001 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415891+0000", 2023-09-23T15:35:05.239 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2160999999999999e-05 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418056+0000", 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0021654120000001 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418069+0000", 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2099999999999999e-05 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.240 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.396257+0000", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97818863700000003 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.396349+0000", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1415999999999993e-05 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414241+0000", 2023-09-23T15:35:05.241 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.017892049 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414371+0000", 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013000300000000001 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88027 9.c 9:3e9c13fa:::benchmark_data_smithi017_543597_object4236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:35:05.242 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 444.74673738000001, 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.0608875040000001, 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88027 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.243 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361215+0000", 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361214+0000", 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:05.244 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361221+0000", 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7739999999999999e-06 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361233+0000", 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2184000000000001e-05 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.245 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.361243+0000", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0249e-05 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.363856+0000", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026123880000000002 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.364327+0000", 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00047180900000000001 2023-09-23T15:35:05.246 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415920+0000", 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.051592551 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:40.415929+0000", 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3179999999999996e-06 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.247 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418103+0000", 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.002173486 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.418112+0000", 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1540000000000008e-06 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397094+0000", 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.97898225000000005 2023-09-23T15:35:05.248 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397186+0000", 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.234e-05 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 4,7", 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.397276+0000", 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9622000000000004e-05 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.249 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.410961+0000", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.013685075999999999 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.414425+0000", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0034636319999999999 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.420393+0000", 2023-09-23T15:35:05.250 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059684600000000001 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.420440+0000", 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7029e-05 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.422103+0000", 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0016625019999999999 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.251 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:140575 10.3 e440/413 10:cbf42ed9:::benchmark_data_smithi017_604517_object34464:head v 440'8931, mlcod=440'8927)", 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:32:11.716590+0000", 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 173.39136215799999, 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.146058417, 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716590+0000", 2023-09-23T15:35:05.252 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716590+0000", 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716590+0000", 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.253 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716598+0000", 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2549999999999997e-06 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716609+0000", 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.172e-05 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.254 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716617+0000", 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.6480000000000003e-06 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.723826+0000", 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0072088450000000002 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.723846+0000", 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9811999999999999e-05 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.255 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.862606+0000", 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.13876024100000001 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.862649+0000", 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.2852000000000002e-05 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.256 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_repop(client.46520.0:140579 10.3 e440/413 10:c1ad9d6c:::benchmark_data_smithi017_604517_object34468:head v 440'8932, mlcod=440'8927)", 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:32:11.716664+0000", 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 173.39128815999999, 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.15083919700000001, 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716664+0000", 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.257 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716664+0000", 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716664+0000", 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:05.258 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716669+0000", 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6759999999999998e-06 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716676+0000", 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9759999999999998e-06 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.716681+0000", 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8899999999999998e-06 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.259 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.862882+0000", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.14620160300000001 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.862902+0000", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9701999999999999e-05 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.867467+0000", 2023-09-23T15:35:05.260 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0045648609999999999 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:32:11.867504+0000", 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.6900999999999998e-05 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:05.261 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:05.744 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:35:06.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:05 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/885641712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:05 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:05 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/885641712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:05 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:05 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/885641712' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:05 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:35:06.639 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[215646]: pgmap v1441: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 372 op/s 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[215646]: failed to encode map e492 with expected crc 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[215646]: failed to encode map e492 with expected crc 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[215646]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[222878]: pgmap v1441: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 372 op/s 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[222878]: failed to encode map e492 with expected crc 2023-09-23T15:35:06.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[222878]: failed to encode map e492 with expected crc 2023-09-23T15:35:06.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:35:06.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:06 smithi017 ceph-mon[222878]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T15:35:06.972 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:35:07.018 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:06 smithi027 ceph-mon[195233]: pgmap v1441: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.2 MiB/s rd, 1.7 MiB/s wr, 372 op/s 2023-09-23T15:35:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:06 smithi027 ceph-mon[195233]: failed to encode map e492 with expected crc 2023-09-23T15:35:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:06 smithi027 ceph-mon[195233]: failed to encode map e492 with expected crc 2023-09-23T15:35:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:06 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:35:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:06 smithi027 ceph-mon[195233]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T15:35:07.402 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:07.749 INFO:teuthology.orchestra.run.smithi017.stderr:noscrub is unset 2023-09-23T15:35:07.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:07 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:07.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:07 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:07.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:07 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T15:35:08.379 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:08] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2023-09-23T15:35:08.844 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T15:35:09.067 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[222878]: pgmap v1443: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:09.067 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[222878]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.067 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[222878]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.067 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:09.067 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[222878]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T15:35:09.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[215646]: pgmap v1443: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:09.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[215646]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[215646]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:09.068 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:08 smithi017 ceph-mon[215646]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T15:35:09.073 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:09.073 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:09.073 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:35:09.074 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:09.095 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:08 smithi027 ceph-mon[195233]: pgmap v1443: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:09.096 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:08 smithi027 ceph-mon[195233]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.096 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:08 smithi027 ceph-mon[195233]: failed to encode map e493 with expected crc 2023-09-23T15:35:09.097 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:08 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/468944005' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T15:35:09.097 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:08 smithi027 ceph-mon[195233]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T15:35:09.281 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:09.658 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T15:35:09.753 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:35:10.021 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1162061643' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1162061643' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1162061643' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: pgmap v1445: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:10.751 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[215646]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: pgmap v1445: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: failed to encode map e494 with expected crc 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:10.752 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:10 smithi017 ceph-mon[222878]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T15:35:10.919 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:35:10.919 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:35:11.032 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: pgmap v1445: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:11.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: failed to encode map e494 with expected crc 2023-09-23T15:35:11.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: failed to encode map e494 with expected crc 2023-09-23T15:35:11.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: failed to encode map e494 with expected crc 2023-09-23T15:35:11.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:11.033 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:10 smithi027 ceph-mon[195233]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T15:35:11.345 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:11.516 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:11.516 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:11.516 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:35:11.516 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:35:11.516 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:11.767 INFO:teuthology.orchestra.run.smithi017.stderr:nodeep-scrub is unset 2023-09-23T15:35:11.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T15:35:11.796 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3158486999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.797 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: failed to encode map e495 with expected crc 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3158486999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:11.798 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:11 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:12.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: failed to encode map e495 with expected crc 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3158486999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:12.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:11 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T15:35:12.190 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T15:35:12.555 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:13.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: pgmap v1448: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:13.023 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[215646]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: pgmap v1448: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.024 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:35:13.025 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:12 smithi017 ceph-mon[222878]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T15:35:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: pgmap v1448: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T15:35:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:35:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T15:35:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:12 smithi027 ceph-mon[195233]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T15:35:13.427 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:35:13.429 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:35:13.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[222878]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:13.765 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:13 smithi027 ceph-mon[195233]: failed to encode map e496 with expected crc 2023-09-23T15:35:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:13.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88043 9.e 9:7d9f7ec3:::benchmark_data_smithi017_543597_object4252:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:35:14.134 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 453.73534239399999, 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0278136939999998, 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88043 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.135 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395866+0000", 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.136 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395865+0000", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395870+0000", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1939999999999997e-06 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395882+0000", 2023-09-23T15:35:14.137 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2089999999999999e-05 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.395890+0000", 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0800000000000006e-06 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400157+0000", 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042668879999999999 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.138 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400169+0000", 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1718e-05 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397541+0000", 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99737234299999999 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:14.139 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397554+0000", 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.274e-05 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.409726+0000", 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.012172013 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.409887+0000", 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016131399999999999 2023-09-23T15:35:14.140 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.410042+0000", 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015405900000000001 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.415246+0000", 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052046540000000004 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.141 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418242+0000", 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029955680000000001 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423508+0000", 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052663409999999999 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423543+0000", 2023-09-23T15:35:14.142 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.464e-05 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.423680+0000", 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000137267 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.143 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88049 9.e 9:733405ce:::benchmark_data_smithi017_543597_object4258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 453.70972658099998, 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0045682330000001, 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88049 2023-09-23T15:35:14.144 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421482+0000", 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.145 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421481+0000", 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421487+0000", 2023-09-23T15:35:14.146 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5189999999999998e-06 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421501+0000", 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3557000000000001e-05 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421507+0000", 2023-09-23T15:35:14.147 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9089999999999997e-06 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421556+0000", 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9153000000000002e-05 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.421570+0000", 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3984e-05 2023-09-23T15:35:14.148 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397577+0000", 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.97600752700000004 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.397585+0000", 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5680000000000002e-06 2023-09-23T15:35:14.149 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.412942+0000", 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.015356929 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.413039+0000", 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6915000000000003e-05 2023-09-23T15:35:14.150 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.413128+0000", 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8647999999999996e-05 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418274+0000", 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051468340000000003 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.151 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.418285+0000", 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0795000000000001e-05 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.425895+0000", 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076100789999999996 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.152 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.425933+0000", 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7435000000000003e-05 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.426050+0000", 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011771499999999999 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.153 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:166451 10.1 10:871eff90:::benchmark_data_smithi017_604517_object60340:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:51.079658+0000", 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 143.05155044899999, 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24898743800000001, 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.154 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 166451 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079658+0000", 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.155 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079658+0000", 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079658+0000", 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.156 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079661+0000", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9739999999999998e-06 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079676+0000", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5003e-05 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079680+0000", 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2050000000000004e-06 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.157 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079740+0000", 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0368999999999997e-05 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079863+0000", 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000122228 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.158 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.079982+0000", 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119282 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.085382+0000", 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054001639999999998 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.159 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.087011+0000", 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001628483 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.328584+0000", 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24157382799999999 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.328620+0000", 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5277999999999997e-05 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.160 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:51.328646+0000", 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6044999999999999e-05 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:169303 10.e 10:713c7a24:::benchmark_data_smithi017_604517_object63192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e453)", 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:55.152473+0000", 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 138.97873538499999, 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.30071320000000001, 2023-09-23T15:35:14.161 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 169303 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152473+0000", 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.162 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152473+0000", 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152472+0000", 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.163 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152478+0000", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7710000000000001e-06 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152495+0000", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6260999999999999e-05 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152502+0000", 2023-09-23T15:35:14.164 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7920000000000004e-06 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152554+0000", 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.215e-05 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152669+0000", 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115752 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.165 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.152837+0000", 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016757499999999999 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.159099+0000", 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062617000000000003 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.166 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.163905+0000", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048067029999999998 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.453095+0000", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.28918969900000002 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.453158+0000", 2023-09-23T15:35:14.167 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2459999999999995e-05 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:55.453187+0000", 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.8982000000000001e-05 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:178433 10.1 10:8802bf53:::benchmark_data_smithi017_604517_object72322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:08.791583+0000", 2023-09-23T15:35:14.168 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 125.339625497, 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.238437016, 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178433 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791583+0000", 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.169 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791583+0000", 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791583+0000", 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.170 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791586+0000", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3220000000000001e-06 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791593+0000", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9589999999999998e-06 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791598+0000", 2023-09-23T15:35:14.171 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.507e-06 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791649+0000", 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0460000000000001e-05 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791778+0000", 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012938899999999999 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.172 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.791899+0000", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120642 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.795151+0000", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032515460000000001 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.797901+0000", 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027504890000000001 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.173 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.029918+0000", 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.232017219 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.029965+0000", 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6613999999999998e-05 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.174 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.030020+0000", 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5380000000000002e-05 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:178439 10.1 10:82e6493a:::benchmark_data_smithi017_604517_object72328:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:08.798436+0000", 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 125.33277304800001, 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23504418499999999, 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.175 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178439 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798436+0000", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798436+0000", 2023-09-23T15:35:14.176 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798434+0000", 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798440+0000", 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0979999999999999e-06 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.177 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798452+0000", 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1695999999999999e-05 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798458+0000", 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2750000000000001e-06 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798508+0000", 2023-09-23T15:35:14.178 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9277999999999998e-05 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798650+0000", 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014218400000000001 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798775+0000", 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012509199999999999 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.179 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.801806+0000", 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030310319999999999 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.804033+0000", 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002227559 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.033405+0000", 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.229371515 2023-09-23T15:35:14.180 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.033453+0000", 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8210000000000001e-05 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.033480+0000", 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6695000000000001e-05 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.181 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:178441 10.e 10:7cd7aac8:::benchmark_data_smithi017_604517_object72330:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e457)", 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:08.798492+0000", 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 125.33271669299999, 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.246802775, 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 178441 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.182 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798492+0000", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798492+0000", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798491+0000", 2023-09-23T15:35:14.183 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798495+0000", 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5449999999999999e-06 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798502+0000", 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3490000000000003e-06 2023-09-23T15:35:14.184 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.798507+0000", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5299999999999998e-06 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.799191+0000", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068446399999999997 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.799308+0000", 2023-09-23T15:35:14.185 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011676300000000001 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.799434+0000", 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000126403 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.802396+0000", 2023-09-23T15:35:14.186 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029611529999999998 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:08.802595+0000", 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000199843 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.045235+0000", 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24263929300000001 2023-09-23T15:35:14.187 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.045269+0000", 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.417e-05 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:09.045295+0000", 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5846999999999998e-05 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.188 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:182828 10.e 10:7ef620c5:::benchmark_data_smithi017_604517_object76717:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.066713+0000", 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.06449614500001, 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19936425999999999, 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182828 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.189 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066713+0000", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066713+0000", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066711+0000", 2023-09-23T15:35:14.190 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066718+0000", 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7770000000000002e-06 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066733+0000", 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.417e-05 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.191 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066739+0000", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4699999999999999e-06 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066777+0000", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.769e-05 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.066892+0000", 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115345 2023-09-23T15:35:14.192 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.067011+0000", 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011903199999999999 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.069944+0000", 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029325390000000001 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.193 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071121+0000", 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001177743 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.265986+0000", 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19486503699999999 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.266036+0000", 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9802000000000002e-05 2023-09-23T15:35:14.194 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.266077+0000", 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0565000000000002e-05 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:182831 10.e 10:7b29eee1:::benchmark_data_smithi017_604517_object76720:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.071436+0000", 2023-09-23T15:35:14.195 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.05977225700001, 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.194750274, 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182831 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071436+0000", 2023-09-23T15:35:14.196 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071436+0000", 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071435+0000", 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:14.197 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071441+0000", 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2700000000000004e-06 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071449+0000", 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4100000000000008e-06 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.198 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071455+0000", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9909999999999999e-06 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071503+0000", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7716999999999997e-05 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071614+0000", 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011157799999999999 2023-09-23T15:35:14.199 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255398+0000", 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.183783588 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.261787+0000", 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0063892640000000004 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.200 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.261795+0000", 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7989999999999994e-06 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.266114+0000", 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004319196 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.201 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.266155+0000", 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0884999999999999e-05 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.266187+0000", 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1557999999999998e-05 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.202 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:182834 10.1 10:8e12a0e9:::benchmark_data_smithi017_604517_object76723:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.071471+0000", 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.05973782700001, 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.199617718, 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182834 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.203 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071471+0000", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071471+0000", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071471+0000", 2023-09-23T15:35:14.204 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071473+0000", 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9129999999999999e-06 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071483+0000", 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1309999999999995e-06 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.205 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071487+0000", 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8860000000000003e-06 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071541+0000", 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3443999999999999e-05 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.206 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071620+0000", 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9332000000000005e-05 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071685+0000", 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4659000000000006e-05 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.267171+0000", 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19548599699999999 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.207 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.270007+0000", 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002835855 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.270999+0000", 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00099237900000000005 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.208 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.271049+0000", 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0005e-05 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.271089+0000", 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9501e-05 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:182839 10.1 10:8c1519ca:::benchmark_data_smithi017_604517_object76728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.071501+0000", 2023-09-23T15:35:14.209 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.059708128, 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20287317099999999, 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182839 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.210 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071501+0000", 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071501+0000", 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071500+0000", 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.211 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071503+0000", 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6699999999999998e-06 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071511+0000", 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0490000000000004e-06 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.212 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071515+0000", 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2969999999999997e-06 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.258348+0000", 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18683249199999999 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.258470+0000", 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012243900000000001 2023-09-23T15:35:14.213 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.258599+0000", 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012872799999999999 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.269974+0000", 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011375092999999999 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.214 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.270055+0000", 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0604000000000003e-05 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274282+0000", 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042279329999999997 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.215 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274331+0000", 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8461000000000002e-05 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274374+0000", 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2834999999999999e-05 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.216 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:182832 10.c e458/413 10:32290afe:::benchmark_data_smithi017_604517_object76721:head v 458'11465, mlcod=458'11463)", 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.071766+0000", 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.05944308799999, 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.197069998, 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071766+0000", 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.217 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071766+0000", 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071765+0000", 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.218 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.071770+0000", 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7520000000000004e-06 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255232+0000", 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18346203699999999 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.219 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255242+0000", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9129999999999998e-06 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255335+0000", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3595000000000001e-05 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255356+0000", 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0126999999999999e-05 2023-09-23T15:35:14.220 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.268793+0000", 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013437414 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.268836+0000", 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2688000000000002e-05 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.221 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:182843 10.1 10:87128a18:::benchmark_data_smithi017_604517_object76732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e458)", 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:14.078386+0000", 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 120.052822683, 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19642045299999999, 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.222 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 182843 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.078386+0000", 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.223 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.078386+0000", 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.078385+0000", 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.078390+0000", 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7149999999999997e-06 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.224 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255227+0000", 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17683699 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.255237+0000", 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0039e-05 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.225 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.264208+0000", 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0089708180000000002 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.264314+0000", 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000106457 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,7", 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.264420+0000", 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010596 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.226 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.270011+0000", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055903990000000002 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274408+0000", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043972940000000004 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274723+0000", 2023-09-23T15:35:14.227 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000315574 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274770+0000", 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6909000000000001e-05 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:14.274806+0000", 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6148e-05 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.228 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:208166 10.2 e465/413 10:4aa23d95:::benchmark_data_smithi017_604517_object102055:head v 465'13057, mlcod=465'13055)", 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:47.957785+0000", 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 86.173424079, 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.241293018, 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.229 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957785+0000", 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957785+0000", 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.230 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957784+0000", 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957790+0000", 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6470000000000001e-06 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957801+0000", 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0573e-05 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.231 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957807+0000", 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.485e-06 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957845+0000", 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8674000000000003e-05 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.232 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.957858+0000", 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2779000000000001e-05 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199029+0000", 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.24117129100000001 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199078+0000", 2023-09-23T15:35:14.233 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8368000000000002e-05 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:208172 10.e 10:76c2de94:::benchmark_data_smithi017_604517_object102061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e465)", 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:47.960057+0000", 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 86.171151512999998, 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.25163603499999998, 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.234 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 208172 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960057+0000", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.235 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960057+0000", 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960056+0000", 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960062+0000", 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7620000000000001e-06 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.236 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960073+0000", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.095e-05 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960078+0000", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0989999999999998e-06 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.202466+0000", 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.242388461 2023-09-23T15:35:14.237 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.202600+0000", 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013397800000000001 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.202724+0000", 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012378899999999999 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.238 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.210115+0000", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0073907840000000001 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.210131+0000", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5908000000000001e-05 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.211632+0000", 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0015011740000000001 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.239 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.211663+0000", 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1429e-05 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.211693+0000", 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9992e-05 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.240 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:208165 10.7 e465/463 10:ea6cc8e5:::benchmark_data_smithi017_604517_object102054:head v 465'12734, mlcod=465'12732)", 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:47.960473+0000", 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 86.170736163000001, 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.247813902, 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960473+0000", 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.241 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960473+0000", 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960472+0000", 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.242 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.960479+0000", 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.345e-06 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199025+0000", 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23854562200000001 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199035+0000", 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0611000000000001e-05 2023-09-23T15:35:14.243 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199149+0000", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011408000000000001 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199185+0000", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6241000000000001e-05 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.208246+0000", 2023-09-23T15:35:14.244 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0090610259999999998 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.208286+0000", 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9974e-05 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_repop(client.46520.0:208169 10.6 e465/451 10:630d0bb8:::benchmark_data_smithi017_604517_object102058:head v 465'12963, mlcod=465'12961)", 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:47.964756+0000", 2023-09-23T15:35:14.245 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 86.166452325999998, 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.243826878, 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.964756+0000", 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.964756+0000", 2023-09-23T15:35:14.246 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.964756+0000", 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.247 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:47.964763+0000", 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8620000000000003e-06 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199011+0000", 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.23424726300000001 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199030+0000", 2023-09-23T15:35:14.248 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9847000000000002e-05 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199077+0000", 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6400999999999998e-05 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.199104+0000", 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7035000000000001e-05 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.249 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.208547+0000", 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094431270000000008 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:48.208583+0000", 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6210000000000001e-05 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.250 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:211402 10.e 10:7006f0f2:::benchmark_data_smithi017_604517_object105291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:52.113736+0000", 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.017472689000002, 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22181812100000001, 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 211402 2023-09-23T15:35:14.251 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113736+0000", 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113736+0000", 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.252 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113736+0000", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113739+0000", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2990000000000001e-06 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113745+0000", 2023-09-23T15:35:14.253 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4520000000000002e-06 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.113749+0000", 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1740000000000002e-06 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.118615+0000", 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0048657250000000004 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.254 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.118722+0000", 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107167 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.118802+0000", 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9765999999999995e-05 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.255 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.123576+0000", 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0047742590000000003 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.137258+0000", 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013681929000000001 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.335320+0000", 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.19806153700000001 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.256 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.335502+0000", 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000182647 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.335554+0000", 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1647999999999997e-05 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.257 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:211406 10.e 10:73b80516:::benchmark_data_smithi017_604517_object105295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:52.120462+0000", 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.010747159000005, 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.223012821, 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 211406 2023-09-23T15:35:14.258 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120462+0000", 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120462+0000", 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.259 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.260 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120460+0000", 2023-09-23T15:35:14.260 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:14.260 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.260 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.260 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.261 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120468+0000", 2023-09-23T15:35:14.261 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.208e-06 2023-09-23T15:35:14.261 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120481+0000", 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3121e-05 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.120488+0000", 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8560000000000001e-06 2023-09-23T15:35:14.262 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.121192+0000", 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070366700000000005 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.121277+0000", 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5817999999999997e-05 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.263 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.121363+0000", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5729000000000005e-05 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.123721+0000", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0023579109999999999 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.137771+0000", 2023-09-23T15:35:14.264 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014050013 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343385+0000", 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.205614042 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343434+0000", 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8618000000000001e-05 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.265 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343474+0000", 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0734e-05 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:211409 10.e 10:70d54cce:::benchmark_data_smithi017_604517_object105298:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e466)", 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:52.125760+0000", 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 82.005449158000005, 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21780743699999999, 2023-09-23T15:35:14.266 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 211409 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125760+0000", 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.267 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125760+0000", 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125757+0000", 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.268 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125766+0000", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4239999999999993e-06 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125781+0000", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5495e-05 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125790+0000", 2023-09-23T15:35:14.269 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1449999999999997e-06 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125842+0000", 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1941000000000003e-05 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.125963+0000", 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120647 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.270 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,6", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.126091+0000", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000128157 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.132739+0000", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.006648358 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.142345+0000", 2023-09-23T15:35:14.271 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0096056240000000001 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343507+0000", 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.20116154 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343534+0000", 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7223e-05 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:14.272 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:52.343567+0000", 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3194000000000001e-05 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:14.273 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:14.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[215646]: failed to encode map e496 with expected crc 2023-09-23T15:35:14.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:14.274 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:14.275 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1450,"stamp":"2023-09-23T15:35:13.067038+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11973,"num_read_kb":26407,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":4204721,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6998,"ondisk_log_size":6998,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4207276,"kb_used_data":66292,"kb_used_omap":57,"kb_used_meta":4140870,"kb_avail":652008788,"statfs":{"total":671965249536,"available":667656998912,"internally_reserved":0,"allocated":67883008,"data_stored":54026564,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4240250905},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.324011"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":602,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755528+0000","last_change":"2023-09-23T15:33:08.496898+0000","last_active":"2023-09-23T15:35:06.755528+0000","last_peered":"2023-09-23T15:35:06.755528+0000","last_clean":"2023-09-23T15:35:06.755528+0000","last_became_active":"2023-09-23T15:32:18.611133+0000","last_became_peered":"2023-09-23T15:32:18.611133+0000","last_unstale":"2023-09-23T15:35:06.755528+0000","last_undegraded":"2023-09-23T15:35:06.755528+0000","last_fullsized":"2023-09-23T15:35:06.755528+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:08.496746+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:08.496746+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:53.844651+0000","scrub_duration":0.0035525999999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742426+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:35:05.742426+0000","last_peered":"2023-09-23T15:35:05.742426+0000","last_clean":"2023-09-23T15:35:05.742426+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:35:05.742426+0000","last_undegraded":"2023-09-23T15:35:05.742426+0000","last_fullsized":"2023-09-23T15:35:05.742426+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019761+0000","last_change":"2023-09-23T15:32:16.240512+0000","last_active":"2023-09-23T15:35:10.019761+0000","last_peered":"2023-09-23T15:35:10.019761+0000","last_clean":"2023-09-23T15:35:10.019761+0000","last_became_active":"2023-09-23T15:32:16.240101+0000","last_became_peered":"2023-09-23T15:32:16.240101+0000","last_unstale":"2023-09-23T15:35:10.019761+0000","last_undegraded":"2023-09-23T15:35:10.019761+0000","last_fullsized":"2023-09-23T15:35:10.019761+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:08.260986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":553,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742580+0000","last_change":"2023-09-23T15:33:05.987466+0000","last_active":"2023-09-23T15:35:05.742580+0000","last_peered":"2023-09-23T15:35:05.742580+0000","last_clean":"2023-09-23T15:35:05.742580+0000","last_became_active":"2023-09-23T15:32:18.614293+0000","last_became_peered":"2023-09-23T15:32:18.614293+0000","last_unstale":"2023-09-23T15:35:05.742580+0000","last_undegraded":"2023-09-23T15:35:05.742580+0000","last_fullsized":"2023-09-23T15:35:05.742580+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.987399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.987399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:35.508777+0000","scrub_duration":0.024984823,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":422,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018519+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:35:10.018519+0000","last_peered":"2023-09-23T15:35:10.018519+0000","last_clean":"2023-09-23T15:35:10.018519+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:35:10.018519+0000","last_undegraded":"2023-09-23T15:35:10.018519+0000","last_fullsized":"2023-09-23T15:35:10.018519+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":490,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090700+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:35:03.090700+0000","last_peered":"2023-09-23T15:35:03.090700+0000","last_clean":"2023-09-23T15:35:03.090700+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:35:03.090700+0000","last_undegraded":"2023-09-23T15:35:03.090700+0000","last_fullsized":"2023-09-23T15:35:03.090700+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":628,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742558+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:35:05.742558+0000","last_peered":"2023-09-23T15:35:05.742558+0000","last_clean":"2023-09-23T15:35:05.742558+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:35:05.742558+0000","last_undegraded":"2023-09-23T15:35:05.742558+0000","last_fullsized":"2023-09-23T15:35:05.742558+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":429,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090223+0000","last_change":"2023-09-23T15:30:26.692470+0000","last_active":"2023-09-23T15:35:03.090223+0000","last_peered":"2023-09-23T15:35:03.090223+0000","last_clean":"2023-09-23T15:35:03.090223+0000","last_became_active":"2023-09-23T15:30:26.692148+0000","last_became_peered":"2023-09-23T15:30:26.692148+0000","last_unstale":"2023-09-23T15:35:03.090223+0000","last_undegraded":"2023-09-23T15:35:03.090223+0000","last_fullsized":"2023-09-23T15:35:03.090223+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:32:36.258289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":453,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744895+0000","last_change":"2023-09-23T15:32:16.240747+0000","last_active":"2023-09-23T15:35:05.744895+0000","last_peered":"2023-09-23T15:35:05.744895+0000","last_clean":"2023-09-23T15:35:05.744895+0000","last_became_active":"2023-09-23T15:32:16.240381+0000","last_became_peered":"2023-09-23T15:32:16.240381+0000","last_unstale":"2023-09-23T15:35:05.744895+0000","last_undegraded":"2023-09-23T15:35:05.744895+0000","last_fullsized":"2023-09-23T15:35:05.744895+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:43.105398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":781,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755808+0000","last_change":"2023-09-23T15:32:18.619160+0000","last_active":"2023-09-23T15:35:06.755808+0000","last_peered":"2023-09-23T15:35:06.755808+0000","last_clean":"2023-09-23T15:35:06.755808+0000","last_became_active":"2023-09-23T15:32:18.618856+0000","last_became_peered":"2023-09-23T15:32:18.618856+0000","last_unstale":"2023-09-23T15:35:06.755808+0000","last_undegraded":"2023-09-23T15:35:06.755808+0000","last_fullsized":"2023-09-23T15:35:06.755808+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:08:05.782217+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":452,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090832+0000","last_change":"2023-09-23T15:32:16.242054+0000","last_active":"2023-09-23T15:35:03.090832+0000","last_peered":"2023-09-23T15:35:03.090832+0000","last_clean":"2023-09-23T15:35:03.090832+0000","last_became_active":"2023-09-23T15:32:16.241626+0000","last_became_peered":"2023-09-23T15:32:16.241626+0000","last_unstale":"2023-09-23T15:35:03.090832+0000","last_undegraded":"2023-09-23T15:35:03.090832+0000","last_fullsized":"2023-09-23T15:35:03.090832+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:25.373723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":430,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018888+0000","last_change":"2023-09-23T15:30:26.706619+0000","last_active":"2023-09-23T15:35:10.018888+0000","last_peered":"2023-09-23T15:35:10.018888+0000","last_clean":"2023-09-23T15:35:10.018888+0000","last_became_active":"2023-09-23T15:30:26.706343+0000","last_became_peered":"2023-09-23T15:30:26.706343+0000","last_unstale":"2023-09-23T15:35:10.018888+0000","last_undegraded":"2023-09-23T15:35:10.018888+0000","last_fullsized":"2023-09-23T15:35:10.018888+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:47.811881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":903,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755881+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:35:06.755881+0000","last_peered":"2023-09-23T15:35:06.755881+0000","last_clean":"2023-09-23T15:35:06.755881+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:35:06.755881+0000","last_undegraded":"2023-09-23T15:35:06.755881+0000","last_fullsized":"2023-09-23T15:35:06.755881+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":446,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756724+0000","last_change":"2023-09-23T15:32:16.248918+0000","last_active":"2023-09-23T15:35:06.756724+0000","last_peered":"2023-09-23T15:35:06.756724+0000","last_clean":"2023-09-23T15:35:06.756724+0000","last_became_active":"2023-09-23T15:32:16.248407+0000","last_became_peered":"2023-09-23T15:32:16.248407+0000","last_unstale":"2023-09-23T15:35:06.756724+0000","last_undegraded":"2023-09-23T15:35:06.756724+0000","last_fullsized":"2023-09-23T15:35:06.756724+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:08.191600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":399,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755579+0000","last_change":"2023-09-23T15:30:26.703714+0000","last_active":"2023-09-23T15:35:06.755579+0000","last_peered":"2023-09-23T15:35:06.755579+0000","last_clean":"2023-09-23T15:35:06.755579+0000","last_became_active":"2023-09-23T15:30:26.703402+0000","last_became_peered":"2023-09-23T15:30:26.703402+0000","last_unstale":"2023-09-23T15:35:06.755579+0000","last_undegraded":"2023-09-23T15:35:06.755579+0000","last_fullsized":"2023-09-23T15:35:06.755579+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:48.014628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":566,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090723+0000","last_change":"2023-09-23T15:33:06.824993+0000","last_active":"2023-09-23T15:35:03.090723+0000","last_peered":"2023-09-23T15:35:03.090723+0000","last_clean":"2023-09-23T15:35:03.090723+0000","last_became_active":"2023-09-23T15:32:18.639717+0000","last_became_peered":"2023-09-23T15:32:18.639717+0000","last_unstale":"2023-09-23T15:35:03.090723+0000","last_undegraded":"2023-09-23T15:35:03.090723+0000","last_fullsized":"2023-09-23T15:35:03.090723+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:06.824864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:06.824864+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.202322+0000","scrub_duration":0.0065159160000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":238,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756733+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:35:06.756733+0000","last_peered":"2023-09-23T15:35:06.756733+0000","last_clean":"2023-09-23T15:35:06.756733+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:35:06.756733+0000","last_undegraded":"2023-09-23T15:35:06.756733+0000","last_fullsized":"2023-09-23T15:35:06.756733+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":390,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020247+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:35:10.020247+0000","last_peered":"2023-09-23T15:35:10.020247+0000","last_clean":"2023-09-23T15:35:10.020247+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:35:10.020247+0000","last_undegraded":"2023-09-23T15:35:10.020247+0000","last_fullsized":"2023-09-23T15:35:10.020247+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":647,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020847+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:35:10.020847+0000","last_peered":"2023-09-23T15:35:10.020847+0000","last_clean":"2023-09-23T15:35:10.020847+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:35:10.020847+0000","last_undegraded":"2023-09-23T15:35:10.020847+0000","last_fullsized":"2023-09-23T15:35:10.020847+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":159,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743051+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:35:05.743051+0000","last_peered":"2023-09-23T15:35:05.743051+0000","last_clean":"2023-09-23T15:35:05.743051+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:35:05.743051+0000","last_undegraded":"2023-09-23T15:35:05.743051+0000","last_fullsized":"2023-09-23T15:35:05.743051+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743053+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:35:05.743053+0000","last_peered":"2023-09-23T15:35:05.743053+0000","last_clean":"2023-09-23T15:35:05.743053+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:35:05.743053+0000","last_undegraded":"2023-09-23T15:35:05.743053+0000","last_fullsized":"2023-09-23T15:35:05.743053+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":822,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090325+0000","last_change":"2023-09-23T15:32:18.621224+0000","last_active":"2023-09-23T15:35:03.090325+0000","last_peered":"2023-09-23T15:35:03.090325+0000","last_clean":"2023-09-23T15:35:03.090325+0000","last_became_active":"2023-09-23T15:32:18.620853+0000","last_became_peered":"2023-09-23T15:32:18.620853+0000","last_unstale":"2023-09-23T15:35:03.090325+0000","last_undegraded":"2023-09-23T15:35:03.090325+0000","last_fullsized":"2023-09-23T15:35:03.090325+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:48.423947+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":242,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756342+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:35:06.756342+0000","last_peered":"2023-09-23T15:35:06.756342+0000","last_clean":"2023-09-23T15:35:06.756342+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:35:06.756342+0000","last_undegraded":"2023-09-23T15:35:06.756342+0000","last_fullsized":"2023-09-23T15:35:06.756342+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":42,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755783+0000","last_change":"2023-09-23T15:33:30.765277+0000","last_active":"2023-09-23T15:35:06.755783+0000","last_peered":"2023-09-23T15:35:06.755783+0000","last_clean":"2023-09-23T15:35:06.755783+0000","last_became_active":"2023-09-23T15:33:30.764903+0000","last_became_peered":"2023-09-23T15:33:30.764903+0000","last_unstale":"2023-09-23T15:35:06.755783+0000","last_undegraded":"2023-09-23T15:35:06.755783+0000","last_fullsized":"2023-09-23T15:35:06.755783+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:24:08.782060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":691,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755749+0000","last_change":"2023-09-23T15:32:18.640390+0000","last_active":"2023-09-23T15:35:06.755749+0000","last_peered":"2023-09-23T15:35:06.755749+0000","last_clean":"2023-09-23T15:35:06.755749+0000","last_became_active":"2023-09-23T15:32:18.640018+0000","last_became_peered":"2023-09-23T15:32:18.640018+0000","last_unstale":"2023-09-23T15:35:06.755749+0000","last_undegraded":"2023-09-23T15:35:06.755749+0000","last_fullsized":"2023-09-23T15:35:06.755749+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:30.211725+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":488,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090231+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:35:03.090231+0000","last_peered":"2023-09-23T15:35:03.090231+0000","last_clean":"2023-09-23T15:35:03.090231+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:35:03.090231+0000","last_undegraded":"2023-09-23T15:35:03.090231+0000","last_fullsized":"2023-09-23T15:35:03.090231+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":431,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756915+0000","last_change":"2023-09-23T15:32:16.258406+0000","last_active":"2023-09-23T15:35:06.756915+0000","last_peered":"2023-09-23T15:35:06.756915+0000","last_clean":"2023-09-23T15:35:06.756915+0000","last_became_active":"2023-09-23T15:32:16.257317+0000","last_became_peered":"2023-09-23T15:32:16.257317+0000","last_unstale":"2023-09-23T15:35:06.756915+0000","last_undegraded":"2023-09-23T15:35:06.756915+0000","last_fullsized":"2023-09-23T15:35:06.756915+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:40.361415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":823,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019408+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:35:10.019408+0000","last_peered":"2023-09-23T15:35:10.019408+0000","last_clean":"2023-09-23T15:35:10.019408+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:35:10.019408+0000","last_undegraded":"2023-09-23T15:35:10.019408+0000","last_fullsized":"2023-09-23T15:35:10.019408+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":236,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756292+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:35:06.756292+0000","last_peered":"2023-09-23T15:35:06.756292+0000","last_clean":"2023-09-23T15:35:06.756292+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:35:06.756292+0000","last_undegraded":"2023-09-23T15:35:06.756292+0000","last_fullsized":"2023-09-23T15:35:06.756292+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":476,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090755+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:35:03.090755+0000","last_peered":"2023-09-23T15:35:03.090755+0000","last_clean":"2023-09-23T15:35:03.090755+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:35:03.090755+0000","last_undegraded":"2023-09-23T15:35:03.090755+0000","last_fullsized":"2023-09-23T15:35:03.090755+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":529,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019871+0000","last_change":"2023-09-23T15:32:18.637941+0000","last_active":"2023-09-23T15:35:10.019871+0000","last_peered":"2023-09-23T15:35:10.019871+0000","last_clean":"2023-09-23T15:35:10.019871+0000","last_became_active":"2023-09-23T15:32:18.637624+0000","last_became_peered":"2023-09-23T15:32:18.637624+0000","last_unstale":"2023-09-23T15:35:10.019871+0000","last_undegraded":"2023-09-23T15:35:10.019871+0000","last_fullsized":"2023-09-23T15:35:10.019871+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:45:17.385979+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":449,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742526+0000","last_change":"2023-09-23T15:32:16.261689+0000","last_active":"2023-09-23T15:35:05.742526+0000","last_peered":"2023-09-23T15:35:05.742526+0000","last_clean":"2023-09-23T15:35:05.742526+0000","last_became_active":"2023-09-23T15:32:16.251466+0000","last_became_peered":"2023-09-23T15:32:16.251466+0000","last_unstale":"2023-09-23T15:35:05.742526+0000","last_undegraded":"2023-09-23T15:35:05.742526+0000","last_fullsized":"2023-09-23T15:35:05.742526+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:44.874231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":449,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755889+0000","last_change":"2023-09-23T15:30:26.706427+0000","last_active":"2023-09-23T15:35:06.755889+0000","last_peered":"2023-09-23T15:35:06.755889+0000","last_clean":"2023-09-23T15:35:06.755889+0000","last_became_active":"2023-09-23T15:30:26.706086+0000","last_became_peered":"2023-09-23T15:30:26.706086+0000","last_unstale":"2023-09-23T15:35:06.755889+0000","last_undegraded":"2023-09-23T15:35:06.755889+0000","last_fullsized":"2023-09-23T15:35:06.755889+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:42.578372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":789,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756832+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:35:06.756832+0000","last_peered":"2023-09-23T15:35:06.756832+0000","last_clean":"2023-09-23T15:35:06.756832+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:35:06.756832+0000","last_undegraded":"2023-09-23T15:35:06.756832+0000","last_fullsized":"2023-09-23T15:35:06.756832+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":232,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756859+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:35:06.756859+0000","last_peered":"2023-09-23T15:35:06.756859+0000","last_clean":"2023-09-23T15:35:06.756859+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:35:06.756859+0000","last_undegraded":"2023-09-23T15:35:06.756859+0000","last_fullsized":"2023-09-23T15:35:06.756859+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":438,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755915+0000","last_change":"2023-09-23T15:32:16.256834+0000","last_active":"2023-09-23T15:35:06.755915+0000","last_peered":"2023-09-23T15:35:06.755915+0000","last_clean":"2023-09-23T15:35:06.755915+0000","last_became_active":"2023-09-23T15:32:16.256515+0000","last_became_peered":"2023-09-23T15:32:16.256515+0000","last_unstale":"2023-09-23T15:35:06.755915+0000","last_undegraded":"2023-09-23T15:35:06.755915+0000","last_fullsized":"2023-09-23T15:35:06.755915+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:09.264970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":661,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090504+0000","last_change":"2023-09-23T15:30:48.890337+0000","last_active":"2023-09-23T15:35:03.090504+0000","last_peered":"2023-09-23T15:35:03.090504+0000","last_clean":"2023-09-23T15:35:03.090504+0000","last_became_active":"2023-09-23T15:30:26.693580+0000","last_became_peered":"2023-09-23T15:30:26.693580+0000","last_unstale":"2023-09-23T15:35:03.090504+0000","last_undegraded":"2023-09-23T15:35:03.090504+0000","last_fullsized":"2023-09-23T15:35:03.090504+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:25.353085+0000","scrub_duration":0.011955009000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":232,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756643+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:35:06.756643+0000","last_peered":"2023-09-23T15:35:06.756643+0000","last_clean":"2023-09-23T15:35:06.756643+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:35:06.756643+0000","last_undegraded":"2023-09-23T15:35:06.756643+0000","last_fullsized":"2023-09-23T15:35:06.756643+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":145,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742667+0000","last_change":"2023-09-23T15:33:30.765181+0000","last_active":"2023-09-23T15:35:05.742667+0000","last_peered":"2023-09-23T15:35:05.742667+0000","last_clean":"2023-09-23T15:35:05.742667+0000","last_became_active":"2023-09-23T15:33:30.764802+0000","last_became_peered":"2023-09-23T15:33:30.764802+0000","last_unstale":"2023-09-23T15:35:05.742667+0000","last_undegraded":"2023-09-23T15:35:05.742667+0000","last_fullsized":"2023-09-23T15:35:05.742667+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:36:15.536014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":760,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744827+0000","last_change":"2023-09-23T15:32:18.654963+0000","last_active":"2023-09-23T15:35:05.744827+0000","last_peered":"2023-09-23T15:35:05.744827+0000","last_clean":"2023-09-23T15:35:05.744827+0000","last_became_active":"2023-09-23T15:32:18.654586+0000","last_became_peered":"2023-09-23T15:32:18.654586+0000","last_unstale":"2023-09-23T15:35:05.744827+0000","last_undegraded":"2023-09-23T15:35:05.744827+0000","last_fullsized":"2023-09-23T15:35:05.744827+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:47.851804+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":234,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756478+0000","last_change":"2023-09-23T15:32:16.278269+0000","last_active":"2023-09-23T15:35:06.756478+0000","last_peered":"2023-09-23T15:35:06.756478+0000","last_clean":"2023-09-23T15:35:06.756478+0000","last_became_active":"2023-09-23T15:32:16.256883+0000","last_became_peered":"2023-09-23T15:32:16.256883+0000","last_unstale":"2023-09-23T15:35:06.756478+0000","last_undegraded":"2023-09-23T15:35:06.756478+0000","last_fullsized":"2023-09-23T15:35:06.756478+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:24.023700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":473,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744252+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:35:05.744252+0000","last_peered":"2023-09-23T15:35:05.744252+0000","last_clean":"2023-09-23T15:35:05.744252+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:35:05.744252+0000","last_undegraded":"2023-09-23T15:35:05.744252+0000","last_fullsized":"2023-09-23T15:35:05.744252+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":710,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743309+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:35:05.743309+0000","last_peered":"2023-09-23T15:35:05.743309+0000","last_clean":"2023-09-23T15:35:05.743309+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:35:05.743309+0000","last_undegraded":"2023-09-23T15:35:05.743309+0000","last_fullsized":"2023-09-23T15:35:05.743309+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":335,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744547+0000","last_change":"2023-09-23T15:32:16.251305+0000","last_active":"2023-09-23T15:35:05.744547+0000","last_peered":"2023-09-23T15:35:05.744547+0000","last_clean":"2023-09-23T15:35:05.744547+0000","last_became_active":"2023-09-23T15:32:16.250667+0000","last_became_peered":"2023-09-23T15:32:16.250667+0000","last_unstale":"2023-09-23T15:35:05.744547+0000","last_undegraded":"2023-09-23T15:35:05.744547+0000","last_fullsized":"2023-09-23T15:35:05.744547+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:38:30.961785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":495,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090372+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:35:03.090372+0000","last_peered":"2023-09-23T15:35:03.090372+0000","last_clean":"2023-09-23T15:35:03.090372+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:35:03.090372+0000","last_undegraded":"2023-09-23T15:35:03.090372+0000","last_fullsized":"2023-09-23T15:35:03.090372+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":745,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756197+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:35:06.756197+0000","last_peered":"2023-09-23T15:35:06.756197+0000","last_clean":"2023-09-23T15:35:06.756197+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:35:06.756197+0000","last_undegraded":"2023-09-23T15:35:06.756197+0000","last_fullsized":"2023-09-23T15:35:06.756197+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":146,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090398+0000","last_change":"2023-09-23T15:30:26.690336+0000","last_active":"2023-09-23T15:35:03.090398+0000","last_peered":"2023-09-23T15:35:03.090398+0000","last_clean":"2023-09-23T15:35:03.090398+0000","last_became_active":"2023-09-23T15:30:26.690141+0000","last_became_peered":"2023-09-23T15:30:26.690141+0000","last_unstale":"2023-09-23T15:35:03.090398+0000","last_undegraded":"2023-09-23T15:35:03.090398+0000","last_fullsized":"2023-09-23T15:35:03.090398+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:44.578933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742903+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:35:05.742903+0000","last_peered":"2023-09-23T15:35:05.742903+0000","last_clean":"2023-09-23T15:35:05.742903+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:35:05.742903+0000","last_undegraded":"2023-09-23T15:35:05.742903+0000","last_fullsized":"2023-09-23T15:35:05.742903+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":598,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755355+0000","last_change":"2023-09-23T15:33:05.479356+0000","last_active":"2023-09-23T15:35:06.755355+0000","last_peered":"2023-09-23T15:35:06.755355+0000","last_clean":"2023-09-23T15:35:06.755355+0000","last_became_active":"2023-09-23T15:32:18.637504+0000","last_became_peered":"2023-09-23T15:32:18.637504+0000","last_unstale":"2023-09-23T15:35:06.755355+0000","last_undegraded":"2023-09-23T15:35:06.755355+0000","last_fullsized":"2023-09-23T15:35:06.755355+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:05.479226+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.479226+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:39.341195+0000","scrub_duration":0.0055363749999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":460,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744273+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:35:05.744273+0000","last_peered":"2023-09-23T15:35:05.744273+0000","last_clean":"2023-09-23T15:35:05.744273+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:35:05.744273+0000","last_undegraded":"2023-09-23T15:35:05.744273+0000","last_fullsized":"2023-09-23T15:35:05.744273+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743219+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:35:05.743219+0000","last_peered":"2023-09-23T15:35:05.743219+0000","last_clean":"2023-09-23T15:35:05.743219+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:35:05.743219+0000","last_undegraded":"2023-09-23T15:35:05.743219+0000","last_fullsized":"2023-09-23T15:35:05.743219+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":794,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019316+0000","last_change":"2023-09-23T15:32:18.615573+0000","last_active":"2023-09-23T15:35:10.019316+0000","last_peered":"2023-09-23T15:35:10.019316+0000","last_clean":"2023-09-23T15:35:10.019316+0000","last_became_active":"2023-09-23T15:32:18.615232+0000","last_became_peered":"2023-09-23T15:32:18.615232+0000","last_unstale":"2023-09-23T15:35:10.019316+0000","last_undegraded":"2023-09-23T15:35:10.019316+0000","last_fullsized":"2023-09-23T15:35:10.019316+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:11.272034+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":149,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742683+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:35:05.742683+0000","last_peered":"2023-09-23T15:35:05.742683+0000","last_clean":"2023-09-23T15:35:05.742683+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:35:05.742683+0000","last_undegraded":"2023-09-23T15:35:05.742683+0000","last_fullsized":"2023-09-23T15:35:05.742683+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":145,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743334+0000","last_change":"2023-09-23T15:32:16.253887+0000","last_active":"2023-09-23T15:35:05.743334+0000","last_peered":"2023-09-23T15:35:05.743334+0000","last_clean":"2023-09-23T15:35:05.743334+0000","last_became_active":"2023-09-23T15:32:16.253556+0000","last_became_peered":"2023-09-23T15:32:16.253556+0000","last_unstale":"2023-09-23T15:35:05.743334+0000","last_undegraded":"2023-09-23T15:35:05.743334+0000","last_fullsized":"2023-09-23T15:35:05.743334+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:16.930439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":593,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755369+0000","last_change":"2023-09-23T15:33:30.357966+0000","last_active":"2023-09-23T15:35:06.755369+0000","last_peered":"2023-09-23T15:35:06.755369+0000","last_clean":"2023-09-23T15:35:06.755369+0000","last_became_active":"2023-09-23T15:32:18.637860+0000","last_became_peered":"2023-09-23T15:32:18.637860+0000","last_unstale":"2023-09-23T15:35:06.755369+0000","last_undegraded":"2023-09-23T15:35:06.755369+0000","last_fullsized":"2023-09-23T15:35:06.755369+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:30.357860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:30.357860+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:04.776475+0000","scrub_duration":0.0015641730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":463,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018813+0000","last_change":"2023-09-23T15:30:26.705171+0000","last_active":"2023-09-23T15:35:10.018813+0000","last_peered":"2023-09-23T15:35:10.018813+0000","last_clean":"2023-09-23T15:35:10.018813+0000","last_became_active":"2023-09-23T15:30:26.704853+0000","last_became_peered":"2023-09-23T15:30:26.704853+0000","last_unstale":"2023-09-23T15:35:10.018813+0000","last_undegraded":"2023-09-23T15:35:10.018813+0000","last_fullsized":"2023-09-23T15:35:10.018813+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:11.875343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":431,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021970+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:35:10.021970+0000","last_peered":"2023-09-23T15:35:10.021970+0000","last_clean":"2023-09-23T15:35:10.021970+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:35:10.021970+0000","last_undegraded":"2023-09-23T15:35:10.021970+0000","last_fullsized":"2023-09-23T15:35:10.021970+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":733,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744833+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:35:05.744833+0000","last_peered":"2023-09-23T15:35:05.744833+0000","last_clean":"2023-09-23T15:35:05.744833+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:35:05.744833+0000","last_undegraded":"2023-09-23T15:35:05.744833+0000","last_fullsized":"2023-09-23T15:35:05.744833+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"488'163","reported_seq":1015,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020669+0000","last_change":"2023-09-23T15:34:54.709256+0000","last_active":"2023-09-23T15:35:10.020669+0000","last_peered":"2023-09-23T15:35:10.020669+0000","last_clean":"2023-09-23T15:35:10.020669+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:35:10.020669+0000","last_undegraded":"2023-09-23T15:35:10.020669+0000","last_fullsized":"2023-09-23T15:35:10.020669+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021979399999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":726,"num_read_kb":1976,"num_write":174,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755809+0000","last_change":"2023-09-23T15:32:16.240883+0000","last_active":"2023-09-23T15:35:06.755809+0000","last_peered":"2023-09-23T15:35:06.755809+0000","last_clean":"2023-09-23T15:35:06.755809+0000","last_became_active":"2023-09-23T15:32:16.240541+0000","last_became_peered":"2023-09-23T15:32:16.240541+0000","last_unstale":"2023-09-23T15:35:06.755809+0000","last_undegraded":"2023-09-23T15:35:06.755809+0000","last_fullsized":"2023-09-23T15:35:06.755809+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:05.987331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":467,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755869+0000","last_change":"2023-09-23T15:32:16.239137+0000","last_active":"2023-09-23T15:35:06.755869+0000","last_peered":"2023-09-23T15:35:06.755869+0000","last_clean":"2023-09-23T15:35:06.755869+0000","last_became_active":"2023-09-23T15:32:16.238791+0000","last_became_peered":"2023-09-23T15:32:16.238791+0000","last_unstale":"2023-09-23T15:35:06.755869+0000","last_undegraded":"2023-09-23T15:35:06.755869+0000","last_fullsized":"2023-09-23T15:35:06.755869+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:51.104255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":590,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090363+0000","last_change":"2023-09-23T15:33:05.797574+0000","last_active":"2023-09-23T15:35:03.090363+0000","last_peered":"2023-09-23T15:35:03.090363+0000","last_clean":"2023-09-23T15:35:03.090363+0000","last_became_active":"2023-09-23T15:32:18.631881+0000","last_became_peered":"2023-09-23T15:32:18.631881+0000","last_unstale":"2023-09-23T15:35:03.090363+0000","last_undegraded":"2023-09-23T15:35:03.090363+0000","last_fullsized":"2023-09-23T15:35:03.090363+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.797446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.797446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:23.420392+0000","scrub_duration":0.0090775909999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"488'157","reported_seq":1122,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020107+0000","last_change":"2023-09-23T15:34:54.709240+0000","last_active":"2023-09-23T15:35:10.020107+0000","last_peered":"2023-09-23T15:35:10.020107+0000","last_clean":"2023-09-23T15:35:10.020107+0000","last_became_active":"2023-09-23T15:32:16.240946+0000","last_became_peered":"2023-09-23T15:32:16.240946+0000","last_unstale":"2023-09-23T15:35:10.020107+0000","last_undegraded":"2023-09-23T15:35:10.020107+0000","last_fullsized":"2023-09-23T15:35:10.020107+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:38.182103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033387800000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1406,"num_read_kb":7738,"num_write":130,"num_write_kb":5863,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742880+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:35:05.742880+0000","last_peered":"2023-09-23T15:35:05.742880+0000","last_clean":"2023-09-23T15:35:05.742880+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:35:05.742880+0000","last_undegraded":"2023-09-23T15:35:05.742880+0000","last_fullsized":"2023-09-23T15:35:05.742880+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":396,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020454+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:35:10.020454+0000","last_peered":"2023-09-23T15:35:10.020454+0000","last_clean":"2023-09-23T15:35:10.020454+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:35:10.020454+0000","last_undegraded":"2023-09-23T15:35:10.020454+0000","last_fullsized":"2023-09-23T15:35:10.020454+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":856,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756421+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:35:06.756421+0000","last_peered":"2023-09-23T15:35:06.756421+0000","last_clean":"2023-09-23T15:35:06.756421+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:35:06.756421+0000","last_undegraded":"2023-09-23T15:35:06.756421+0000","last_fullsized":"2023-09-23T15:35:06.756421+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"488'400","reported_seq":1919,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755958+0000","last_change":"2023-09-23T15:34:54.705587+0000","last_active":"2023-09-23T15:35:06.755958+0000","last_peered":"2023-09-23T15:35:06.755958+0000","last_clean":"2023-09-23T15:35:06.755958+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:35:06.755958+0000","last_undegraded":"2023-09-23T15:35:06.755958+0000","last_fullsized":"2023-09-23T15:35:06.755958+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047212799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2483,"num_read_kb":3714,"num_write":351,"num_write_kb":5627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":392,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018974+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:35:10.018974+0000","last_peered":"2023-09-23T15:35:10.018974+0000","last_clean":"2023-09-23T15:35:10.018974+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:35:10.018974+0000","last_undegraded":"2023-09-23T15:35:10.018974+0000","last_fullsized":"2023-09-23T15:35:10.018974+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":409,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021896+0000","last_change":"2023-09-23T15:32:16.250237+0000","last_active":"2023-09-23T15:35:10.021896+0000","last_peered":"2023-09-23T15:35:10.021896+0000","last_clean":"2023-09-23T15:35:10.021896+0000","last_became_active":"2023-09-23T15:32:16.249918+0000","last_became_peered":"2023-09-23T15:32:16.249918+0000","last_unstale":"2023-09-23T15:35:10.021896+0000","last_undegraded":"2023-09-23T15:35:10.021896+0000","last_fullsized":"2023-09-23T15:35:10.021896+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:33:31.035969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"488'306","reported_seq":973,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755377+0000","last_change":"2023-09-23T15:34:54.705642+0000","last_active":"2023-09-23T15:35:06.755377+0000","last_peered":"2023-09-23T15:35:06.755377+0000","last_clean":"2023-09-23T15:35:06.755377+0000","last_became_active":"2023-09-23T15:32:18.623239+0000","last_became_peered":"2023-09-23T15:32:18.623239+0000","last_unstale":"2023-09-23T15:35:06.755377+0000","last_undegraded":"2023-09-23T15:35:06.755377+0000","last_fullsized":"2023-09-23T15:35:06.755377+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:50.264791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022811199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1196,"num_read_kb":2267,"num_write":287,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":799,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756062+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:35:06.756062+0000","last_peered":"2023-09-23T15:35:06.756062+0000","last_clean":"2023-09-23T15:35:06.756062+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:35:06.756062+0000","last_undegraded":"2023-09-23T15:35:06.756062+0000","last_fullsized":"2023-09-23T15:35:06.756062+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":344,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744473+0000","last_change":"2023-09-23T15:32:16.251284+0000","last_active":"2023-09-23T15:35:05.744473+0000","last_peered":"2023-09-23T15:35:05.744473+0000","last_clean":"2023-09-23T15:35:05.744473+0000","last_became_active":"2023-09-23T15:32:16.250677+0000","last_became_peered":"2023-09-23T15:32:16.250677+0000","last_unstale":"2023-09-23T15:35:05.744473+0000","last_undegraded":"2023-09-23T15:35:05.744473+0000","last_fullsized":"2023-09-23T15:35:05.744473+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:45.512698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":485,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744630+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:35:05.744630+0000","last_peered":"2023-09-23T15:35:05.744630+0000","last_clean":"2023-09-23T15:35:05.744630+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:35:05.744630+0000","last_undegraded":"2023-09-23T15:35:05.744630+0000","last_fullsized":"2023-09-23T15:35:05.744630+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":840,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019175+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:35:10.019175+0000","last_peered":"2023-09-23T15:35:10.019175+0000","last_clean":"2023-09-23T15:35:10.019175+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:35:10.019175+0000","last_undegraded":"2023-09-23T15:35:10.019175+0000","last_fullsized":"2023-09-23T15:35:10.019175+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"488'240","reported_seq":623,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755965+0000","last_change":"2023-09-23T15:34:54.708597+0000","last_active":"2023-09-23T15:35:06.755965+0000","last_peered":"2023-09-23T15:35:06.755965+0000","last_clean":"2023-09-23T15:35:06.755965+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:35:06.755965+0000","last_undegraded":"2023-09-23T15:35:06.755965+0000","last_fullsized":"2023-09-23T15:35:06.755965+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013449600000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1148,"num_read_kb":1983,"num_write":238,"num_write_kb":6264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":490,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090265+0000","last_change":"2023-09-23T15:33:30.772235+0000","last_active":"2023-09-23T15:35:03.090265+0000","last_peered":"2023-09-23T15:35:03.090265+0000","last_clean":"2023-09-23T15:35:03.090265+0000","last_became_active":"2023-09-23T15:33:30.771906+0000","last_became_peered":"2023-09-23T15:33:30.771906+0000","last_unstale":"2023-09-23T15:35:03.090265+0000","last_undegraded":"2023-09-23T15:35:03.090265+0000","last_fullsized":"2023-09-23T15:35:03.090265+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:20.505903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":475,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090617+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:35:03.090617+0000","last_peered":"2023-09-23T15:35:03.090617+0000","last_clean":"2023-09-23T15:35:03.090617+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:35:03.090617+0000","last_undegraded":"2023-09-23T15:35:03.090617+0000","last_fullsized":"2023-09-23T15:35:03.090617+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":819,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744650+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:35:05.744650+0000","last_peered":"2023-09-23T15:35:05.744650+0000","last_clean":"2023-09-23T15:35:05.744650+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:35:05.744650+0000","last_undegraded":"2023-09-23T15:35:05.744650+0000","last_fullsized":"2023-09-23T15:35:05.744650+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"488'207","reported_seq":1447,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756576+0000","last_change":"2023-09-23T15:34:54.709815+0000","last_active":"2023-09-23T15:35:06.756576+0000","last_peered":"2023-09-23T15:35:06.756576+0000","last_clean":"2023-09-23T15:35:06.756576+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:35:06.756576+0000","last_undegraded":"2023-09-23T15:35:06.756576+0000","last_fullsized":"2023-09-23T15:35:06.756576+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":207,"log_dups_size":0,"ondisk_log_size":207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038879700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1613,"num_read_kb":2033,"num_write":161,"num_write_kb":409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":385,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020610+0000","last_change":"2023-09-23T15:32:16.239062+0000","last_active":"2023-09-23T15:35:10.020610+0000","last_peered":"2023-09-23T15:35:10.020610+0000","last_clean":"2023-09-23T15:35:10.020610+0000","last_became_active":"2023-09-23T15:32:16.238742+0000","last_became_peered":"2023-09-23T15:32:16.238742+0000","last_unstale":"2023-09-23T15:35:10.020610+0000","last_undegraded":"2023-09-23T15:35:10.020610+0000","last_fullsized":"2023-09-23T15:35:10.020610+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:28.697401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":483,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090505+0000","last_change":"2023-09-23T15:32:16.250477+0000","last_active":"2023-09-23T15:35:03.090505+0000","last_peered":"2023-09-23T15:35:03.090505+0000","last_clean":"2023-09-23T15:35:03.090505+0000","last_became_active":"2023-09-23T15:32:16.249968+0000","last_became_peered":"2023-09-23T15:32:16.249968+0000","last_unstale":"2023-09-23T15:35:03.090505+0000","last_undegraded":"2023-09-23T15:35:03.090505+0000","last_fullsized":"2023-09-23T15:35:03.090505+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:45:43.524682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":784,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743295+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:35:05.743295+0000","last_peered":"2023-09-23T15:35:05.743295+0000","last_clean":"2023-09-23T15:35:05.743295+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:35:05.743295+0000","last_undegraded":"2023-09-23T15:35:05.743295+0000","last_fullsized":"2023-09-23T15:35:05.743295+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":266,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755939+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:35:06.755939+0000","last_peered":"2023-09-23T15:35:06.755939+0000","last_clean":"2023-09-23T15:35:06.755939+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:35:06.755939+0000","last_undegraded":"2023-09-23T15:35:06.755939+0000","last_fullsized":"2023-09-23T15:35:06.755939+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"488'320","reported_seq":1833,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021119+0000","last_change":"2023-09-23T15:34:54.710828+0000","last_active":"2023-09-23T15:35:10.021119+0000","last_peered":"2023-09-23T15:35:10.021119+0000","last_clean":"2023-09-23T15:35:10.021119+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:35:10.021119+0000","last_undegraded":"2023-09-23T15:35:10.021119+0000","last_fullsized":"2023-09-23T15:35:10.021119+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":320,"log_dups_size":0,"ondisk_log_size":320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011064320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2324,"num_read_kb":4368,"num_write":216,"num_write_kb":2345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":440,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755579+0000","last_change":"2023-09-23T15:30:26.710395+0000","last_active":"2023-09-23T15:35:06.755579+0000","last_peered":"2023-09-23T15:35:06.755579+0000","last_clean":"2023-09-23T15:35:06.755579+0000","last_became_active":"2023-09-23T15:30:26.710076+0000","last_became_peered":"2023-09-23T15:30:26.710076+0000","last_unstale":"2023-09-23T15:35:06.755579+0000","last_undegraded":"2023-09-23T15:35:06.755579+0000","last_fullsized":"2023-09-23T15:35:06.755579+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.313559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":669,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755620+0000","last_change":"2023-09-23T15:32:18.639607+0000","last_active":"2023-09-23T15:35:06.755620+0000","last_peered":"2023-09-23T15:35:06.755620+0000","last_clean":"2023-09-23T15:35:06.755620+0000","last_became_active":"2023-09-23T15:32:18.639236+0000","last_became_peered":"2023-09-23T15:32:18.639236+0000","last_unstale":"2023-09-23T15:35:06.755620+0000","last_undegraded":"2023-09-23T15:35:06.755620+0000","last_fullsized":"2023-09-23T15:35:06.755620+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:41:34.044610+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"488'157","reported_seq":1105,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744904+0000","last_change":"2023-09-23T15:34:54.714657+0000","last_active":"2023-09-23T15:35:05.744904+0000","last_peered":"2023-09-23T15:35:05.744904+0000","last_clean":"2023-09-23T15:35:05.744904+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:35:05.744904+0000","last_undegraded":"2023-09-23T15:35:05.744904+0000","last_fullsized":"2023-09-23T15:35:05.744904+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031768699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1077,"num_read_kb":2328,"num_write":133,"num_write_kb":2319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":534,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090690+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:35:03.090690+0000","last_peered":"2023-09-23T15:35:03.090690+0000","last_clean":"2023-09-23T15:35:03.090690+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:35:03.090690+0000","last_undegraded":"2023-09-23T15:35:03.090690+0000","last_fullsized":"2023-09-23T15:35:03.090690+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":145,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742799+0000","last_change":"2023-09-23T15:32:16.252192+0000","last_active":"2023-09-23T15:35:05.742799+0000","last_peered":"2023-09-23T15:35:05.742799+0000","last_clean":"2023-09-23T15:35:05.742799+0000","last_became_active":"2023-09-23T15:32:16.251805+0000","last_became_peered":"2023-09-23T15:32:16.251805+0000","last_unstale":"2023-09-23T15:35:05.742799+0000","last_undegraded":"2023-09-23T15:35:05.742799+0000","last_fullsized":"2023-09-23T15:35:05.742799+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:59.903570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":702,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090181+0000","last_change":"2023-09-23T15:30:26.740237+0000","last_active":"2023-09-23T15:35:03.090181+0000","last_peered":"2023-09-23T15:35:03.090181+0000","last_clean":"2023-09-23T15:35:03.090181+0000","last_became_active":"2023-09-23T15:30:26.690718+0000","last_became_peered":"2023-09-23T15:30:26.690718+0000","last_unstale":"2023-09-23T15:35:03.090181+0000","last_undegraded":"2023-09-23T15:35:03.090181+0000","last_fullsized":"2023-09-23T15:35:03.090181+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:00.439552+0000","scrub_duration":0.013441308000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":31,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090473+0000","last_change":"2023-09-23T15:34:12.777336+0000","last_active":"2023-09-23T15:35:03.090473+0000","last_peered":"2023-09-23T15:35:03.090473+0000","last_clean":"2023-09-23T15:35:03.090473+0000","last_became_active":"2023-09-23T15:34:12.777101+0000","last_became_peered":"2023-09-23T15:34:12.777101+0000","last_unstale":"2023-09-23T15:35:03.090473+0000","last_undegraded":"2023-09-23T15:35:03.090473+0000","last_fullsized":"2023-09-23T15:35:03.090473+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:13:25.546836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":24,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756005+0000","last_change":"2023-09-23T15:34:44.672844+0000","last_active":"2023-09-23T15:35:06.756005+0000","last_peered":"2023-09-23T15:35:06.756005+0000","last_clean":"2023-09-23T15:35:06.756005+0000","last_became_active":"2023-09-23T15:34:44.672523+0000","last_became_peered":"2023-09-23T15:34:44.672523+0000","last_unstale":"2023-09-23T15:35:06.756005+0000","last_undegraded":"2023-09-23T15:35:06.756005+0000","last_fullsized":"2023-09-23T15:35:06.756005+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:26.341913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":670,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744632+0000","last_change":"2023-09-23T15:30:28.373196+0000","last_active":"2023-09-23T15:35:05.744632+0000","last_peered":"2023-09-23T15:35:05.744632+0000","last_clean":"2023-09-23T15:35:05.744632+0000","last_became_active":"2023-09-23T15:30:26.699434+0000","last_became_peered":"2023-09-23T15:30:26.699434+0000","last_unstale":"2023-09-23T15:35:05.744632+0000","last_undegraded":"2023-09-23T15:35:05.744632+0000","last_fullsized":"2023-09-23T15:35:05.744632+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:26.970670+0000","scrub_duration":0.0018471900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":463,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742929+0000","last_change":"2023-09-23T15:32:16.271454+0000","last_active":"2023-09-23T15:35:05.742929+0000","last_peered":"2023-09-23T15:35:05.742929+0000","last_clean":"2023-09-23T15:35:05.742929+0000","last_became_active":"2023-09-23T15:32:16.246283+0000","last_became_peered":"2023-09-23T15:32:16.246283+0000","last_unstale":"2023-09-23T15:35:05.742929+0000","last_undegraded":"2023-09-23T15:35:05.742929+0000","last_fullsized":"2023-09-23T15:35:05.742929+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:36.794240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743010+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:35:05.743010+0000","last_peered":"2023-09-23T15:35:05.743010+0000","last_clean":"2023-09-23T15:35:05.743010+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:35:05.743010+0000","last_undegraded":"2023-09-23T15:35:05.743010+0000","last_fullsized":"2023-09-23T15:35:05.743010+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":443,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.743074+0000","last_change":"2023-09-23T15:32:16.280285+0000","last_active":"2023-09-23T15:35:05.743074+0000","last_peered":"2023-09-23T15:35:05.743074+0000","last_clean":"2023-09-23T15:35:05.743074+0000","last_became_active":"2023-09-23T15:32:16.251845+0000","last_became_peered":"2023-09-23T15:32:16.251845+0000","last_unstale":"2023-09-23T15:35:05.743074+0000","last_undegraded":"2023-09-23T15:35:05.743074+0000","last_fullsized":"2023-09-23T15:35:05.743074+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:07.938264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":173,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090312+0000","last_change":"2023-09-23T15:33:30.769202+0000","last_active":"2023-09-23T15:35:03.090312+0000","last_peered":"2023-09-23T15:35:03.090312+0000","last_clean":"2023-09-23T15:35:03.090312+0000","last_became_active":"2023-09-23T15:33:30.768887+0000","last_became_peered":"2023-09-23T15:33:30.768887+0000","last_unstale":"2023-09-23T15:35:03.090312+0000","last_undegraded":"2023-09-23T15:35:03.090312+0000","last_fullsized":"2023-09-23T15:35:03.090312+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:15:33.336344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":680,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.756404+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:35:06.756404+0000","last_peered":"2023-09-23T15:35:06.756404+0000","last_clean":"2023-09-23T15:35:06.756404+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:35:06.756404+0000","last_undegraded":"2023-09-23T15:35:06.756404+0000","last_fullsized":"2023-09-23T15:35:06.756404+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":125,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.744792+0000","last_change":"2023-09-23T15:30:26.699597+0000","last_active":"2023-09-23T15:35:05.744792+0000","last_peered":"2023-09-23T15:35:05.744792+0000","last_clean":"2023-09-23T15:35:05.744792+0000","last_became_active":"2023-09-23T15:30:26.699233+0000","last_became_peered":"2023-09-23T15:30:26.699233+0000","last_unstale":"2023-09-23T15:35:05.744792+0000","last_undegraded":"2023-09-23T15:35:05.744792+0000","last_fullsized":"2023-09-23T15:35:05.744792+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.804745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":417,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:06.755905+0000","last_change":"2023-09-23T15:30:26.685257+0000","last_active":"2023-09-23T15:35:06.755905+0000","last_peered":"2023-09-23T15:35:06.755905+0000","last_clean":"2023-09-23T15:35:06.755905+0000","last_became_active":"2023-09-23T15:30:26.684954+0000","last_became_peered":"2023-09-23T15:30:26.684954+0000","last_unstale":"2023-09-23T15:35:06.755905+0000","last_undegraded":"2023-09-23T15:35:06.755905+0000","last_fullsized":"2023-09-23T15:35:06.755905+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:49.244950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":677,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020238+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:35:10.020238+0000","last_peered":"2023-09-23T15:35:10.020238+0000","last_clean":"2023-09-23T15:35:10.020238+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:35:10.020238+0000","last_undegraded":"2023-09-23T15:35:10.020238+0000","last_fullsized":"2023-09-23T15:35:10.020238+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":463,"reported_epoch":491,"state":"active+clean","last_fresh":"2023-09-23T15:35:03.090891+0000","last_change":"2023-09-23T15:32:16.261750+0000","last_active":"2023-09-23T15:35:03.090891+0000","last_peered":"2023-09-23T15:35:03.090891+0000","last_clean":"2023-09-23T15:35:03.090891+0000","last_became_active":"2023-09-23T15:32:16.241839+0000","last_became_peered":"2023-09-23T15:32:16.241839+0000","last_unstale":"2023-09-23T15:35:03.090891+0000","last_undegraded":"2023-09-23T15:35:03.090891+0000","last_fullsized":"2023-09-23T15:35:03.090891+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:29:45.309569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":135,"reported_epoch":492,"state":"active+clean","last_fresh":"2023-09-23T15:35:05.742331+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:35:05.742331+0000","last_peered":"2023-09-23T15:35:05.742331+0000","last_clean":"2023-09-23T15:35:05.742331+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:35:05.742331+0000","last_undegraded":"2023-09-23T15:35:05.742331+0000","last_fullsized":"2023-09-23T15:35:05.742331+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":10398,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11973,"num_read_kb":26407,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768795,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":149916,"kb_used_data":9556,"kb_used_omap":13,"kb_used_meta":140338,"kb_avail":93595236,"statfs":{"total":95995035648,"available":95841521664,"internally_reserved":0,"allocated":9785344,"data_stored":8254036,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":143706206},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":5.5439999999999996,"15min":5.6959999999999997},"min":{"1min":0.26800000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.72799999999999998,"5min":121.012,"15min":121.012},"last":0.496},{"interface":"front","average":{"1min":0.57899999999999996,"5min":5.2309999999999999,"15min":5.6539999999999999},"min":{"1min":0.435,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.73999999999999999,"5min":120.97,"15min":120.97},"last":0.34499999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":2.2469999999999999,"15min":13.936999999999999},"min":{"1min":0.32900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.63200000000000001,"5min":56.816000000000003,"15min":157.096},"last":0.437},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.9830000000000001,"15min":13.875},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.68799999999999994,"5min":58.274000000000001,"15min":157.011},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":8.3000000000000007,"15min":3.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":220.875,"15min":220.875},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.54300000000000004,"5min":8.0809999999999995,"15min":3.3450000000000002},"min":{"1min":0.32200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.75700000000000001,"5min":220.84399999999999,"15min":220.84399999999999},"last":0.52400000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.724,"15min":3.3969999999999998},"min":{"1min":0.40100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67500000000000004,"5min":10.061,"15min":76.019000000000005},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.8779999999999999,"15min":3.766},"min":{"1min":0.33900000000000002,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":0.71999999999999997,"5min":12.606999999999999,"15min":76.317999999999998},"last":0.39700000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":3.6520000000000001,"15min":1.6259999999999999},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68300000000000005,"5min":170.15799999999999,"15min":170.15799999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.50900000000000001,"5min":3.7799999999999998,"15min":1.6479999999999999},"min":{"1min":0.33400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.71999999999999997,"5min":170.017,"15min":170.017},"last":0.33100000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.004,"15min":0.86699999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":0.749,"5min":7.2910000000000004,"15min":7.2910000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.627,"5min":1.4750000000000001,"15min":1.0029999999999999},"min":{"1min":0.36499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.79500000000000004,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":0.51200000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":4.7359999999999998,"15min":2.073},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.80200000000000005,"5min":262.38099999999997,"15min":262.38099999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.62,"5min":4.7850000000000001,"15min":2.0960000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81499999999999995,"5min":261.488,"15min":261.488},"last":0.54100000000000004}]}]},{"osd":6,"up_from":304,"seq":1305670058138,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213140,"kb_used_data":5516,"kb_used_omap":13,"kb_used_meta":207602,"kb_avail":93532012,"statfs":{"total":95995035648,"available":95776780288,"internally_reserved":0,"allocated":5648384,"data_stored":4125151,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":212584803},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":3.6600000000000001,"15min":3.319},"min":{"1min":0.26900000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.99399999999999999,"5min":49.441000000000003,"15min":184.53200000000001},"last":0.83799999999999997},{"interface":"front","average":{"1min":0.48399999999999999,"5min":4.3380000000000001,"15min":3.4039999999999999},"min":{"1min":0.25,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.67200000000000004,"5min":48.880000000000003,"15min":86.227999999999994},"last":0.57699999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.4379999999999999,"15min":2.3740000000000001},"min":{"1min":0.29599999999999999,"5min":0.248,"15min":0.221},"max":{"1min":0.60999999999999999,"5min":7.907,"15min":86.534000000000006},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.2629999999999999,"15min":2.202},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70999999999999996,"5min":6.4909999999999997,"15min":86.144000000000005},"last":0.73599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":2.847,"15min":2.1339999999999999},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.751,"5min":63.512,"15min":63.512},"last":0.77000000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":3.1739999999999999,"15min":2.2549999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":0.73399999999999999,"5min":63.526000000000003,"15min":63.526000000000003},"last":0.67200000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.478,"15min":2.456},"min":{"1min":0.378,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.081,"5min":11.176,"15min":142.67599999999999},"last":0.80200000000000005},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.069,"15min":2.8559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.754,"5min":11.795,"15min":142.541},"last":0.72199999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":2.3820000000000001,"15min":1.4490000000000001},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.23999999999999999},"max":{"1min":0.73199999999999998,"5min":128.11600000000001,"15min":128.11600000000001},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":2.9460000000000002,"15min":2.016},"min":{"1min":0.312,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.79500000000000004,"5min":128.095,"15min":128.095},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":3.0350000000000001,"15min":2.698},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.248},"max":{"1min":0.70899999999999996,"5min":124.929,"15min":130.91300000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.3759999999999999,"15min":2.2719999999999998},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.29099999999999998},"max":{"1min":0.83699999999999997,"5min":124.901,"15min":130.92599999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.5389999999999999,"15min":1.3240000000000001},"min":{"1min":0.42899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77800000000000002,"5min":10.087,"15min":10.087},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.0109999999999999,"15min":1.3089999999999999},"min":{"1min":0.48799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.85999999999999999,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.65000000000000002}]}]},{"osd":1,"up_from":183,"seq":785979015439,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700528,"kb_used_data":10600,"kb_used_omap":6,"kb_used_meta":689913,"kb_avail":93044624,"statfs":{"total":95995035648,"available":95277694976,"internally_reserved":0,"allocated":10854400,"data_stored":8406529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":706470987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":5.1740000000000004,"5min":5.2039999999999997,"15min":3.7290000000000001},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.191},"max":{"1min":47.207999999999998,"5min":101.27800000000001,"15min":101.27800000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":5.1289999999999996,"5min":5.4489999999999998,"15min":4.0570000000000004},"min":{"1min":0.245,"5min":0.23799999999999999,"15min":0.21099999999999999},"max":{"1min":48.692999999999998,"5min":101.051,"15min":101.051},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.5510000000000002,"5min":6.1239999999999997,"15min":3.831},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":42.497,"5min":186.304,"15min":186.304},"last":0.29799999999999999},{"interface":"front","average":{"1min":3.3860000000000001,"5min":5.4960000000000004,"15min":3.4710000000000001},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":42.371000000000002,"5min":186.541,"15min":186.541},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.9399999999999999,"15min":2.8860000000000001},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.6709999999999994,"5min":13.744,"15min":64.540999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.502,"5min":1.746,"15min":2.8809999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.066000000000001,"5min":13.98,"15min":67.465000000000003},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.9810000000000001,"15min":2.9319999999999999},"min":{"1min":0.378,"5min":0.33800000000000002,"15min":0.29799999999999999},"max":{"1min":8.5289999999999999,"5min":13.840999999999999,"15min":80.570999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.397,"5min":1.5409999999999999,"15min":2.5779999999999998},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":80.75},"last":1.0029999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.79,"15min":3.2949999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.27400000000000002},"max":{"1min":10.936999999999999,"5min":30.009,"15min":173.18799999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.5309999999999999,"5min":2.4399999999999999,"15min":3.6480000000000001},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.29099999999999998},"max":{"1min":8.6370000000000005,"5min":29.983000000000001,"15min":172.822},"last":0.86599999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:34:45 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":2.2069999999999999,"15min":2.7559999999999998},"min":{"1min":0.40799999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.6150000000000002,"5min":13.912000000000001,"15min":54.198},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.012,"5min":1.4750000000000001,"15min":1.885},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.2710000000000008,"5min":10.996,"15min":15.605},"last":0.66100000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.482,"15min":1.9059999999999999},"min":{"1min":0.44600000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.117,"5min":11.032999999999999,"15min":11.032999999999999},"last":0.754},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.129,"15min":3.9079999999999999},"min":{"1min":0.49199999999999999,"5min":0.41899999999999998,"15min":0.40899999999999997},"max":{"1min":3.028,"5min":13.945,"15min":54.223999999999997},"last":0.89200000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893485,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898404,"kb_used_data":14556,"kb_used_omap":8,"kb_used_meta":883831,"kb_avail":92846748,"statfs":{"total":95995035648,"available":95075069952,"internally_reserved":0,"allocated":14905344,"data_stored":12627393,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":905043540},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:34:18 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":2.899,"15min":4.4119999999999999},"min":{"1min":0.23999999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":63.393000000000001,"5min":63.393000000000001,"15min":141.52199999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":3.6070000000000002,"5min":2.5209999999999999,"15min":4.2999999999999998},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":63.417999999999999,"5min":63.417999999999999,"15min":141.62799999999999},"last":0.38600000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":3.0419999999999998,"15min":2.5049999999999999},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":0.59699999999999998,"5min":66.132999999999996,"15min":66.132999999999996},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.42599999999999999,"5min":3.4460000000000002,"15min":2.5489999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.20300000000000001},"max":{"1min":0.66300000000000003,"5min":68.296999999999997,"15min":68.296999999999997},"last":0.46100000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:34:09 2023","interfaces":[{"interface":"back","average":{"1min":1.962,"5min":3.7989999999999999,"15min":4.8540000000000001},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":9.4649999999999999,"5min":65.251999999999995,"15min":197.78899999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.3719999999999999,"5min":4.0090000000000003,"15min":4.9199999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":9.6769999999999996,"5min":65.269999999999996,"15min":200.58799999999999},"last":0.57699999999999996}]},{"osd":4,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":2.7890000000000001,"15min":2.2770000000000001},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.81999999999999995,"5min":60.771999999999998,"15min":60.771999999999998},"last":0.78600000000000003},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.7429999999999999,"15min":2.2909999999999999},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.86799999999999999,"5min":60.725999999999999,"15min":60.725999999999999},"last":0.67600000000000005}]},{"osd":5,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.081,"15min":2.141},"min":{"1min":0.38800000000000001,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":1.167,"5min":33.158999999999999,"15min":44.57},"last":0.73999999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":2.1840000000000002,"15min":2.3879999999999999},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.89300000000000002,"5min":33.140000000000001,"15min":44.588000000000001},"last":0.71499999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":2.8410000000000002,"15min":2.6819999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82499999999999996,"5min":62.168999999999997,"15min":62.168999999999997},"last":0.754},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.8119999999999998,"15min":2.9550000000000001},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91700000000000004,"5min":61.972000000000001,"15min":61.972000000000001},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.5219999999999998,"15min":2.2919999999999998},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":1.141,"5min":36.848999999999997,"15min":36.848999999999997},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.65600000000000003,"5min":2.6280000000000001,"15min":3.4420000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.86799999999999999,"5min":36.947000000000003,"15min":36.947000000000003},"last":0.63}]}]},{"osd":2,"up_from":105,"seq":450971566435,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854560,"kb_used_data":10392,"kb_used_omap":11,"kb_used_meta":844148,"kb_avail":92890592,"statfs":{"total":95995035648,"available":95119966208,"internally_reserved":0,"allocated":10641408,"data_stored":8374650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":864408336},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":3.6230000000000002,"15min":5.4619999999999997},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":2.3730000000000002,"5min":82.878,"15min":235.517},"last":0.28499999999999998},{"interface":"front","average":{"1min":0.71899999999999997,"5min":3.3809999999999998,"15min":5.3490000000000002},"min":{"1min":0.26400000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":7.4509999999999996,"5min":82.762,"15min":235.33799999999999},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:34:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":2.488,"15min":3.3010000000000002},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.20499999999999999},"max":{"1min":9.8339999999999996,"5min":65.906999999999996,"15min":71.819000000000003},"last":0.254},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.4809999999999999,"15min":3.149},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":12.984999999999999,"5min":65.222999999999999,"15min":71.805000000000007},"last":0.31}]},{"osd":3,"last update":"Sat Sep 23 15:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.7210000000000001,"15min":3.5489999999999999},"min":{"1min":0.25,"5min":0.216,"15min":0.216},"max":{"1min":2.0110000000000001,"5min":18.606999999999999,"15min":77.031999999999996},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.863,"15min":3.6579999999999999},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":3.9350000000000001,"5min":11.420999999999999,"15min":76.742000000000004},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":2.25,"15min":1.7929999999999999},"min":{"1min":0.36199999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7880000000000003,"5min":63.747999999999998,"15min":63.747999999999998},"last":1.353},{"interface":"front","average":{"1min":1.2230000000000001,"5min":2.5659999999999998,"15min":1.952},"min":{"1min":0.37,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9509999999999996,"5min":63.954000000000001,"15min":63.954000000000001},"last":1.4470000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.4810000000000001,"15min":2.2639999999999998},"min":{"1min":0.39200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.7739999999999991,"5min":9.9559999999999995,"15min":107.021},"last":1.409},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.589,"15min":2.1899999999999999},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.31},"max":{"1min":7.2759999999999998,"5min":9.577,"15min":107.136},"last":1.3939999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:34:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":3.3399999999999999,"15min":2.4860000000000002},"min":{"1min":0.40500000000000003,"5min":0.318,"15min":0.318},"max":{"1min":10.51,"5min":156.16,"15min":156.16},"last":1.371},{"interface":"front","average":{"1min":0.71299999999999997,"5min":3.3460000000000001,"15min":2.258},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":2.8879999999999999,"5min":155.226,"15min":155.226},"last":1.302}]},{"osd":7,"last update":"Sat Sep 23 15:34:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.909,"15min":2.6779999999999999},"min":{"1min":0.47699999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.835,"5min":12.282,"15min":112.70699999999999},"last":1.4199999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.9159999999999999,"15min":3.125},"min":{"1min":0.48299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.84899999999999998,"5min":9.9280000000000008,"15min":109.402},"last":1.3169999999999999}]}]},{"osd":3,"up_from":110,"seq":472446402907,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866036,"kb_used_data":10476,"kb_used_omap":3,"kb_used_meta":855548,"kb_avail":92879116,"statfs":{"total":95995035648,"available":95108214784,"internally_reserved":0,"allocated":10727424,"data_stored":8441831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":876081321},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":4.3849999999999998,"15min":3.992},"min":{"1min":0.20999999999999999,"5min":0.187,"15min":0.183},"max":{"1min":0.57099999999999995,"5min":141.934,"15min":141.934},"last":0.441},{"interface":"front","average":{"1min":0.42799999999999999,"5min":5.0069999999999997,"15min":4.5229999999999997},"min":{"1min":0.23499999999999999,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":1.075,"5min":141.72800000000001,"15min":141.72800000000001},"last":0.29099999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:34:29 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.1190000000000002,"15min":3.8759999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":3.9300000000000002,"5min":16.623000000000001,"15min":126.875},"last":0.314},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.798,"15min":4.1879999999999997},"min":{"1min":0.33300000000000002,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":4.7729999999999997,"5min":13.401999999999999,"15min":128.249},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":5.194,"15min":3.036},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.0529999999999999,"5min":144.35599999999999,"15min":144.35599999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":5.7140000000000004,"15min":3.585},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.246},"max":{"1min":1.21,"5min":144.494,"15min":144.494},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":2.3799999999999999,"15min":2.0030000000000001},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":1.1120000000000001,"5min":44.289000000000001,"15min":44.289000000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.3010000000000002,"15min":1.889},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":1.0669999999999999,"5min":44.055,"15min":44.055},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":3.2930000000000001,"15min":3.2519999999999998},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":1.028,"5min":122.452,"15min":148.619},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.623,"5min":3.1299999999999999,"15min":3.0139999999999998},"min":{"1min":0.439,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":1.103,"5min":122.33499999999999,"15min":148.56999999999999},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:34:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.8690000000000002,"15min":2.9249999999999998},"min":{"1min":0.40200000000000002,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":36.165999999999997,"5min":44.356000000000002,"15min":84.665000000000006},"last":0.42599999999999999},{"interface":"front","average":{"1min":3.2589999999999999,"5min":3.056,"15min":3.3900000000000001},"min":{"1min":0.46800000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":36.226999999999997,"5min":44.453000000000003,"15min":84.897999999999996},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:34:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":2.726,"15min":8.3719999999999999},"min":{"1min":0.45300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.96599999999999997,"5min":121.738,"15min":216.56899999999999},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.3130000000000002,"15min":9.3520000000000003},"min":{"1min":0.49199999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2310000000000001,"5min":122.10599999999999,"15min":216.548},"last":0.55400000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986674,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":524692,"kb_used_data":5196,"kb_used_omap":0,"kb_used_meta":519488,"kb_avail":93220460,"statfs":{"total":95995035648,"available":95457751040,"internally_reserved":0,"allocated":5320704,"data_stored":3796974,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":531955712},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":2.6000000000000001,"15min":3.6749999999999998},"min":{"1min":0.27700000000000002,"5min":0.222,"15min":0.222},"max":{"1min":4.577,"5min":36.865000000000002,"15min":162.75999999999999},"last":0.79900000000000004},{"interface":"front","average":{"1min":0.70699999999999996,"5min":2.7890000000000001,"15min":3.7389999999999999},"min":{"1min":0.247,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":3.9009999999999998,"5min":37.081000000000003,"15min":162.34800000000001},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.976,"15min":3.7690000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":7.0830000000000002,"5min":41.354999999999997,"15min":144.76900000000001},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.736,"15min":3.8149999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25},"max":{"1min":3.1000000000000001,"5min":41.343000000000004,"15min":144.58799999999999},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:34:35 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":4.0970000000000004,"15min":2.972},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":5.5049999999999999,"5min":89.980000000000004,"15min":89.980000000000004},"last":0.77400000000000002},{"interface":"front","average":{"1min":1.016,"5min":3.8410000000000002,"15min":2.98},"min":{"1min":0.38500000000000001,"5min":0.36199999999999999,"15min":0.30599999999999999},"max":{"1min":3.9729999999999999,"5min":90.016999999999996,"15min":90.016999999999996},"last":0.54300000000000004}]},{"osd":3,"last update":"Sat Sep 23 15:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.9730000000000001,"15min":3.6749999999999998},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.30399999999999999},"max":{"1min":4.0679999999999996,"5min":35.692,"15min":156.46000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.0249999999999999,"15min":3.7040000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":3.0049999999999999,"5min":35.920999999999999,"15min":156.74299999999999},"last":0.70499999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:34:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.3089999999999999,"15min":1.226},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.221},"max":{"1min":1.28,"5min":11.904999999999999,"15min":40.685000000000002},"last":0.505},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.379,"15min":1.341},"min":{"1min":0.33900000000000002,"5min":0.27500000000000002,"15min":0.246},"max":{"1min":3.044,"5min":12.882999999999999,"15min":38.439},"last":0.63400000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":4.0609999999999999,"15min":2.1299999999999999},"min":{"1min":0.34399999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.0699999999999998,"5min":228.93199999999999,"15min":228.93199999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.76200000000000001,"5min":4.4850000000000003,"15min":2.4860000000000002},"min":{"1min":0.38900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.1870000000000003,"5min":232.54900000000001,"15min":232.54900000000001},"last":0.48899999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:34:49 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":3.117,"15min":1.9379999999999999},"min":{"1min":0.36199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.3029999999999999,"5min":183.94800000000001,"15min":183.94800000000001},"last":0.71799999999999997},{"interface":"front","average":{"1min":0.92300000000000004,"5min":3.323,"15min":1.7949999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":7.5060000000000002,"5min":183.72399999999999,"15min":183.72399999999999},"last":0.75700000000000001}]}]},{"osd":5,"up_from":0,"seq":167503725037,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:35:14.281 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:35:14.651 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:14.755 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T15:35:15.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:14 smithi017 ceph-mon[222878]: pgmap v1450: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:15.079 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:14 smithi017 ceph-mon[222878]: from='client.57457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:15.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:14 smithi017 ceph-mon[215646]: pgmap v1450: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:15.080 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:14 smithi017 ceph-mon[215646]: from='client.57457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:14 smithi027 ceph-mon[195233]: pgmap v1450: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:14 smithi027 ceph-mon[195233]: from='client.57457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:15.115 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:16.411 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:35:16.413 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:35:16.714 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:16.714 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:16.714 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:35:16.714 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:16.952 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1451,"stamp":"2023-09-23T15:35:15.067407+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11973,"num_read_kb":26407,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":162,"num_bytes_recovered":4204721,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6998,"ondisk_log_size":6998,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4207508,"kb_used_data":66460,"kb_used_omap":57,"kb_used_meta":4140934,"kb_avail":652008556,"statfs":{"total":671965249536,"available":667656761344,"internally_reserved":0,"allocated":68055040,"data_stored":54177472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4240316441},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001246"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":605,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.807750+0000","last_change":"2023-09-23T15:33:08.496898+0000","last_active":"2023-09-23T15:35:10.807750+0000","last_peered":"2023-09-23T15:35:10.807750+0000","last_clean":"2023-09-23T15:35:10.807750+0000","last_became_active":"2023-09-23T15:32:18.611133+0000","last_became_peered":"2023-09-23T15:32:18.611133+0000","last_unstale":"2023-09-23T15:35:10.807750+0000","last_undegraded":"2023-09-23T15:35:10.807750+0000","last_fullsized":"2023-09-23T15:35:10.807750+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:08.496746+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:08.496746+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:53.844651+0000","scrub_duration":0.0035525999999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759825+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:35:10.759825+0000","last_peered":"2023-09-23T15:35:10.759825+0000","last_clean":"2023-09-23T15:35:10.759825+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:35:10.759825+0000","last_undegraded":"2023-09-23T15:35:10.759825+0000","last_fullsized":"2023-09-23T15:35:10.759825+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":64,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019761+0000","last_change":"2023-09-23T15:32:16.240512+0000","last_active":"2023-09-23T15:35:10.019761+0000","last_peered":"2023-09-23T15:35:10.019761+0000","last_clean":"2023-09-23T15:35:10.019761+0000","last_became_active":"2023-09-23T15:32:16.240101+0000","last_became_peered":"2023-09-23T15:32:16.240101+0000","last_unstale":"2023-09-23T15:35:10.019761+0000","last_undegraded":"2023-09-23T15:35:10.019761+0000","last_fullsized":"2023-09-23T15:35:10.019761+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:08.260986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":556,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760066+0000","last_change":"2023-09-23T15:33:05.987466+0000","last_active":"2023-09-23T15:35:10.760066+0000","last_peered":"2023-09-23T15:35:10.760066+0000","last_clean":"2023-09-23T15:35:10.760066+0000","last_became_active":"2023-09-23T15:32:18.614293+0000","last_became_peered":"2023-09-23T15:32:18.614293+0000","last_unstale":"2023-09-23T15:35:10.760066+0000","last_undegraded":"2023-09-23T15:35:10.760066+0000","last_fullsized":"2023-09-23T15:35:10.760066+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.987399+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.987399+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:35.508777+0000","scrub_duration":0.024984823,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":422,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018519+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:35:10.018519+0000","last_peered":"2023-09-23T15:35:10.018519+0000","last_clean":"2023-09-23T15:35:10.018519+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:35:10.018519+0000","last_undegraded":"2023-09-23T15:35:10.018519+0000","last_fullsized":"2023-09-23T15:35:10.018519+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":494,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758910+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:35:10.758910+0000","last_peered":"2023-09-23T15:35:10.758910+0000","last_clean":"2023-09-23T15:35:10.758910+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:35:10.758910+0000","last_undegraded":"2023-09-23T15:35:10.758910+0000","last_fullsized":"2023-09-23T15:35:10.758910+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":631,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759901+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:35:10.759901+0000","last_peered":"2023-09-23T15:35:10.759901+0000","last_clean":"2023-09-23T15:35:10.759901+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:35:10.759901+0000","last_undegraded":"2023-09-23T15:35:10.759901+0000","last_fullsized":"2023-09-23T15:35:10.759901+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":433,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758520+0000","last_change":"2023-09-23T15:30:26.692470+0000","last_active":"2023-09-23T15:35:10.758520+0000","last_peered":"2023-09-23T15:35:10.758520+0000","last_clean":"2023-09-23T15:35:10.758520+0000","last_became_active":"2023-09-23T15:30:26.692148+0000","last_became_peered":"2023-09-23T15:30:26.692148+0000","last_unstale":"2023-09-23T15:35:10.758520+0000","last_undegraded":"2023-09-23T15:35:10.758520+0000","last_fullsized":"2023-09-23T15:35:10.758520+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:32:36.258289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":456,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760567+0000","last_change":"2023-09-23T15:32:16.240747+0000","last_active":"2023-09-23T15:35:10.760567+0000","last_peered":"2023-09-23T15:35:10.760567+0000","last_clean":"2023-09-23T15:35:10.760567+0000","last_became_active":"2023-09-23T15:32:16.240381+0000","last_became_peered":"2023-09-23T15:32:16.240381+0000","last_unstale":"2023-09-23T15:35:10.760567+0000","last_undegraded":"2023-09-23T15:35:10.760567+0000","last_fullsized":"2023-09-23T15:35:10.760567+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:43.105398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":784,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812844+0000","last_change":"2023-09-23T15:32:18.619160+0000","last_active":"2023-09-23T15:35:10.812844+0000","last_peered":"2023-09-23T15:35:10.812844+0000","last_clean":"2023-09-23T15:35:10.812844+0000","last_became_active":"2023-09-23T15:32:18.618856+0000","last_became_peered":"2023-09-23T15:32:18.618856+0000","last_unstale":"2023-09-23T15:35:10.812844+0000","last_undegraded":"2023-09-23T15:35:10.812844+0000","last_fullsized":"2023-09-23T15:35:10.812844+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:08:05.782217+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":456,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759324+0000","last_change":"2023-09-23T15:32:16.242054+0000","last_active":"2023-09-23T15:35:10.759324+0000","last_peered":"2023-09-23T15:35:10.759324+0000","last_clean":"2023-09-23T15:35:10.759324+0000","last_became_active":"2023-09-23T15:32:16.241626+0000","last_became_peered":"2023-09-23T15:32:16.241626+0000","last_unstale":"2023-09-23T15:35:10.759324+0000","last_undegraded":"2023-09-23T15:35:10.759324+0000","last_fullsized":"2023-09-23T15:35:10.759324+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:22:25.373723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":430,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018888+0000","last_change":"2023-09-23T15:30:26.706619+0000","last_active":"2023-09-23T15:35:10.018888+0000","last_peered":"2023-09-23T15:35:10.018888+0000","last_clean":"2023-09-23T15:35:10.018888+0000","last_became_active":"2023-09-23T15:30:26.706343+0000","last_became_peered":"2023-09-23T15:30:26.706343+0000","last_unstale":"2023-09-23T15:35:10.018888+0000","last_undegraded":"2023-09-23T15:35:10.018888+0000","last_fullsized":"2023-09-23T15:35:10.018888+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:11:47.811881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":906,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808400+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:35:10.808400+0000","last_peered":"2023-09-23T15:35:10.808400+0000","last_clean":"2023-09-23T15:35:10.808400+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:35:10.808400+0000","last_undegraded":"2023-09-23T15:35:10.808400+0000","last_fullsized":"2023-09-23T15:35:10.808400+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":449,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812963+0000","last_change":"2023-09-23T15:32:16.248918+0000","last_active":"2023-09-23T15:35:10.812963+0000","last_peered":"2023-09-23T15:35:10.812963+0000","last_clean":"2023-09-23T15:35:10.812963+0000","last_became_active":"2023-09-23T15:32:16.248407+0000","last_became_peered":"2023-09-23T15:32:16.248407+0000","last_unstale":"2023-09-23T15:35:10.812963+0000","last_undegraded":"2023-09-23T15:35:10.812963+0000","last_fullsized":"2023-09-23T15:35:10.812963+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:08.191600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":402,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812503+0000","last_change":"2023-09-23T15:30:26.703714+0000","last_active":"2023-09-23T15:35:10.812503+0000","last_peered":"2023-09-23T15:35:10.812503+0000","last_clean":"2023-09-23T15:35:10.812503+0000","last_became_active":"2023-09-23T15:30:26.703402+0000","last_became_peered":"2023-09-23T15:30:26.703402+0000","last_unstale":"2023-09-23T15:35:10.812503+0000","last_undegraded":"2023-09-23T15:35:10.812503+0000","last_fullsized":"2023-09-23T15:35:10.812503+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:48.014628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":570,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759104+0000","last_change":"2023-09-23T15:33:06.824993+0000","last_active":"2023-09-23T15:35:10.759104+0000","last_peered":"2023-09-23T15:35:10.759104+0000","last_clean":"2023-09-23T15:35:10.759104+0000","last_became_active":"2023-09-23T15:32:18.639717+0000","last_became_peered":"2023-09-23T15:32:18.639717+0000","last_unstale":"2023-09-23T15:35:10.759104+0000","last_undegraded":"2023-09-23T15:35:10.759104+0000","last_fullsized":"2023-09-23T15:35:10.759104+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:06.824864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:06.824864+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.202322+0000","scrub_duration":0.0065159160000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[0,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":241,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.813075+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:35:10.813075+0000","last_peered":"2023-09-23T15:35:10.813075+0000","last_clean":"2023-09-23T15:35:10.813075+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:35:10.813075+0000","last_undegraded":"2023-09-23T15:35:10.813075+0000","last_fullsized":"2023-09-23T15:35:10.813075+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":390,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020247+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:35:10.020247+0000","last_peered":"2023-09-23T15:35:10.020247+0000","last_clean":"2023-09-23T15:35:10.020247+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:35:10.020247+0000","last_undegraded":"2023-09-23T15:35:10.020247+0000","last_fullsized":"2023-09-23T15:35:10.020247+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":647,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020847+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:35:10.020847+0000","last_peered":"2023-09-23T15:35:10.020847+0000","last_clean":"2023-09-23T15:35:10.020847+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:35:10.020847+0000","last_undegraded":"2023-09-23T15:35:10.020847+0000","last_fullsized":"2023-09-23T15:35:10.020847+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":162,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759698+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:35:10.759698+0000","last_peered":"2023-09-23T15:35:10.759698+0000","last_clean":"2023-09-23T15:35:10.759698+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:35:10.759698+0000","last_undegraded":"2023-09-23T15:35:10.759698+0000","last_fullsized":"2023-09-23T15:35:10.759698+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760534+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:35:10.760534+0000","last_peered":"2023-09-23T15:35:10.760534+0000","last_clean":"2023-09-23T15:35:10.760534+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:35:10.760534+0000","last_undegraded":"2023-09-23T15:35:10.760534+0000","last_fullsized":"2023-09-23T15:35:10.760534+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":826,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758573+0000","last_change":"2023-09-23T15:32:18.621224+0000","last_active":"2023-09-23T15:35:10.758573+0000","last_peered":"2023-09-23T15:35:10.758573+0000","last_clean":"2023-09-23T15:35:10.758573+0000","last_became_active":"2023-09-23T15:32:18.620853+0000","last_became_peered":"2023-09-23T15:32:18.620853+0000","last_unstale":"2023-09-23T15:35:10.758573+0000","last_undegraded":"2023-09-23T15:35:10.758573+0000","last_fullsized":"2023-09-23T15:35:10.758573+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:48.423947+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":245,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812504+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:35:10.812504+0000","last_peered":"2023-09-23T15:35:10.812504+0000","last_clean":"2023-09-23T15:35:10.812504+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:35:10.812504+0000","last_undegraded":"2023-09-23T15:35:10.812504+0000","last_fullsized":"2023-09-23T15:35:10.812504+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":45,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808300+0000","last_change":"2023-09-23T15:33:30.765277+0000","last_active":"2023-09-23T15:35:10.808300+0000","last_peered":"2023-09-23T15:35:10.808300+0000","last_clean":"2023-09-23T15:35:10.808300+0000","last_became_active":"2023-09-23T15:33:30.764903+0000","last_became_peered":"2023-09-23T15:33:30.764903+0000","last_unstale":"2023-09-23T15:35:10.808300+0000","last_undegraded":"2023-09-23T15:35:10.808300+0000","last_fullsized":"2023-09-23T15:35:10.808300+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:24:08.782060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":694,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812441+0000","last_change":"2023-09-23T15:32:18.640390+0000","last_active":"2023-09-23T15:35:10.812441+0000","last_peered":"2023-09-23T15:35:10.812441+0000","last_clean":"2023-09-23T15:35:10.812441+0000","last_became_active":"2023-09-23T15:32:18.640018+0000","last_became_peered":"2023-09-23T15:32:18.640018+0000","last_unstale":"2023-09-23T15:35:10.812441+0000","last_undegraded":"2023-09-23T15:35:10.812441+0000","last_fullsized":"2023-09-23T15:35:10.812441+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:30.211725+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":492,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758604+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:35:10.758604+0000","last_peered":"2023-09-23T15:35:10.758604+0000","last_clean":"2023-09-23T15:35:10.758604+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:35:10.758604+0000","last_undegraded":"2023-09-23T15:35:10.758604+0000","last_fullsized":"2023-09-23T15:35:10.758604+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":434,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.813174+0000","last_change":"2023-09-23T15:32:16.258406+0000","last_active":"2023-09-23T15:35:10.813174+0000","last_peered":"2023-09-23T15:35:10.813174+0000","last_clean":"2023-09-23T15:35:10.813174+0000","last_became_active":"2023-09-23T15:32:16.257317+0000","last_became_peered":"2023-09-23T15:32:16.257317+0000","last_unstale":"2023-09-23T15:35:10.813174+0000","last_undegraded":"2023-09-23T15:35:10.813174+0000","last_fullsized":"2023-09-23T15:35:10.813174+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:40.361415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":823,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019408+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:35:10.019408+0000","last_peered":"2023-09-23T15:35:10.019408+0000","last_clean":"2023-09-23T15:35:10.019408+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:35:10.019408+0000","last_undegraded":"2023-09-23T15:35:10.019408+0000","last_fullsized":"2023-09-23T15:35:10.019408+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":239,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812811+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:35:10.812811+0000","last_peered":"2023-09-23T15:35:10.812811+0000","last_clean":"2023-09-23T15:35:10.812811+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:35:10.812811+0000","last_undegraded":"2023-09-23T15:35:10.812811+0000","last_fullsized":"2023-09-23T15:35:10.812811+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":480,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758981+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:35:10.758981+0000","last_peered":"2023-09-23T15:35:10.758981+0000","last_clean":"2023-09-23T15:35:10.758981+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:35:10.758981+0000","last_undegraded":"2023-09-23T15:35:10.758981+0000","last_fullsized":"2023-09-23T15:35:10.758981+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":529,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019871+0000","last_change":"2023-09-23T15:32:18.637941+0000","last_active":"2023-09-23T15:35:10.019871+0000","last_peered":"2023-09-23T15:35:10.019871+0000","last_clean":"2023-09-23T15:35:10.019871+0000","last_became_active":"2023-09-23T15:32:18.637624+0000","last_became_peered":"2023-09-23T15:32:18.637624+0000","last_unstale":"2023-09-23T15:35:10.019871+0000","last_undegraded":"2023-09-23T15:35:10.019871+0000","last_fullsized":"2023-09-23T15:35:10.019871+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:45:17.385979+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":452,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759539+0000","last_change":"2023-09-23T15:32:16.261689+0000","last_active":"2023-09-23T15:35:10.759539+0000","last_peered":"2023-09-23T15:35:10.759539+0000","last_clean":"2023-09-23T15:35:10.759539+0000","last_became_active":"2023-09-23T15:32:16.251466+0000","last_became_peered":"2023-09-23T15:32:16.251466+0000","last_unstale":"2023-09-23T15:35:10.759539+0000","last_undegraded":"2023-09-23T15:35:10.759539+0000","last_fullsized":"2023-09-23T15:35:10.759539+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:44.874231+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":452,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812752+0000","last_change":"2023-09-23T15:30:26.706427+0000","last_active":"2023-09-23T15:35:10.812752+0000","last_peered":"2023-09-23T15:35:10.812752+0000","last_clean":"2023-09-23T15:35:10.812752+0000","last_became_active":"2023-09-23T15:30:26.706086+0000","last_became_peered":"2023-09-23T15:30:26.706086+0000","last_unstale":"2023-09-23T15:35:10.812752+0000","last_undegraded":"2023-09-23T15:35:10.812752+0000","last_fullsized":"2023-09-23T15:35:10.812752+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:42.578372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":792,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.813157+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:35:10.813157+0000","last_peered":"2023-09-23T15:35:10.813157+0000","last_clean":"2023-09-23T15:35:10.813157+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:35:10.813157+0000","last_undegraded":"2023-09-23T15:35:10.813157+0000","last_fullsized":"2023-09-23T15:35:10.813157+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":235,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812853+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:35:10.812853+0000","last_peered":"2023-09-23T15:35:10.812853+0000","last_clean":"2023-09-23T15:35:10.812853+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:35:10.812853+0000","last_undegraded":"2023-09-23T15:35:10.812853+0000","last_fullsized":"2023-09-23T15:35:10.812853+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":441,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812344+0000","last_change":"2023-09-23T15:32:16.256834+0000","last_active":"2023-09-23T15:35:10.812344+0000","last_peered":"2023-09-23T15:35:10.812344+0000","last_clean":"2023-09-23T15:35:10.812344+0000","last_became_active":"2023-09-23T15:32:16.256515+0000","last_became_peered":"2023-09-23T15:32:16.256515+0000","last_unstale":"2023-09-23T15:35:10.812344+0000","last_undegraded":"2023-09-23T15:35:10.812344+0000","last_fullsized":"2023-09-23T15:35:10.812344+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:09.264970+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":665,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758893+0000","last_change":"2023-09-23T15:30:48.890337+0000","last_active":"2023-09-23T15:35:10.758893+0000","last_peered":"2023-09-23T15:35:10.758893+0000","last_clean":"2023-09-23T15:35:10.758893+0000","last_became_active":"2023-09-23T15:30:26.693580+0000","last_became_peered":"2023-09-23T15:30:26.693580+0000","last_unstale":"2023-09-23T15:35:10.758893+0000","last_undegraded":"2023-09-23T15:35:10.758893+0000","last_fullsized":"2023-09-23T15:35:10.758893+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:55:25.353085+0000","scrub_duration":0.011955009000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":235,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812855+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:35:10.812855+0000","last_peered":"2023-09-23T15:35:10.812855+0000","last_clean":"2023-09-23T15:35:10.812855+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:35:10.812855+0000","last_undegraded":"2023-09-23T15:35:10.812855+0000","last_fullsized":"2023-09-23T15:35:10.812855+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":148,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759332+0000","last_change":"2023-09-23T15:33:30.765181+0000","last_active":"2023-09-23T15:35:10.759332+0000","last_peered":"2023-09-23T15:35:10.759332+0000","last_clean":"2023-09-23T15:35:10.759332+0000","last_became_active":"2023-09-23T15:33:30.764802+0000","last_became_peered":"2023-09-23T15:33:30.764802+0000","last_unstale":"2023-09-23T15:35:10.759332+0000","last_undegraded":"2023-09-23T15:35:10.759332+0000","last_fullsized":"2023-09-23T15:35:10.759332+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:36:15.536014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":763,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760411+0000","last_change":"2023-09-23T15:32:18.654963+0000","last_active":"2023-09-23T15:35:10.760411+0000","last_peered":"2023-09-23T15:35:10.760411+0000","last_clean":"2023-09-23T15:35:10.760411+0000","last_became_active":"2023-09-23T15:32:18.654586+0000","last_became_peered":"2023-09-23T15:32:18.654586+0000","last_unstale":"2023-09-23T15:35:10.760411+0000","last_undegraded":"2023-09-23T15:35:10.760411+0000","last_fullsized":"2023-09-23T15:35:10.760411+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:27:47.851804+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":237,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812916+0000","last_change":"2023-09-23T15:32:16.278269+0000","last_active":"2023-09-23T15:35:10.812916+0000","last_peered":"2023-09-23T15:35:10.812916+0000","last_clean":"2023-09-23T15:35:10.812916+0000","last_became_active":"2023-09-23T15:32:16.256883+0000","last_became_peered":"2023-09-23T15:32:16.256883+0000","last_unstale":"2023-09-23T15:35:10.812916+0000","last_undegraded":"2023-09-23T15:35:10.812916+0000","last_fullsized":"2023-09-23T15:35:10.812916+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:24.023700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":140,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":476,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760086+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:35:10.760086+0000","last_peered":"2023-09-23T15:35:10.760086+0000","last_clean":"2023-09-23T15:35:10.760086+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:35:10.760086+0000","last_undegraded":"2023-09-23T15:35:10.760086+0000","last_fullsized":"2023-09-23T15:35:10.760086+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":713,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760276+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:35:10.760276+0000","last_peered":"2023-09-23T15:35:10.760276+0000","last_clean":"2023-09-23T15:35:10.760276+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:35:10.760276+0000","last_undegraded":"2023-09-23T15:35:10.760276+0000","last_fullsized":"2023-09-23T15:35:10.760276+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":338,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759832+0000","last_change":"2023-09-23T15:32:16.251305+0000","last_active":"2023-09-23T15:35:10.759832+0000","last_peered":"2023-09-23T15:35:10.759832+0000","last_clean":"2023-09-23T15:35:10.759832+0000","last_became_active":"2023-09-23T15:32:16.250667+0000","last_became_peered":"2023-09-23T15:32:16.250667+0000","last_unstale":"2023-09-23T15:35:10.759832+0000","last_undegraded":"2023-09-23T15:35:10.759832+0000","last_fullsized":"2023-09-23T15:35:10.759832+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:38:30.961785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":499,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759060+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:35:10.759060+0000","last_peered":"2023-09-23T15:35:10.759060+0000","last_clean":"2023-09-23T15:35:10.759060+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:35:10.759060+0000","last_undegraded":"2023-09-23T15:35:10.759060+0000","last_fullsized":"2023-09-23T15:35:10.759060+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":748,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812623+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:35:10.812623+0000","last_peered":"2023-09-23T15:35:10.812623+0000","last_clean":"2023-09-23T15:35:10.812623+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:35:10.812623+0000","last_undegraded":"2023-09-23T15:35:10.812623+0000","last_fullsized":"2023-09-23T15:35:10.812623+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":150,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759121+0000","last_change":"2023-09-23T15:30:26.690336+0000","last_active":"2023-09-23T15:35:10.759121+0000","last_peered":"2023-09-23T15:35:10.759121+0000","last_clean":"2023-09-23T15:35:10.759121+0000","last_became_active":"2023-09-23T15:30:26.690141+0000","last_became_peered":"2023-09-23T15:30:26.690141+0000","last_unstale":"2023-09-23T15:35:10.759121+0000","last_undegraded":"2023-09-23T15:35:10.759121+0000","last_fullsized":"2023-09-23T15:35:10.759121+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:44.578933+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759651+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:35:10.759651+0000","last_peered":"2023-09-23T15:35:10.759651+0000","last_clean":"2023-09-23T15:35:10.759651+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:35:10.759651+0000","last_undegraded":"2023-09-23T15:35:10.759651+0000","last_fullsized":"2023-09-23T15:35:10.759651+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":601,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.807771+0000","last_change":"2023-09-23T15:33:05.479356+0000","last_active":"2023-09-23T15:35:10.807771+0000","last_peered":"2023-09-23T15:35:10.807771+0000","last_clean":"2023-09-23T15:35:10.807771+0000","last_became_active":"2023-09-23T15:32:18.637504+0000","last_became_peered":"2023-09-23T15:32:18.637504+0000","last_unstale":"2023-09-23T15:35:10.807771+0000","last_undegraded":"2023-09-23T15:35:10.807771+0000","last_fullsized":"2023-09-23T15:35:10.807771+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:05.479226+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.479226+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:39.341195+0000","scrub_duration":0.0055363749999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":463,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760177+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:35:10.760177+0000","last_peered":"2023-09-23T15:35:10.760177+0000","last_clean":"2023-09-23T15:35:10.760177+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:35:10.760177+0000","last_undegraded":"2023-09-23T15:35:10.760177+0000","last_fullsized":"2023-09-23T15:35:10.760177+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759608+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:35:10.759608+0000","last_peered":"2023-09-23T15:35:10.759608+0000","last_clean":"2023-09-23T15:35:10.759608+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:35:10.759608+0000","last_undegraded":"2023-09-23T15:35:10.759608+0000","last_fullsized":"2023-09-23T15:35:10.759608+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":794,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019316+0000","last_change":"2023-09-23T15:32:18.615573+0000","last_active":"2023-09-23T15:35:10.019316+0000","last_peered":"2023-09-23T15:35:10.019316+0000","last_clean":"2023-09-23T15:35:10.019316+0000","last_became_active":"2023-09-23T15:32:18.615232+0000","last_became_peered":"2023-09-23T15:32:18.615232+0000","last_unstale":"2023-09-23T15:35:10.019316+0000","last_undegraded":"2023-09-23T15:35:10.019316+0000","last_fullsized":"2023-09-23T15:35:10.019316+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:11.272034+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":152,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759421+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:35:10.759421+0000","last_peered":"2023-09-23T15:35:10.759421+0000","last_clean":"2023-09-23T15:35:10.759421+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:35:10.759421+0000","last_undegraded":"2023-09-23T15:35:10.759421+0000","last_fullsized":"2023-09-23T15:35:10.759421+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":148,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760360+0000","last_change":"2023-09-23T15:32:16.253887+0000","last_active":"2023-09-23T15:35:10.760360+0000","last_peered":"2023-09-23T15:35:10.760360+0000","last_clean":"2023-09-23T15:35:10.760360+0000","last_became_active":"2023-09-23T15:32:16.253556+0000","last_became_peered":"2023-09-23T15:32:16.253556+0000","last_unstale":"2023-09-23T15:35:10.760360+0000","last_undegraded":"2023-09-23T15:35:10.760360+0000","last_fullsized":"2023-09-23T15:35:10.760360+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:16.930439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":596,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808214+0000","last_change":"2023-09-23T15:33:30.357966+0000","last_active":"2023-09-23T15:35:10.808214+0000","last_peered":"2023-09-23T15:35:10.808214+0000","last_clean":"2023-09-23T15:35:10.808214+0000","last_became_active":"2023-09-23T15:32:18.637860+0000","last_became_peered":"2023-09-23T15:32:18.637860+0000","last_unstale":"2023-09-23T15:35:10.808214+0000","last_undegraded":"2023-09-23T15:35:10.808214+0000","last_fullsized":"2023-09-23T15:35:10.808214+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:30.357860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:30.357860+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:04.776475+0000","scrub_duration":0.0015641730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":463,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018813+0000","last_change":"2023-09-23T15:30:26.705171+0000","last_active":"2023-09-23T15:35:10.018813+0000","last_peered":"2023-09-23T15:35:10.018813+0000","last_clean":"2023-09-23T15:35:10.018813+0000","last_became_active":"2023-09-23T15:30:26.704853+0000","last_became_peered":"2023-09-23T15:30:26.704853+0000","last_unstale":"2023-09-23T15:35:10.018813+0000","last_undegraded":"2023-09-23T15:35:10.018813+0000","last_fullsized":"2023-09-23T15:35:10.018813+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:11.875343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":431,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021970+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:35:10.021970+0000","last_peered":"2023-09-23T15:35:10.021970+0000","last_clean":"2023-09-23T15:35:10.021970+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:35:10.021970+0000","last_undegraded":"2023-09-23T15:35:10.021970+0000","last_fullsized":"2023-09-23T15:35:10.021970+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":736,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760490+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:35:10.760490+0000","last_peered":"2023-09-23T15:35:10.760490+0000","last_clean":"2023-09-23T15:35:10.760490+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:35:10.760490+0000","last_undegraded":"2023-09-23T15:35:10.760490+0000","last_fullsized":"2023-09-23T15:35:10.760490+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"488'163","reported_seq":1015,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020669+0000","last_change":"2023-09-23T15:34:54.709256+0000","last_active":"2023-09-23T15:35:10.020669+0000","last_peered":"2023-09-23T15:35:10.020669+0000","last_clean":"2023-09-23T15:35:10.020669+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:35:10.020669+0000","last_undegraded":"2023-09-23T15:35:10.020669+0000","last_fullsized":"2023-09-23T15:35:10.020669+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021979399999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":726,"num_read_kb":1976,"num_write":174,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808578+0000","last_change":"2023-09-23T15:32:16.240883+0000","last_active":"2023-09-23T15:35:10.808578+0000","last_peered":"2023-09-23T15:35:10.808578+0000","last_clean":"2023-09-23T15:35:10.808578+0000","last_became_active":"2023-09-23T15:32:16.240541+0000","last_became_peered":"2023-09-23T15:32:16.240541+0000","last_unstale":"2023-09-23T15:35:10.808578+0000","last_undegraded":"2023-09-23T15:35:10.808578+0000","last_fullsized":"2023-09-23T15:35:10.808578+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:05.987331+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":470,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808324+0000","last_change":"2023-09-23T15:32:16.239137+0000","last_active":"2023-09-23T15:35:10.808324+0000","last_peered":"2023-09-23T15:35:10.808324+0000","last_clean":"2023-09-23T15:35:10.808324+0000","last_became_active":"2023-09-23T15:32:16.238791+0000","last_became_peered":"2023-09-23T15:32:16.238791+0000","last_unstale":"2023-09-23T15:35:10.808324+0000","last_undegraded":"2023-09-23T15:35:10.808324+0000","last_fullsized":"2023-09-23T15:35:10.808324+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:44:51.104255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,0],"acting":[2,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":594,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758654+0000","last_change":"2023-09-23T15:33:05.797574+0000","last_active":"2023-09-23T15:35:10.758654+0000","last_peered":"2023-09-23T15:35:10.758654+0000","last_clean":"2023-09-23T15:35:10.758654+0000","last_became_active":"2023-09-23T15:32:18.631881+0000","last_became_peered":"2023-09-23T15:32:18.631881+0000","last_unstale":"2023-09-23T15:35:10.758654+0000","last_undegraded":"2023-09-23T15:35:10.758654+0000","last_fullsized":"2023-09-23T15:35:10.758654+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.797446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.797446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:23.420392+0000","scrub_duration":0.0090775909999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,4],"acting":[0,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"488'157","reported_seq":1122,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020107+0000","last_change":"2023-09-23T15:34:54.709240+0000","last_active":"2023-09-23T15:35:10.020107+0000","last_peered":"2023-09-23T15:35:10.020107+0000","last_clean":"2023-09-23T15:35:10.020107+0000","last_became_active":"2023-09-23T15:32:16.240946+0000","last_became_peered":"2023-09-23T15:32:16.240946+0000","last_unstale":"2023-09-23T15:35:10.020107+0000","last_undegraded":"2023-09-23T15:35:10.020107+0000","last_fullsized":"2023-09-23T15:35:10.020107+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:38.182103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033387800000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1406,"num_read_kb":7738,"num_write":130,"num_write_kb":5863,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760394+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:35:10.760394+0000","last_peered":"2023-09-23T15:35:10.760394+0000","last_clean":"2023-09-23T15:35:10.760394+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:35:10.760394+0000","last_undegraded":"2023-09-23T15:35:10.760394+0000","last_fullsized":"2023-09-23T15:35:10.760394+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":396,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020454+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:35:10.020454+0000","last_peered":"2023-09-23T15:35:10.020454+0000","last_clean":"2023-09-23T15:35:10.020454+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:35:10.020454+0000","last_undegraded":"2023-09-23T15:35:10.020454+0000","last_fullsized":"2023-09-23T15:35:10.020454+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":859,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812726+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:35:10.812726+0000","last_peered":"2023-09-23T15:35:10.812726+0000","last_clean":"2023-09-23T15:35:10.812726+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:35:10.812726+0000","last_undegraded":"2023-09-23T15:35:10.812726+0000","last_fullsized":"2023-09-23T15:35:10.812726+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"488'400","reported_seq":1922,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808737+0000","last_change":"2023-09-23T15:34:54.705587+0000","last_active":"2023-09-23T15:35:10.808737+0000","last_peered":"2023-09-23T15:35:10.808737+0000","last_clean":"2023-09-23T15:35:10.808737+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:35:10.808737+0000","last_undegraded":"2023-09-23T15:35:10.808737+0000","last_fullsized":"2023-09-23T15:35:10.808737+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047212799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2483,"num_read_kb":3714,"num_write":351,"num_write_kb":5627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":392,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.018974+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:35:10.018974+0000","last_peered":"2023-09-23T15:35:10.018974+0000","last_clean":"2023-09-23T15:35:10.018974+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:35:10.018974+0000","last_undegraded":"2023-09-23T15:35:10.018974+0000","last_fullsized":"2023-09-23T15:35:10.018974+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":409,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021896+0000","last_change":"2023-09-23T15:32:16.250237+0000","last_active":"2023-09-23T15:35:10.021896+0000","last_peered":"2023-09-23T15:35:10.021896+0000","last_clean":"2023-09-23T15:35:10.021896+0000","last_became_active":"2023-09-23T15:32:16.249918+0000","last_became_peered":"2023-09-23T15:32:16.249918+0000","last_unstale":"2023-09-23T15:35:10.021896+0000","last_undegraded":"2023-09-23T15:35:10.021896+0000","last_fullsized":"2023-09-23T15:35:10.021896+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:33:31.035969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"488'306","reported_seq":976,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.809074+0000","last_change":"2023-09-23T15:34:54.705642+0000","last_active":"2023-09-23T15:35:10.809074+0000","last_peered":"2023-09-23T15:35:10.809074+0000","last_clean":"2023-09-23T15:35:10.809074+0000","last_became_active":"2023-09-23T15:32:18.623239+0000","last_became_peered":"2023-09-23T15:32:18.623239+0000","last_unstale":"2023-09-23T15:35:10.809074+0000","last_undegraded":"2023-09-23T15:35:10.809074+0000","last_fullsized":"2023-09-23T15:35:10.809074+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:50.264791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022811199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1196,"num_read_kb":2267,"num_write":287,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":802,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812521+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:35:10.812521+0000","last_peered":"2023-09-23T15:35:10.812521+0000","last_clean":"2023-09-23T15:35:10.812521+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:35:10.812521+0000","last_undegraded":"2023-09-23T15:35:10.812521+0000","last_fullsized":"2023-09-23T15:35:10.812521+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":347,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760351+0000","last_change":"2023-09-23T15:32:16.251284+0000","last_active":"2023-09-23T15:35:10.760351+0000","last_peered":"2023-09-23T15:35:10.760351+0000","last_clean":"2023-09-23T15:35:10.760351+0000","last_became_active":"2023-09-23T15:32:16.250677+0000","last_became_peered":"2023-09-23T15:32:16.250677+0000","last_unstale":"2023-09-23T15:35:10.760351+0000","last_undegraded":"2023-09-23T15:35:10.760351+0000","last_fullsized":"2023-09-23T15:35:10.760351+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:45.512698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":488,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760265+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:35:10.760265+0000","last_peered":"2023-09-23T15:35:10.760265+0000","last_clean":"2023-09-23T15:35:10.760265+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:35:10.760265+0000","last_undegraded":"2023-09-23T15:35:10.760265+0000","last_fullsized":"2023-09-23T15:35:10.760265+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":840,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.019175+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:35:10.019175+0000","last_peered":"2023-09-23T15:35:10.019175+0000","last_clean":"2023-09-23T15:35:10.019175+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:35:10.019175+0000","last_undegraded":"2023-09-23T15:35:10.019175+0000","last_fullsized":"2023-09-23T15:35:10.019175+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"488'240","reported_seq":626,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812402+0000","last_change":"2023-09-23T15:34:54.708597+0000","last_active":"2023-09-23T15:35:10.812402+0000","last_peered":"2023-09-23T15:35:10.812402+0000","last_clean":"2023-09-23T15:35:10.812402+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:35:10.812402+0000","last_undegraded":"2023-09-23T15:35:10.812402+0000","last_fullsized":"2023-09-23T15:35:10.812402+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013449600000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1148,"num_read_kb":1983,"num_write":238,"num_write_kb":6264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":503,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.809999+0000","last_change":"2023-09-23T15:35:10.809299+0000","last_active":"2023-09-23T15:35:10.809999+0000","last_peered":"2023-09-23T15:35:10.809999+0000","last_clean":"2023-09-23T15:35:10.809999+0000","last_became_active":"2023-09-23T15:35:10.808955+0000","last_became_peered":"2023-09-23T15:35:10.808955+0000","last_unstale":"2023-09-23T15:35:10.809999+0000","last_undegraded":"2023-09-23T15:35:10.809999+0000","last_fullsized":"2023-09-23T15:35:10.809999+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:53.650771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":479,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758752+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:35:10.758752+0000","last_peered":"2023-09-23T15:35:10.758752+0000","last_clean":"2023-09-23T15:35:10.758752+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:35:10.758752+0000","last_undegraded":"2023-09-23T15:35:10.758752+0000","last_fullsized":"2023-09-23T15:35:10.758752+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":822,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760638+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:35:10.760638+0000","last_peered":"2023-09-23T15:35:10.760638+0000","last_clean":"2023-09-23T15:35:10.760638+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:35:10.760638+0000","last_undegraded":"2023-09-23T15:35:10.760638+0000","last_fullsized":"2023-09-23T15:35:10.760638+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"488'207","reported_seq":1450,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812979+0000","last_change":"2023-09-23T15:34:54.709815+0000","last_active":"2023-09-23T15:35:10.812979+0000","last_peered":"2023-09-23T15:35:10.812979+0000","last_clean":"2023-09-23T15:35:10.812979+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:35:10.812979+0000","last_undegraded":"2023-09-23T15:35:10.812979+0000","last_fullsized":"2023-09-23T15:35:10.812979+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":207,"log_dups_size":0,"ondisk_log_size":207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038879700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1613,"num_read_kb":2033,"num_write":161,"num_write_kb":409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":385,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020610+0000","last_change":"2023-09-23T15:32:16.239062+0000","last_active":"2023-09-23T15:35:10.020610+0000","last_peered":"2023-09-23T15:35:10.020610+0000","last_clean":"2023-09-23T15:35:10.020610+0000","last_became_active":"2023-09-23T15:32:16.238742+0000","last_became_peered":"2023-09-23T15:32:16.238742+0000","last_unstale":"2023-09-23T15:35:10.020610+0000","last_undegraded":"2023-09-23T15:35:10.020610+0000","last_fullsized":"2023-09-23T15:35:10.020610+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:00:28.697401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":487,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759216+0000","last_change":"2023-09-23T15:32:16.250477+0000","last_active":"2023-09-23T15:35:10.759216+0000","last_peered":"2023-09-23T15:35:10.759216+0000","last_clean":"2023-09-23T15:35:10.759216+0000","last_became_active":"2023-09-23T15:32:16.249968+0000","last_became_peered":"2023-09-23T15:32:16.249968+0000","last_unstale":"2023-09-23T15:35:10.759216+0000","last_undegraded":"2023-09-23T15:35:10.759216+0000","last_fullsized":"2023-09-23T15:35:10.759216+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:45:43.524682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,1],"acting":[0,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":787,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759685+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:35:10.759685+0000","last_peered":"2023-09-23T15:35:10.759685+0000","last_clean":"2023-09-23T15:35:10.759685+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:35:10.759685+0000","last_undegraded":"2023-09-23T15:35:10.759685+0000","last_fullsized":"2023-09-23T15:35:10.759685+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":269,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808818+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:35:10.808818+0000","last_peered":"2023-09-23T15:35:10.808818+0000","last_clean":"2023-09-23T15:35:10.808818+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:35:10.808818+0000","last_undegraded":"2023-09-23T15:35:10.808818+0000","last_fullsized":"2023-09-23T15:35:10.808818+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"488'320","reported_seq":1833,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.021119+0000","last_change":"2023-09-23T15:34:54.710828+0000","last_active":"2023-09-23T15:35:10.021119+0000","last_peered":"2023-09-23T15:35:10.021119+0000","last_clean":"2023-09-23T15:35:10.021119+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:35:10.021119+0000","last_undegraded":"2023-09-23T15:35:10.021119+0000","last_fullsized":"2023-09-23T15:35:10.021119+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":320,"log_dups_size":0,"ondisk_log_size":320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011064320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2324,"num_read_kb":4368,"num_write":216,"num_write_kb":2345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":443,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812539+0000","last_change":"2023-09-23T15:30:26.710395+0000","last_active":"2023-09-23T15:35:10.812539+0000","last_peered":"2023-09-23T15:35:10.812539+0000","last_clean":"2023-09-23T15:35:10.812539+0000","last_became_active":"2023-09-23T15:30:26.710076+0000","last_became_peered":"2023-09-23T15:30:26.710076+0000","last_unstale":"2023-09-23T15:35:10.812539+0000","last_undegraded":"2023-09-23T15:35:10.812539+0000","last_fullsized":"2023-09-23T15:35:10.812539+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:57.313559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":672,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812733+0000","last_change":"2023-09-23T15:32:18.639607+0000","last_active":"2023-09-23T15:35:10.812733+0000","last_peered":"2023-09-23T15:35:10.812733+0000","last_clean":"2023-09-23T15:35:10.812733+0000","last_became_active":"2023-09-23T15:32:18.639236+0000","last_became_peered":"2023-09-23T15:32:18.639236+0000","last_unstale":"2023-09-23T15:35:10.812733+0000","last_undegraded":"2023-09-23T15:35:10.812733+0000","last_fullsized":"2023-09-23T15:35:10.812733+0000","mapping_epoch":445,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":446,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:41:34.044610+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"488'157","reported_seq":1108,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760654+0000","last_change":"2023-09-23T15:34:54.714657+0000","last_active":"2023-09-23T15:35:10.760654+0000","last_peered":"2023-09-23T15:35:10.760654+0000","last_clean":"2023-09-23T15:35:10.760654+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:35:10.760654+0000","last_undegraded":"2023-09-23T15:35:10.760654+0000","last_fullsized":"2023-09-23T15:35:10.760654+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031768699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1077,"num_read_kb":2328,"num_write":133,"num_write_kb":2319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":538,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758833+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:35:10.758833+0000","last_peered":"2023-09-23T15:35:10.758833+0000","last_clean":"2023-09-23T15:35:10.758833+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:35:10.758833+0000","last_undegraded":"2023-09-23T15:35:10.758833+0000","last_fullsized":"2023-09-23T15:35:10.758833+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":148,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759500+0000","last_change":"2023-09-23T15:32:16.252192+0000","last_active":"2023-09-23T15:35:10.759500+0000","last_peered":"2023-09-23T15:35:10.759500+0000","last_clean":"2023-09-23T15:35:10.759500+0000","last_became_active":"2023-09-23T15:32:16.251805+0000","last_became_peered":"2023-09-23T15:32:16.251805+0000","last_unstale":"2023-09-23T15:35:10.759500+0000","last_undegraded":"2023-09-23T15:35:10.759500+0000","last_fullsized":"2023-09-23T15:35:10.759500+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:26:59.903570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":706,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.758380+0000","last_change":"2023-09-23T15:30:26.740237+0000","last_active":"2023-09-23T15:35:10.758380+0000","last_peered":"2023-09-23T15:35:10.758380+0000","last_clean":"2023-09-23T15:35:10.758380+0000","last_became_active":"2023-09-23T15:30:26.690718+0000","last_became_peered":"2023-09-23T15:30:26.690718+0000","last_unstale":"2023-09-23T15:35:10.758380+0000","last_undegraded":"2023-09-23T15:35:10.758380+0000","last_fullsized":"2023-09-23T15:35:10.758380+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:00.439552+0000","scrub_duration":0.013441308000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":35,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759219+0000","last_change":"2023-09-23T15:34:12.777336+0000","last_active":"2023-09-23T15:35:10.759219+0000","last_peered":"2023-09-23T15:35:10.759219+0000","last_clean":"2023-09-23T15:35:10.759219+0000","last_became_active":"2023-09-23T15:34:12.777101+0000","last_became_peered":"2023-09-23T15:34:12.777101+0000","last_unstale":"2023-09-23T15:35:10.759219+0000","last_undegraded":"2023-09-23T15:35:10.759219+0000","last_fullsized":"2023-09-23T15:35:10.759219+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:13:25.546836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":27,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808480+0000","last_change":"2023-09-23T15:34:44.672844+0000","last_active":"2023-09-23T15:35:10.808480+0000","last_peered":"2023-09-23T15:35:10.808480+0000","last_clean":"2023-09-23T15:35:10.808480+0000","last_became_active":"2023-09-23T15:34:44.672523+0000","last_became_peered":"2023-09-23T15:34:44.672523+0000","last_unstale":"2023-09-23T15:35:10.808480+0000","last_undegraded":"2023-09-23T15:35:10.808480+0000","last_fullsized":"2023-09-23T15:35:10.808480+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:26.341913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":673,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760690+0000","last_change":"2023-09-23T15:30:28.373196+0000","last_active":"2023-09-23T15:35:10.760690+0000","last_peered":"2023-09-23T15:35:10.760690+0000","last_clean":"2023-09-23T15:35:10.760690+0000","last_became_active":"2023-09-23T15:30:26.699434+0000","last_became_peered":"2023-09-23T15:30:26.699434+0000","last_unstale":"2023-09-23T15:35:10.760690+0000","last_undegraded":"2023-09-23T15:35:10.760690+0000","last_fullsized":"2023-09-23T15:35:10.760690+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:44:26.970670+0000","scrub_duration":0.0018471900000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":466,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759609+0000","last_change":"2023-09-23T15:32:16.271454+0000","last_active":"2023-09-23T15:35:10.759609+0000","last_peered":"2023-09-23T15:35:10.759609+0000","last_clean":"2023-09-23T15:35:10.759609+0000","last_became_active":"2023-09-23T15:32:16.246283+0000","last_became_peered":"2023-09-23T15:32:16.246283+0000","last_unstale":"2023-09-23T15:35:10.759609+0000","last_undegraded":"2023-09-23T15:35:10.759609+0000","last_fullsized":"2023-09-23T15:35:10.759609+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:36.794240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760467+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:35:10.760467+0000","last_peered":"2023-09-23T15:35:10.760467+0000","last_clean":"2023-09-23T15:35:10.760467+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:35:10.760467+0000","last_undegraded":"2023-09-23T15:35:10.760467+0000","last_fullsized":"2023-09-23T15:35:10.760467+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":446,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759438+0000","last_change":"2023-09-23T15:32:16.280285+0000","last_active":"2023-09-23T15:35:10.759438+0000","last_peered":"2023-09-23T15:35:10.759438+0000","last_clean":"2023-09-23T15:35:10.759438+0000","last_became_active":"2023-09-23T15:32:16.251845+0000","last_became_peered":"2023-09-23T15:32:16.251845+0000","last_unstale":"2023-09-23T15:35:10.759438+0000","last_undegraded":"2023-09-23T15:35:10.759438+0000","last_fullsized":"2023-09-23T15:35:10.759438+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:07.938264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":11,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.809488+0000","last_change":"2023-09-23T15:35:10.809180+0000","last_active":"2023-09-23T15:35:10.809488+0000","last_peered":"2023-09-23T15:35:10.809488+0000","last_clean":"2023-09-23T15:35:10.809488+0000","last_became_active":"2023-09-23T15:35:10.808819+0000","last_became_peered":"2023-09-23T15:35:10.808819+0000","last_unstale":"2023-09-23T15:35:10.809488+0000","last_undegraded":"2023-09-23T15:35:10.809488+0000","last_fullsized":"2023-09-23T15:35:10.809488+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:22.677054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":683,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.812572+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:35:10.812572+0000","last_peered":"2023-09-23T15:35:10.812572+0000","last_clean":"2023-09-23T15:35:10.812572+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:35:10.812572+0000","last_undegraded":"2023-09-23T15:35:10.812572+0000","last_fullsized":"2023-09-23T15:35:10.812572+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":128,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.760803+0000","last_change":"2023-09-23T15:30:26.699597+0000","last_active":"2023-09-23T15:35:10.760803+0000","last_peered":"2023-09-23T15:35:10.760803+0000","last_clean":"2023-09-23T15:35:10.760803+0000","last_became_active":"2023-09-23T15:30:26.699233+0000","last_became_peered":"2023-09-23T15:30:26.699233+0000","last_unstale":"2023-09-23T15:35:10.760803+0000","last_undegraded":"2023-09-23T15:35:10.760803+0000","last_fullsized":"2023-09-23T15:35:10.760803+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.804745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":420,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.808854+0000","last_change":"2023-09-23T15:30:26.685257+0000","last_active":"2023-09-23T15:35:10.808854+0000","last_peered":"2023-09-23T15:35:10.808854+0000","last_clean":"2023-09-23T15:35:10.808854+0000","last_became_active":"2023-09-23T15:30:26.684954+0000","last_became_peered":"2023-09-23T15:30:26.684954+0000","last_unstale":"2023-09-23T15:35:10.808854+0000","last_undegraded":"2023-09-23T15:35:10.808854+0000","last_fullsized":"2023-09-23T15:35:10.808854+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:40:49.244950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":677,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.020238+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:35:10.020238+0000","last_peered":"2023-09-23T15:35:10.020238+0000","last_clean":"2023-09-23T15:35:10.020238+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:35:10.020238+0000","last_undegraded":"2023-09-23T15:35:10.020238+0000","last_fullsized":"2023-09-23T15:35:10.020238+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":467,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759417+0000","last_change":"2023-09-23T15:32:16.261750+0000","last_active":"2023-09-23T15:35:10.759417+0000","last_peered":"2023-09-23T15:35:10.759417+0000","last_clean":"2023-09-23T15:35:10.759417+0000","last_became_active":"2023-09-23T15:32:16.241839+0000","last_became_peered":"2023-09-23T15:32:16.241839+0000","last_unstale":"2023-09-23T15:35:10.759417+0000","last_undegraded":"2023-09-23T15:35:10.759417+0000","last_fullsized":"2023-09-23T15:35:10.759417+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:29:45.309569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":138,"reported_epoch":495,"state":"active+clean","last_fresh":"2023-09-23T15:35:10.759212+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:35:10.759212+0000","last_peered":"2023-09-23T15:35:10.759212+0000","last_clean":"2023-09-23T15:35:10.759212+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:35:10.759212+0000","last_undegraded":"2023-09-23T15:35:10.759212+0000","last_fullsized":"2023-09-23T15:35:10.759212+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":41,"num_bytes_recovered":10398,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":17066,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11973,"num_read_kb":26407,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":121,"num_bytes_recovered":4194323,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768796,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":149940,"kb_used_data":9580,"kb_used_omap":13,"kb_used_meta":140338,"kb_avail":93595212,"statfs":{"total":95995035648,"available":95841497088,"internally_reserved":0,"allocated":9809920,"data_stored":8277592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":143706206},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.48899999999999999,"5min":5.5439999999999996,"15min":5.6959999999999997},"min":{"1min":0.26800000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.72799999999999998,"5min":121.012,"15min":121.012},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.57899999999999996,"5min":5.2309999999999999,"15min":5.6539999999999999},"min":{"1min":0.435,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.73999999999999999,"5min":120.97,"15min":120.97},"last":0.57599999999999996}]},{"osd":1,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":2.2469999999999999,"15min":13.936999999999999},"min":{"1min":0.32900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.63200000000000001,"5min":56.816000000000003,"15min":157.096},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.9830000000000001,"15min":13.875},"min":{"1min":0.28299999999999997,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.68799999999999994,"5min":58.274000000000001,"15min":157.011},"last":0.51800000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":8.3000000000000007,"15min":3.3759999999999999},"min":{"1min":0.30599999999999999,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.70599999999999996,"5min":220.875,"15min":220.875},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.54300000000000004,"5min":8.0809999999999995,"15min":3.3450000000000002},"min":{"1min":0.32200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.75700000000000001,"5min":220.84399999999999,"15min":220.84399999999999},"last":0.625}]},{"osd":3,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.724,"15min":3.3969999999999998},"min":{"1min":0.40100000000000002,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.67500000000000004,"5min":10.061,"15min":76.019000000000005},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":1.8779999999999999,"15min":3.766},"min":{"1min":0.33900000000000002,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":0.71999999999999997,"5min":12.606999999999999,"15min":76.317999999999998},"last":0.60499999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":3.6520000000000001,"15min":1.6259999999999999},"min":{"1min":0.27500000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.68300000000000005,"5min":170.15799999999999,"15min":170.15799999999999},"last":0.442},{"interface":"front","average":{"1min":0.50900000000000001,"5min":3.7799999999999998,"15min":1.6479999999999999},"min":{"1min":0.33400000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.71999999999999997,"5min":170.017,"15min":170.017},"last":0.35599999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.004,"15min":0.86699999999999999},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":0.749,"5min":7.2910000000000004,"15min":7.2910000000000004},"last":0.377},{"interface":"front","average":{"1min":0.627,"5min":1.4750000000000001,"15min":1.0029999999999999},"min":{"1min":0.36499999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.79500000000000004,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":4.7359999999999998,"15min":2.073},"min":{"1min":0.34999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.80200000000000005,"5min":262.38099999999997,"15min":262.38099999999997},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.62,"5min":4.7850000000000001,"15min":2.0960000000000001},"min":{"1min":0.39600000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.81499999999999995,"5min":261.488,"15min":261.488},"last":0.39500000000000002}]}]},{"osd":6,"up_from":304,"seq":1305670058139,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213164,"kb_used_data":5540,"kb_used_omap":13,"kb_used_meta":207602,"kb_avail":93531988,"statfs":{"total":95995035648,"available":95776755712,"internally_reserved":0,"allocated":5672960,"data_stored":4148707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":212584803},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":3.6600000000000001,"15min":3.319},"min":{"1min":0.26900000000000002,"5min":0.217,"15min":0.217},"max":{"1min":0.99399999999999999,"5min":49.441000000000003,"15min":184.53200000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.48399999999999999,"5min":4.3380000000000001,"15min":3.4039999999999999},"min":{"1min":0.25,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.67200000000000004,"5min":48.880000000000003,"15min":86.227999999999994},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.4379999999999999,"15min":2.3740000000000001},"min":{"1min":0.29599999999999999,"5min":0.248,"15min":0.221},"max":{"1min":0.60999999999999999,"5min":7.907,"15min":86.534000000000006},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.51700000000000002,"5min":1.2629999999999999,"15min":2.202},"min":{"1min":0.32600000000000001,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.70999999999999996,"5min":6.4909999999999997,"15min":86.144000000000005},"last":0.49199999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":2.847,"15min":2.1339999999999999},"min":{"1min":0.29499999999999998,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.751,"5min":63.512,"15min":63.512},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":3.1739999999999999,"15min":2.2549999999999999},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":0.73399999999999999,"5min":63.526000000000003,"15min":63.526000000000003},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":1.478,"15min":2.456},"min":{"1min":0.378,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":1.081,"5min":11.176,"15min":142.67599999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.55400000000000005,"5min":2.069,"15min":2.8559999999999999},"min":{"1min":0.34799999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.754,"5min":11.795,"15min":142.541},"last":0.44600000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":2.3820000000000001,"15min":1.4490000000000001},"min":{"1min":0.34499999999999997,"5min":0.26400000000000001,"15min":0.23999999999999999},"max":{"1min":0.73199999999999998,"5min":128.11600000000001,"15min":128.11600000000001},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.51000000000000001,"5min":2.9460000000000002,"15min":2.016},"min":{"1min":0.312,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.79500000000000004,"5min":128.095,"15min":128.095},"last":0.40899999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":3.0350000000000001,"15min":2.698},"min":{"1min":0.35299999999999998,"5min":0.29899999999999999,"15min":0.248},"max":{"1min":0.70899999999999996,"5min":124.929,"15min":130.91300000000001},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.59299999999999997,"5min":2.3759999999999999,"15min":2.2719999999999998},"min":{"1min":0.40100000000000002,"5min":0.307,"15min":0.29099999999999998},"max":{"1min":0.83699999999999997,"5min":124.901,"15min":130.92599999999999},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:34:58 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.5389999999999999,"15min":1.3240000000000001},"min":{"1min":0.42899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.77800000000000002,"5min":10.087,"15min":10.087},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.61899999999999999,"5min":1.0109999999999999,"15min":1.3089999999999999},"min":{"1min":0.48799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":0.85999999999999999,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.47299999999999998}]}]},{"osd":1,"up_from":183,"seq":785979015439,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700528,"kb_used_data":10600,"kb_used_omap":6,"kb_used_meta":689913,"kb_avail":93044624,"statfs":{"total":95995035648,"available":95277694976,"internally_reserved":0,"allocated":10854400,"data_stored":8406529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":706470987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":5.1740000000000004,"5min":5.2039999999999997,"15min":3.7290000000000001},"min":{"1min":0.23200000000000001,"5min":0.216,"15min":0.191},"max":{"1min":47.207999999999998,"5min":101.27800000000001,"15min":101.27800000000001},"last":0.33900000000000002},{"interface":"front","average":{"1min":5.1289999999999996,"5min":5.4489999999999998,"15min":4.0570000000000004},"min":{"1min":0.245,"5min":0.23799999999999999,"15min":0.21099999999999999},"max":{"1min":48.692999999999998,"5min":101.051,"15min":101.051},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":3.5510000000000002,"5min":6.1239999999999997,"15min":3.831},"min":{"1min":0.249,"5min":0.23699999999999999,"15min":0.221},"max":{"1min":42.497,"5min":186.304,"15min":186.304},"last":0.29799999999999999},{"interface":"front","average":{"1min":3.3860000000000001,"5min":5.4960000000000004,"15min":3.4710000000000001},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":42.371000000000002,"5min":186.541,"15min":186.541},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.379,"5min":1.9399999999999999,"15min":2.8860000000000001},"min":{"1min":0.307,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":9.6709999999999994,"5min":13.744,"15min":64.540999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.502,"5min":1.746,"15min":2.8809999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.066000000000001,"5min":13.98,"15min":67.465000000000003},"last":0.436}]},{"osd":4,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.907,"5min":1.9810000000000001,"15min":2.9319999999999999},"min":{"1min":0.378,"5min":0.33800000000000002,"15min":0.29799999999999999},"max":{"1min":8.5289999999999999,"5min":13.840999999999999,"15min":80.570999999999998},"last":0.57899999999999996},{"interface":"front","average":{"1min":1.397,"5min":1.5409999999999999,"15min":2.5779999999999998},"min":{"1min":0.33900000000000002,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":8.7729999999999997,"5min":8.7729999999999997,"15min":80.75},"last":1.0029999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:34:28 2023","interfaces":[{"interface":"back","average":{"1min":1.649,"5min":1.79,"15min":3.2949999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.27400000000000002},"max":{"1min":10.936999999999999,"5min":30.009,"15min":173.18799999999999},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.5309999999999999,"5min":2.4399999999999999,"15min":3.6480000000000001},"min":{"1min":0.39800000000000002,"5min":0.34100000000000003,"15min":0.29099999999999998},"max":{"1min":8.6370000000000005,"5min":29.983000000000001,"15min":172.822},"last":0.86599999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:34:45 2023","interfaces":[{"interface":"back","average":{"1min":1.214,"5min":2.2069999999999999,"15min":2.7559999999999998},"min":{"1min":0.40799999999999997,"5min":0.377,"15min":0.34899999999999998},"max":{"1min":5.6150000000000002,"5min":13.912000000000001,"15min":54.198},"last":0.82399999999999995},{"interface":"front","average":{"1min":1.012,"5min":1.4750000000000001,"15min":1.885},"min":{"1min":0.40400000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":9.2710000000000008,"5min":10.996,"15min":15.605},"last":0.66100000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:34:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.482,"15min":1.9059999999999999},"min":{"1min":0.44600000000000001,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":1.117,"5min":11.032999999999999,"15min":11.032999999999999},"last":0.754},{"interface":"front","average":{"1min":0.83799999999999997,"5min":2.129,"15min":3.9079999999999999},"min":{"1min":0.49199999999999999,"5min":0.41899999999999998,"15min":0.40899999999999997},"max":{"1min":3.028,"5min":13.945,"15min":54.223999999999997},"last":0.89200000000000002}]}]},{"osd":0,"up_from":95,"seq":408021893486,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898440,"kb_used_data":14592,"kb_used_omap":8,"kb_used_meta":883831,"kb_avail":92846712,"statfs":{"total":95995035648,"available":95075033088,"internally_reserved":0,"allocated":14942208,"data_stored":12659235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":905043540},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:34:18 2023","interfaces":[{"interface":"back","average":{"1min":4.3609999999999998,"5min":2.899,"15min":4.4119999999999999},"min":{"1min":0.23999999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":63.393000000000001,"5min":63.393000000000001,"15min":141.52199999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":3.6070000000000002,"5min":2.5209999999999999,"15min":4.2999999999999998},"min":{"1min":0.23899999999999999,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":63.417999999999999,"5min":63.417999999999999,"15min":141.62799999999999},"last":0.35799999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":3.0419999999999998,"15min":2.5049999999999999},"min":{"1min":0.246,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":0.59699999999999998,"5min":66.132999999999996,"15min":66.132999999999996},"last":0.379},{"interface":"front","average":{"1min":0.42599999999999999,"5min":3.4460000000000002,"15min":2.5489999999999999},"min":{"1min":0.25900000000000001,"5min":0.253,"15min":0.20300000000000001},"max":{"1min":0.66300000000000003,"5min":68.296999999999997,"15min":68.296999999999997},"last":0.33000000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:35:09 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":3.0899999999999999,"15min":4.468},"min":{"1min":0.29399999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.66400000000000003,"5min":65.251999999999995,"15min":197.78899999999999},"last":0.42599999999999999},{"interface":"front","average":{"1min":0.47899999999999998,"5min":3.258,"15min":4.4930000000000003},"min":{"1min":0.33300000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68700000000000006,"5min":65.269999999999996,"15min":200.58799999999999},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":2.7890000000000001,"15min":2.2770000000000001},"min":{"1min":0.32500000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.81999999999999995,"5min":60.771999999999998,"15min":60.771999999999998},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.61199999999999999,"5min":2.7429999999999999,"15min":2.2909999999999999},"min":{"1min":0.34200000000000003,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.86799999999999999,"5min":60.725999999999999,"15min":60.725999999999999},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":2.081,"15min":2.141},"min":{"1min":0.38800000000000001,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":1.167,"5min":33.158999999999999,"15min":44.57},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":2.1840000000000002,"15min":2.3879999999999999},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.89300000000000002,"5min":33.140000000000001,"15min":44.588000000000001},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":2.8410000000000002,"15min":2.6819999999999999},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":0.82499999999999996,"5min":62.168999999999997,"15min":62.168999999999997},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.65900000000000003,"5min":2.8119999999999998,"15min":2.9550000000000001},"min":{"1min":0.38800000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.91700000000000004,"5min":61.972000000000001,"15min":61.972000000000001},"last":0.55300000000000005}]},{"osd":7,"last update":"Sat Sep 23 15:34:53 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":2.5219999999999998,"15min":2.2919999999999998},"min":{"1min":0.36899999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":1.141,"5min":36.848999999999997,"15min":36.848999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.65600000000000003,"5min":2.6280000000000001,"15min":3.4420000000000002},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.86799999999999999,"5min":36.947000000000003,"15min":36.947000000000003},"last":0.63300000000000001}]}]},{"osd":2,"up_from":105,"seq":450971566436,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854588,"kb_used_data":10420,"kb_used_omap":11,"kb_used_meta":844148,"kb_avail":92890564,"statfs":{"total":95995035648,"available":95119937536,"internally_reserved":0,"allocated":10670080,"data_stored":8398640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":864408336},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":0.45900000000000002,"5min":3.6230000000000002,"15min":5.4619999999999997},"min":{"1min":0.23499999999999999,"5min":0.20499999999999999,"15min":0.19800000000000001},"max":{"1min":2.3730000000000002,"5min":82.878,"15min":235.517},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.71899999999999997,"5min":3.3809999999999998,"15min":5.3490000000000002},"min":{"1min":0.26400000000000001,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":7.4509999999999996,"5min":82.762,"15min":235.33799999999999},"last":0.34999999999999998}]},{"osd":1,"last update":"Sat Sep 23 15:34:31 2023","interfaces":[{"interface":"back","average":{"1min":1.5620000000000001,"5min":2.488,"15min":3.3010000000000002},"min":{"1min":0.25600000000000001,"5min":0.22700000000000001,"15min":0.20499999999999999},"max":{"1min":9.8339999999999996,"5min":65.906999999999996,"15min":71.819000000000003},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.4809999999999999,"15min":3.149},"min":{"1min":0.23300000000000001,"5min":0.22500000000000001,"15min":0.20899999999999999},"max":{"1min":12.984999999999999,"5min":65.222999999999999,"15min":71.805000000000007},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 15:35:07 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.7210000000000001,"15min":3.5489999999999999},"min":{"1min":0.25,"5min":0.216,"15min":0.216},"max":{"1min":2.0110000000000001,"5min":18.606999999999999,"15min":77.031999999999996},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.863,"15min":3.6579999999999999},"min":{"1min":0.28799999999999998,"5min":0.247,"15min":0.247},"max":{"1min":3.9350000000000001,"5min":11.420999999999999,"15min":76.742000000000004},"last":0.36799999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.5680000000000001,"5min":2.25,"15min":1.7929999999999999},"min":{"1min":0.36199999999999999,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":7.7880000000000003,"5min":63.747999999999998,"15min":63.747999999999998},"last":0.63200000000000001},{"interface":"front","average":{"1min":1.2230000000000001,"5min":2.5659999999999998,"15min":1.952},"min":{"1min":0.37,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.9509999999999996,"5min":63.954000000000001,"15min":63.954000000000001},"last":0.69099999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:34:38 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":1.4810000000000001,"15min":2.2639999999999998},"min":{"1min":0.39200000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":9.7739999999999991,"5min":9.9559999999999995,"15min":107.021},"last":0.65700000000000003},{"interface":"front","average":{"1min":1.1040000000000001,"5min":1.589,"15min":2.1899999999999999},"min":{"1min":0.42099999999999999,"5min":0.31,"15min":0.31},"max":{"1min":7.2759999999999998,"5min":9.577,"15min":107.136},"last":0.59199999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:34:46 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":3.3399999999999999,"15min":2.4860000000000002},"min":{"1min":0.40500000000000003,"5min":0.318,"15min":0.318},"max":{"1min":10.51,"5min":156.16,"15min":156.16},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.71299999999999997,"5min":3.3460000000000001,"15min":2.258},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":2.8879999999999999,"5min":155.226,"15min":155.226},"last":0.63800000000000001}]},{"osd":7,"last update":"Sat Sep 23 15:34:55 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.909,"15min":2.6779999999999999},"min":{"1min":0.47699999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.835,"5min":12.282,"15min":112.70699999999999},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.64000000000000001,"5min":1.9159999999999999,"15min":3.125},"min":{"1min":0.48299999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.84899999999999998,"5min":9.9280000000000008,"15min":109.402},"last":0.57299999999999995}]}]},{"osd":3,"up_from":110,"seq":472446402908,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866068,"kb_used_data":10508,"kb_used_omap":3,"kb_used_meta":855548,"kb_avail":92879084,"statfs":{"total":95995035648,"available":95108182016,"internally_reserved":0,"allocated":10760192,"data_stored":8466239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":876081321},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":4.3849999999999998,"15min":3.992},"min":{"1min":0.20999999999999999,"5min":0.187,"15min":0.183},"max":{"1min":0.57099999999999995,"5min":141.934,"15min":141.934},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.42799999999999999,"5min":5.0069999999999997,"15min":4.5229999999999997},"min":{"1min":0.23499999999999999,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":1.075,"5min":141.72800000000001,"15min":141.72800000000001},"last":0.38400000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:34:29 2023","interfaces":[{"interface":"back","average":{"1min":1.03,"5min":2.1190000000000002,"15min":3.8759999999999999},"min":{"1min":0.23300000000000001,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":3.9300000000000002,"5min":16.623000000000001,"15min":126.875},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.798,"15min":4.1879999999999997},"min":{"1min":0.33300000000000002,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":4.7729999999999997,"5min":13.401999999999999,"15min":128.249},"last":0.49299999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":5.194,"15min":3.036},"min":{"1min":0.26900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.0529999999999999,"5min":144.35599999999999,"15min":144.35599999999999},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.46000000000000002,"5min":5.7140000000000004,"15min":3.585},"min":{"1min":0.26100000000000001,"5min":0.25900000000000001,"15min":0.246},"max":{"1min":1.21,"5min":144.494,"15min":144.494},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":2.3799999999999999,"15min":2.0030000000000001},"min":{"1min":0.32200000000000001,"5min":0.29699999999999999,"15min":0.27500000000000002},"max":{"1min":1.1120000000000001,"5min":44.289000000000001,"15min":44.289000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.60099999999999998,"5min":2.3010000000000002,"15min":1.889},"min":{"1min":0.35799999999999998,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":1.0669999999999999,"5min":44.055,"15min":44.055},"last":0.54200000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:35:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":3.2930000000000001,"15min":3.2519999999999998},"min":{"1min":0.373,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":1.028,"5min":122.452,"15min":148.619},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.623,"5min":3.1299999999999999,"15min":3.0139999999999998},"min":{"1min":0.439,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":1.103,"5min":122.33499999999999,"15min":148.56999999999999},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:34:48 2023","interfaces":[{"interface":"back","average":{"1min":3.2570000000000001,"5min":2.8690000000000002,"15min":2.9249999999999998},"min":{"1min":0.40200000000000002,"5min":0.36399999999999999,"15min":0.32700000000000001},"max":{"1min":36.165999999999997,"5min":44.356000000000002,"15min":84.665000000000006},"last":0.58899999999999997},{"interface":"front","average":{"1min":3.2589999999999999,"5min":3.056,"15min":3.3900000000000001},"min":{"1min":0.46800000000000003,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":36.226999999999997,"5min":44.453000000000003,"15min":84.897999999999996},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:34:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":2.726,"15min":8.3719999999999999},"min":{"1min":0.45300000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":0.96599999999999997,"5min":121.738,"15min":216.56899999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.70799999999999996,"5min":3.3130000000000002,"15min":9.3520000000000003},"min":{"1min":0.49199999999999999,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":1.2310000000000001,"5min":122.10599999999999,"15min":216.548},"last":0.628}]}]},{"osd":4,"up_from":31,"seq":133143986675,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":524780,"kb_used_data":5220,"kb_used_omap":0,"kb_used_meta":519552,"kb_avail":93220372,"statfs":{"total":95995035648,"available":95457660928,"internally_reserved":0,"allocated":5345280,"data_stored":3820530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":532021248},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:34:35 2023","interfaces":[{"interface":"back","average":{"1min":0.86699999999999999,"5min":2.6000000000000001,"15min":3.6749999999999998},"min":{"1min":0.27700000000000002,"5min":0.222,"15min":0.222},"max":{"1min":4.577,"5min":36.865000000000002,"15min":162.75999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.70699999999999996,"5min":2.7890000000000001,"15min":3.7389999999999999},"min":{"1min":0.247,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":3.9009999999999998,"5min":37.081000000000003,"15min":162.34800000000001},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:34:23 2023","interfaces":[{"interface":"back","average":{"1min":1.1659999999999999,"5min":1.976,"15min":3.7690000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":7.0830000000000002,"5min":41.354999999999997,"15min":144.76900000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.79700000000000004,"5min":1.736,"15min":3.8149999999999999},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.25},"max":{"1min":3.1000000000000001,"5min":41.343000000000004,"15min":144.58799999999999},"last":0.45000000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:34:35 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":4.0970000000000004,"15min":2.972},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":5.5049999999999999,"5min":89.980000000000004,"15min":89.980000000000004},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.016,"5min":3.8410000000000002,"15min":2.98},"min":{"1min":0.38500000000000001,"5min":0.36199999999999999,"15min":0.30599999999999999},"max":{"1min":3.9729999999999999,"5min":90.016999999999996,"15min":90.016999999999996},"last":0.36499999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:34:14 2023","interfaces":[{"interface":"back","average":{"1min":0.92500000000000004,"5min":1.9730000000000001,"15min":3.6749999999999998},"min":{"1min":0.35199999999999998,"5min":0.308,"15min":0.30399999999999999},"max":{"1min":4.0679999999999996,"5min":35.692,"15min":156.46000000000001},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.77800000000000002,"5min":2.0249999999999999,"15min":3.7040000000000002},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":3.0049999999999999,"5min":35.920999999999999,"15min":156.74299999999999},"last":0.621}]},{"osd":5,"last update":"Sat Sep 23 15:34:23 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":1.3089999999999999,"15min":1.226},"min":{"1min":0.32500000000000001,"5min":0.27600000000000002,"15min":0.221},"max":{"1min":1.28,"5min":11.904999999999999,"15min":40.685000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.63900000000000001,"5min":1.379,"15min":1.341},"min":{"1min":0.33900000000000002,"5min":0.27500000000000002,"15min":0.246},"max":{"1min":3.044,"5min":12.882999999999999,"15min":38.439},"last":0.58999999999999997}]},{"osd":6,"last update":"Sat Sep 23 15:34:46 2023","interfaces":[{"interface":"back","average":{"1min":0.69199999999999995,"5min":4.0609999999999999,"15min":2.1299999999999999},"min":{"1min":0.34399999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":3.0699999999999998,"5min":228.93199999999999,"15min":228.93199999999999},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.76200000000000001,"5min":4.4850000000000003,"15min":2.4860000000000002},"min":{"1min":0.38900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":4.1870000000000003,"5min":232.54900000000001,"15min":232.54900000000001},"last":0.67000000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:34:49 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":3.117,"15min":1.9379999999999999},"min":{"1min":0.36199999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":4.3029999999999999,"5min":183.94800000000001,"15min":183.94800000000001},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.92300000000000004,"5min":3.323,"15min":1.7949999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":7.5060000000000002,"5min":183.72399999999999,"15min":183.72399999999999},"last":0.63100000000000001}]}]},{"osd":5,"up_from":0,"seq":167503725038,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:35:16.959 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:35:16.959 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T15:35:16.959 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T15:35:16.959 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T15:35:16.959 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T15:35:16.960 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T15:35:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:16 smithi027 ceph-mon[195233]: pgmap v1451: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:17.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:16 smithi017 ceph-mon[215646]: pgmap v1451: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:17.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:16 smithi017 ceph-mon[222878]: pgmap v1451: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:17.325 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:17.340 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T15:35:17.703 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:17.774 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:17 smithi017 ceph-mon[222878]: from='client.47675 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:18.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:17 smithi027 ceph-mon[195233]: from='client.47675 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:18.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:17 smithi017 ceph-mon[215646]: from='client.47675 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:35:18.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:18] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.43.0" 2023-09-23T15:35:18.853 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:18.854 INFO:teuthology.orchestra.run.smithi017.stdout: "success": "" 2023-09-23T15:35:18.854 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:18.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:18 smithi017 ceph-mon[215646]: pgmap v1452: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:18.881 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:18 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:35:18.882 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:18 smithi017 ceph-mon[222878]: pgmap v1452: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:18.882 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:18 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:35:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:18 smithi027 ceph-mon[195233]: pgmap v1452: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:18 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:35:19.391 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:19.392 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:19.392 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:35:19.392 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:35:19.392 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:20.226 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T15:35:20.703 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:20 smithi027 ceph-mon[195233]: pgmap v1453: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:21.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:20 smithi017 ceph-mon[215646]: pgmap v1453: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:21.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:20 smithi017 ceph-mon[222878]: pgmap v1453: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.630 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83112 9.a 9:5853ad9e:::benchmark_data_smithi017_543597_object83111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 506.66952747699997, 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0903713559999999, 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83112 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892026+0000", 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892024+0000", 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892085+0000", 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1613999999999997e-05 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892103+0000", 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7893e-05 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892110+0000", 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892224+0000", 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000114508 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892371+0000", 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014686500000000001 2023-09-23T15:35:22.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892552+0000", 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018046499999999999 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898624+0000", 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060722090000000003 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912313+0000", 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013688707 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912322+0000", 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.4839999999999993e-06 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.970323+0000", 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0580004119999999 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971026+0000", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070397199999999997 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982334+0000", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011307914000000001 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982397+0000", 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2570999999999994e-05 2023-09-23T15:35:22.637 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83117 9.a 9:51c7f339:::benchmark_data_smithi017_543597_object83116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 506.66602870499997, 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0870892780000001, 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.638 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83117 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895524+0000", 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895523+0000", 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895570+0000", 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6748e-05 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895583+0000", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3390000000000001e-05 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895590+0000", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0740000000000004e-06 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895803+0000", 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00021333300000000001 2023-09-23T15:35:22.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.895915+0000", 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000111697 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896025+0000", 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000109511 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.899384+0000", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033596300000000002 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912365+0000", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012981029999999999 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912370+0000", 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5549999999999996e-06 2023-09-23T15:35:22.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.971071+0000", 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.058701627 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.973148+0000", 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020767469999999999 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982568+0000", 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0094202609999999992 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.982614+0000", 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5259000000000002e-05 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83120 9.7 9:e8f3216b:::benchmark_data_smithi017_543597_object83119:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:35:22.645 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 506.66345331500003, 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.062896818, 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83120 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898100+0000", 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898098+0000", 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:22.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898169+0000", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0426000000000006e-05 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898180+0000", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1114999999999999e-05 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898186+0000", 2023-09-23T15:35:22.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0159999999999997e-06 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898238+0000", 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1777000000000002e-05 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898377+0000", 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138825 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.898557+0000", 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000180058 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.901069+0000", 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025124309999999999 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.650 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912790+0000", 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011721352000000001 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.912795+0000", 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5430000000000004e-06 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948511+0000", 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0357156709999999 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.949214+0000", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00070288299999999996 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960946+0000", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011732340000000001 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.960997+0000", 2023-09-23T15:35:22.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0702999999999999e-05 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88042 9.a 9:5af6f1a8:::benchmark_data_smithi017_543597_object4251:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e373)", 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 462.16802128099999, 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9167718300000001, 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:22.653 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88042 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393532+0000", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393530+0000", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393536+0000", 2023-09-23T15:35:22.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7370000000000003e-06 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393552+0000", 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5713999999999999e-05 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393560+0000", 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0800000000000006e-06 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393617+0000", 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6884e-05 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393770+0000", 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015349600000000001 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1", 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.393905+0000", 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013507699999999999 2023-09-23T15:35:22.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400071+0000", 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0061654379999999996 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.400083+0000", 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2109e-05 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389036+0000", 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.98895255699999995 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389049+0000", 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3633e-05 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303003+0000", 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.913953927 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for missing object", 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303184+0000", 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000181175 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310231+0000", 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.007047045 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.310304+0000", 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2308e-05 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88055 9.a 9:5bed7ee7:::benchmark_data_smithi017_543597_object4264:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 462.12155218200002, 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8698463969999999, 2023-09-23T15:35:22.661 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88055 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440001+0000", 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440000+0000", 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440006+0000", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5049999999999998e-06 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440018+0000", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2359e-05 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440025+0000", 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2849999999999997e-06 2023-09-23T15:35:22.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440084+0000", 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.855e-05 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.440092+0000", 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.585e-06 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389071+0000", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.94897856800000002 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.389079+0000", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6590000000000007e-06 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303210+0000", 2023-09-23T15:35:22.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91413175199999996 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303344+0000", 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013339999999999999 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,7", 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.303490+0000", 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014616299999999999 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.307392+0000", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003901655 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309301+0000", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019098609999999999 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309677+0000", 2023-09-23T15:35:22.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037558600000000002 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309714+0000", 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6791000000000001e-05 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.309847+0000", 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000133454 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.669 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88057 9.7 9:eebbf230:::benchmark_data_smithi017_543597_object4266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 462.11761437500002, 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.85708753, 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:22.670 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88057 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443939+0000", 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.671 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443938+0000", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443943+0000", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5079999999999999e-06 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443955+0000", 2023-09-23T15:35:22.672 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2754e-05 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.443965+0000", 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8339999999999994e-06 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444010+0000", 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5330000000000001e-05 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.673 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.444019+0000", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2320000000000001e-06 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387832+0000", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.943813609 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.387845+0000", 2023-09-23T15:35:22.674 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2853e-05 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297101+0000", 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90925576399999997 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297239+0000", 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013821800000000001 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.675 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.297322+0000", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3220999999999995e-05 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.299762+0000", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024393729999999999 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300155+0000", 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00039295999999999998 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.676 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300804+0000", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064965300000000004 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.300868+0000", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3898000000000004e-05 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:42.301026+0000", 2023-09-23T15:35:22.677 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015802500000000001 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137065 10.6 10:6a8d0ffd:::benchmark_data_smithi017_604517_object30954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 195.57464671400001, 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.18030616599999999, 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.678 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137065 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.679 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986906+0000", 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986904+0000", 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986913+0000", 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9069999999999996e-06 2023-09-23T15:35:22.680 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986930+0000", 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6911000000000001e-05 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.986937+0000", 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.092e-06 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.681 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987130+0000", 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00019227400000000001 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987251+0000", 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000121505 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.987367+0000", 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011581799999999999 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.682 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995041+0000", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0076740460000000003 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002478+0000", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074370130000000001 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167121+0000", 2023-09-23T15:35:22.683 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.164642593 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167181+0000", 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0134999999999999e-05 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167213+0000", 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1896000000000001e-05 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.684 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137074 10.6 10:6f342ab4:::benchmark_data_smithi017_604517_object30963:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 195.56633614399999, 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.172095518, 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.685 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137074 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995217+0000", 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.686 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995216+0000", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995222+0000", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.874e-06 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995233+0000", 2023-09-23T15:35:22.687 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1024e-05 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995237+0000", 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.741e-06 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995272+0000", 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4824999999999999e-05 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.688 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995354+0000", 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1582999999999995e-05 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.995426+0000", 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2383000000000001e-05 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002484+0000", 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0070572830000000001 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.689 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.006480+0000", 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039969519999999998 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167255+0000", 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16077435000000001 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.690 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167287+0000", 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2132000000000003e-05 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167312+0000", 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5531e-05 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137075 10.6 10:6cfb72a7:::benchmark_data_smithi017_604517_object30964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:35:22.691 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 195.56475316999999, 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17058852699999999, 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137075 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.692 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996800+0000", 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996799+0000", 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.693 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996805+0000", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4989999999999999e-06 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996826+0000", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1211000000000001e-05 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.996833+0000", 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0299999999999996e-06 2023-09-23T15:35:22.694 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.998844+0000", 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020100719999999999 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.998983+0000", 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013919599999999999 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:22.695 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.999117+0000", 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013448099999999999 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.008974+0000", 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098565070000000005 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009751+0000", 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00077736600000000004 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.696 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167338+0000", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.157587325 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167366+0000", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.7185000000000001e-05 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.167388+0000", 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2855999999999999e-05 2023-09-23T15:35:22.697 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:137078 10.6 10:63c718f8:::benchmark_data_smithi017_604517_object30967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 195.56453455400001, 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.17522478399999999, 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.698 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 137078 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997019+0000", 2023-09-23T15:35:22.699 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997018+0000", 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997025+0000", 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.182e-06 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.700 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997039+0000", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4168000000000001e-05 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:06.997045+0000", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8579999999999996e-06 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002510+0000", 2023-09-23T15:35:22.701 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0054650250000000001 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002630+0000", 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120388 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.002738+0000", 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000107558 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.702 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009016+0000", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0062775180000000002 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.009994+0000", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00097875099999999993 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172153+0000", 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.16215827899999999 2023-09-23T15:35:22.703 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172203+0000", 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0009000000000002e-05 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:07.172243+0000", 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0757999999999998e-05 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.704 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:140577 10.9 10:922c9838:::benchmark_data_smithi017_604517_object34466:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e440)", 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 190.91880791599999, 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.21956315500000001, 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.705 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140577 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642745+0000", 2023-09-23T15:35:22.706 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642743+0000", 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642751+0000", 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1459999999999999e-06 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.707 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642767+0000", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6663e-05 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642775+0000", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2960000000000001e-06 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642824+0000", 2023-09-23T15:35:22.708 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9147999999999998e-05 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.642955+0000", 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000131092 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,7", 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.643074+0000", 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000119286 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.709 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.646978+0000", 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039041100000000001 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.646997+0000", 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9102000000000001e-05 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.710 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862216+0000", 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.215218992 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862267+0000", 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0872999999999999e-05 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.862308+0000", 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1148e-05 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.711 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:140604 10.6 10:640c290b:::benchmark_data_smithi017_604517_object34493:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e441)", 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 190.83512108799999, 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22775473700000001, 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:22.712 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 140604 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726432+0000", 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:22.713 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726430+0000", 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726439+0000", 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1540000000000008e-06 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.714 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726461+0000", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1129e-05 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726469+0000", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1200000000000002e-06 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726524+0000", 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4840999999999999e-05 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.715 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726670+0000", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000146615 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.726781+0000", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011131799999999999 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.729958+0000", 2023-09-23T15:35:22.716 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031767980000000002 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.730207+0000", 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00024893199999999999 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954090+0000", 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.22388297099999999 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.717 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954147+0000", 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7240000000000001e-05 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:32:11.954187+0000", 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9335000000000002e-05 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:22.718 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:22.719 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:22.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:22 smithi027 ceph-mon[195233]: pgmap v1454: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:22.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:22 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:22.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[215646]: pgmap v1454: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:22.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:22.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:22.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[222878]: pgmap v1454: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:22.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:22.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:22 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:23.222 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T15:35:23.598 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:24.735 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T15:35:24.735 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T15:35:24.735 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: pgmap v1455: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: pgmap v1455: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:24.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:35:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:35:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: pgmap v1455: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:25.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:25.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:25.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:35:25.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:35:25.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:25.201 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:25.201 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:35:25.201 INFO:teuthology.orchestra.run.smithi017.stdout: "num_ops": 0 2023-09-23T15:35:25.201 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:25.210 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:26.072 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T15:35:26.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:26 smithi017 ceph-mon[215646]: pgmap v1456: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:26.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:26 smithi017 ceph-mon[222878]: pgmap v1456: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:26.953 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:27.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:26 smithi027 ceph-mon[195233]: pgmap v1456: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:27.643 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:27 smithi017 conmon[215622]: 2023-09-23T15:35:27.461+0000 7f618a1a9700 -1 mon.a@0(leader).osd e496 definitely_dead 0 2023-09-23T15:35:27.677 INFO:teuthology.orchestra.run.smithi017.stderr:marked in osd.5. 2023-09-23T15:35:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:27 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2913235543' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:27.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:27 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:27 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2913235543' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:27.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:27 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:28.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:27 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2913235543' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:28.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:27 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T15:35:28.434 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:28] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.43.0" 2023-09-23T15:35:28.669 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T15:35:28.669 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T15:35:28.692 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: pgmap v1457: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: pgmap v1457: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:28.693 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: failed to encode map e497 with expected crc 2023-09-23T15:35:28.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T15:35:28.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T15:35:28.694 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: pgmap v1457: 103 pgs: 103 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: failed to encode map e497 with expected crc 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: failed to encode map e497 with expected crc 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: failed to encode map e497 with expected crc 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: osdmap e497: 8 total, 8 up, 8 in 2023-09-23T15:35:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:29.134 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:29.134 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [], 2023-09-23T15:35:29.134 INFO:teuthology.orchestra.run.smithi017.stdout: "complaint_time": 30, 2023-09-23T15:35:29.134 INFO:teuthology.orchestra.run.smithi017.stdout: "num_blocked_ops": 0 2023-09-23T15:35:29.136 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:29.161 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:29.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[222878]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:29.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:29 smithi017 ceph-mon[215646]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: osdmap e498: 8 total, 8 up, 8 in 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:29 smithi027 ceph-mon[195233]: failed to encode map e498 with expected crc 2023-09-23T15:35:30.106 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T15:35:30.912 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:30.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: pgmap v1460: 103 pgs: 5 remapped+peering, 98 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 4.6 KiB/s wr, 0 op/s; 3.3 KiB/s, 0 objects/s recovering 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[215646]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: pgmap v1460: 103 pgs: 5 remapped+peering, 98 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 4.6 KiB/s wr, 0 op/s; 3.3 KiB/s, 0 objects/s recovering 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:30.945 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:30 smithi017 ceph-mon[222878]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: pgmap v1460: 103 pgs: 5 remapped+peering, 98 active+clean; 8.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 4.6 KiB/s wr, 0 op/s; 3.3 KiB/s, 0 objects/s recovering 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: osdmap e499: 8 total, 8 up, 8 in 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:30 smithi027 ceph-mon[195233]: failed to encode map e499 with expected crc 2023-09-23T15:35:31.302 INFO:teuthology.orchestra.run.smithi017.stdout:mon.a: {} 2023-09-23T15:35:31.315 INFO:teuthology.orchestra.run.smithi017.stdout:mon.b: {} 2023-09-23T15:35:31.324 INFO:teuthology.orchestra.run.smithi017.stdout:mon.c: {} 2023-09-23T15:35:31.326 INFO:teuthology.orchestra.run.smithi017.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T15:35:31.326 INFO:teuthology.orchestra.run.smithi017.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T15:35:31.326 INFO:teuthology.orchestra.run.smithi017.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T15:35:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.691 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.692 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3510415102' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:35:31.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: failed to encode map e500 with expected crc 2023-09-23T15:35:31.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:31 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3510415102' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:35:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: osdmap e500: 8 total, 8 up, 8 in 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: failed to encode map e500 with expected crc 2023-09-23T15:35:32.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:31 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3510415102' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T15:35:32.181 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T15:35:32.722 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:32.934 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: pgmap v1463: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.7 KiB/s, 1 objects/s recovering 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: Health check failed: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.935 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[215646]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: pgmap v1463: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.7 KiB/s, 1 objects/s recovering 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: Health check failed: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:32.936 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:32 smithi017 ceph-mon[222878]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: pgmap v1463: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 9.2 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.7 KiB/s, 1 objects/s recovering 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: Health check failed: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded (PG_DEGRADED) 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: osdmap e501: 8 total, 8 up, 8 in 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:32 smithi027 ceph-mon[195233]: failed to encode map e501 with expected crc 2023-09-23T15:35:33.272 INFO:teuthology.orchestra.run.smithi017.stdout:{ 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "size": 20, 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 600, 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "ops": [ 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83106 9.e 9:7e487db8:::benchmark_data_smithi017_543597_object83105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 517.35321266000005, 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1221425209999998, 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:35:33.273 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83106 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.274 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916523+0000", 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916522+0000", 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.275 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916551+0000", 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9121000000000001e-05 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916561+0000", 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.4059999999999994e-06 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916570+0000", 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5149999999999995e-06 2023-09-23T15:35:33.276 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918608+0000", 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0020381800000000001 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918619+0000", 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1222e-05 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.277 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913525+0000", 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490589500000004 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913537+0000", 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1749000000000001e-05 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.278 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.028480+0000", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1149425470000001 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.029082+0000", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00060205899999999995 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038593+0000", 2023-09-23T15:35:33.279 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0095110260000000005 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038666+0000", 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3088000000000002e-05 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83114 9.d 9:bfb40ed3:::benchmark_data_smithi017_543597_object83113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:35:33.280 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 517.353160065, 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024477069, 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83114 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.281 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916576+0000", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916575+0000", 2023-09-23T15:35:33.282 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916600+0000", 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4606000000000001e-05 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916607+0000", 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.0489999999999998e-06 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.283 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916613+0000", 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2219999999999999e-06 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916703+0000", 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9252000000000005e-05 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.284 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916711+0000", 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9039999999999998e-06 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.931237+0000", 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0145257830000001 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.285 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933189+0000", 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001951745 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5", 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.933360+0000", 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017091799999999999 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940451+0000", 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0070909079999999999 2023-09-23T15:35:33.286 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940940+0000", 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00048871099999999996 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.940977+0000", 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.7722999999999998e-05 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.287 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.941053+0000", 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.5729000000000006e-05 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83118 9.e 9:7615419c:::benchmark_data_smithi017_543597_object83117:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 517.35311007799999, 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1299481519999999, 2023-09-23T15:35:33.288 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83118 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.289 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916626+0000", 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916625+0000", 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.290 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916651+0000", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5911e-05 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916659+0000", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2590000000000003e-06 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916665+0000", 2023-09-23T15:35:33.291 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9460000000000003e-06 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918662+0000", 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019976350000000002 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918669+0000", 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2910000000000002e-06 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.292 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913570+0000", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99490066300000002 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.913580+0000", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.7350000000000008e-06 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.030629+0000", 2023-09-23T15:35:33.293 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1170488599999999 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032411+0000", 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001782063 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.032533+0000", 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012223299999999999 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.294 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038742+0000", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006208613 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038809+0000", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6854999999999996e-05 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045301+0000", 2023-09-23T15:35:33.295 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0064921479999999997 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045369+0000", 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.8693000000000001e-05 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046574+0000", 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0012047169999999999 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.296 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83121 9.e 9:71f6c9a6:::benchmark_data_smithi017_543597_object83120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 517.35298033000004, 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1305874610000002, 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.297 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83121 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.298 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916756+0000", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916755+0000", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916784+0000", 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.9522e-05 2023-09-23T15:35:33.299 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916793+0000", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1000000000000004e-06 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.916799+0000", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7530000000000004e-06 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918708+0000", 2023-09-23T15:35:33.300 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001908734 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.918717+0000", 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.6629999999999999e-06 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914427+0000", 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.99571021100000001 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.301 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.914437+0000", 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0592e-05 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034594+0000", 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.12015687 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.302 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034760+0000", 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0001661 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.034983+0000", 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00022256399999999999 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038750+0000", 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0037670210000000002 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.303 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038859+0000", 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00010865 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046631+0000", 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0077723879999999999 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.304 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046672+0000", 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.0759000000000001e-05 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047343+0000", 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00067126700000000002 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.305 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83124 9.3 9:c55b3f52:::benchmark_data_smithi017_543597_object83123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e359)", 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 517.350734805, 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.0418616510000001, 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83124 2023-09-23T15:35:33.306 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919001+0000", 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.307 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919000+0000", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919028+0000", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8895e-05 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919041+0000", 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2996000000000001e-05 2023-09-23T15:35:33.308 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919051+0000", 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.5729999999999997e-06 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919101+0000", 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9864999999999998e-05 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.309 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:55.919108+0000", 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.2760000000000003e-06 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.944052+0000", 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.024943395 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.310 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946310+0000", 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0022579900000000001 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3", 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.946472+0000", 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016254900000000001 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.958448+0000", 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011975497999999999 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.311 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960747+0000", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002299592 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960794+0000", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6409000000000002e-05 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.960863+0000", 2023-09-23T15:35:33.312 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.9227999999999995e-05 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83137 9.e 9:7d999fd1:::benchmark_data_smithi017_543597_object83136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 516.27813863599999, 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.056604109, 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.313 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83137 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.314 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991597+0000", 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991596+0000", 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991621+0000", 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4907000000000001e-05 2023-09-23T15:35:33.315 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991634+0000", 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2556e-05 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991642+0000", 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1340000000000004e-06 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.316 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991689+0000", 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7781000000000002e-05 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:56.991696+0000", 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5239999999999997e-06 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.317 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.038894+0000", 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.047198039 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039062+0000", 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016794100000000001 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.039246+0000", 2023-09-23T15:35:33.318 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018399900000000001 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.045231+0000", 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0059852339999999999 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.046596+0000", 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001365125 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.319 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047421+0000", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00082434100000000003 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.047483+0000", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2686000000000003e-05 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048201+0000", 2023-09-23T15:35:33.320 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00071811100000000001 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83228 9.e 9:7f83ce07:::benchmark_data_smithi017_543597_object83227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 516.11358942100003, 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89863112300000003, 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.321 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83228 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.322 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156147+0000", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156145+0000", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156194+0000", 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8671000000000002e-05 2023-09-23T15:35:33.323 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156210+0000", 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6716999999999999e-05 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156219+0000", 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9900000000000003e-06 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.324 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156278+0000", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8897999999999998e-05 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.156292+0000", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.3502e-05 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042118+0000", 2023-09-23T15:35:33.325 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88582658800000003 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042277+0000", 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015909299999999999 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.042470+0000", 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000193022 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.326 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048224+0000", 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0057530630000000001 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048291+0000", 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.7751999999999998e-05 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.327 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.053961+0000", 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0056697969999999999 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054020+0000", 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9419999999999997e-05 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054778+0000", 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00075720099999999999 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.328 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83232 9.e 9:72608dfb:::benchmark_data_smithi017_543597_object83231:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e360)", 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 516.101727783, 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88775156300000002, 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.329 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83232 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168008+0000", 2023-09-23T15:35:33.330 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168006+0000", 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168035+0000", 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.8824e-05 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.331 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168053+0000", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8185e-05 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168065+0000", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1391e-05 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168131+0000", 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6419000000000002e-05 2023-09-23T15:35:33.332 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:57.168140+0000", 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.7129999999999994e-06 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048345+0000", 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88020536000000005 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.333 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048556+0000", 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021095900000000001 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.048793+0000", 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00023668699999999999 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054856+0000", 2023-09-23T15:35:33.334 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060630129999999999 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054871+0000", 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4851000000000001e-05 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054915+0000", 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.4753999999999999e-05 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.335 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.054975+0000", 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.9562999999999999e-05 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00078479899999999998 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.336 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83259 9.4 9:2f57b43f:::benchmark_data_smithi017_543597_object83258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 515.214030826, 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.144342432, 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83259 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.337 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055705+0000", 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.338 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055704+0000", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055749+0000", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5618000000000003e-05 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055760+0000", 2023-09-23T15:35:33.339 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1012000000000001e-05 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055769+0000", 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5979999999999997e-06 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055829+0000", 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0362000000000002e-05 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.340 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056020+0000", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000190998 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056238+0000", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00021735600000000001 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061804+0000", 2023-09-23T15:35:33.341 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0055659680000000001 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065167+0000", 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0033630359999999998 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197822+0000", 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132655089 2023-09-23T15:35:33.342 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.197887+0000", 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4789999999999995e-05 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200048+0000", 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002161035 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.343 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83260 9.4 9:295d7846:::benchmark_data_smithi017_543597_object83259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 515.21395129500002, 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.146144756, 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83260 2023-09-23T15:35:33.344 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055785+0000", 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.345 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055784+0000", 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055805+0000", 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1111000000000002e-05 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.346 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055812+0000", 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4960000000000001e-06 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.055819+0000", 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.7179999999999996e-06 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.058865+0000", 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030458220000000001 2023-09-23T15:35:33.347 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059023+0000", 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015745099999999999 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.059189+0000", 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000166126 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.348 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065176+0000", 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.005987132 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068875+0000", 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003699698 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200125+0000", 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.131249117999999 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.349 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.200200+0000", 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.4951999999999997e-05 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.201929+0000", 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001729897 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.350 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83261 9.4 9:2dd5bf1f:::benchmark_data_smithi017_543597_object83260:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 515.21312598500003, 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.147188163999999, 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83261 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.351 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056610+0000", 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.352 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056609+0000", 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056631+0000", 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1569999999999998e-05 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056646+0000", 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.4924e-05 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.353 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.056654+0000", 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.0160000000000008e-06 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.061842+0000", 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0051878740000000003 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.354 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062037+0000", 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00019499900000000001 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.062246+0000", 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00020959400000000001 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065285+0000", 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030387909999999999 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.355 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069749+0000", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0044636470000000003 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202002+0000", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.132253035 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.202066+0000", 2023-09-23T15:35:33.356 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4215999999999996e-05 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203798+0000", 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0017322940000000001 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:83265 9.4 9:2add9cf3:::benchmark_data_smithi017_543597_object83264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e361)", 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 515.20418271200003, 2023-09-23T15:35:33.357 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.139692201999999, 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 83265 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:35:33.358 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065553+0000", 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.359 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065572+0000", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8663000000000001e-05 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065576+0000", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7530000000000001e-06 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065580+0000", 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.326e-06 2023-09-23T15:35:33.360 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065625+0000", 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.5077999999999998e-05 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065774+0000", 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00014929099999999999 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,3", 2023-09-23T15:35:33.361 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.065946+0000", 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00017193699999999999 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.068958+0000", 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030115939999999998 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:26:58.069898+0000", 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00093998699999999998 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.362 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203861+0000", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 10.133963208000001 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.203918+0000", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6950999999999997e-05 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:08.205245+0000", 2023-09-23T15:35:33.363 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0013277479999999999 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88039 9.d 9:b338c46d:::benchmark_data_smithi017_543597_object4248:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.85895603500001, 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90200742499999997, 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "reached pg", 2023-09-23T15:35:33.364 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88039 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.365 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410780+0000", 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410779+0000", 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410784+0000", 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5480000000000001e-06 2023-09-23T15:35:33.366 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410796+0000", 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1653e-05 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410807+0000", 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1181e-05 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.367 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410861+0000", 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.3981999999999999e-05 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410880+0000", 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.8553999999999999e-05 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420827+0000", 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0099472469999999993 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.368 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.420840+0000", 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.2969999999999999e-05 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.302886+0000", 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88204606500000005 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.369 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for missing object", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303061+0000", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000175076 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312708+0000", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0096470840000000002 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312787+0000", 2023-09-23T15:35:33.370 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.9296999999999997e-05 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88041 9.d 9:b6e8eb25:::benchmark_data_smithi017_543597_object4250:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.858915018, 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90221110000000004, 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.371 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88041 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410821+0000", 2023-09-23T15:35:33.372 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410820+0000", 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410824+0000", 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.3720000000000001e-06 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.373 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410831+0000", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.1489999999999997e-06 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410837+0000", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.19e-06 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410931+0000", 2023-09-23T15:35:33.374 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.3634000000000005e-05 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410938+0000", 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.3000000000000004e-06 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422325+0000", 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011386744000000001 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.375 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422336+0000", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1028e-05 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303400+0000", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88106453799999995 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303535+0000", 2023-09-23T15:35:33.376 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013442700000000001 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303696+0000", 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00016155799999999999 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309737+0000", 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060407810000000003 2023-09-23T15:35:33.377 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309784+0000", 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.7237000000000002e-05 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312835+0000", 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0030510619999999998 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.378 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.312900+0000", 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4393999999999994e-05 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313032+0000", 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013220300000000001 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.379 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88044 9.d 9:bfc8f2ba:::benchmark_data_smithi017_543597_object4253:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.858894599, 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90561986000000005, 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88044 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.380 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.381 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410841+0000", 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410843+0000", 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.2079999999999999e-06 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.382 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410849+0000", 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6489999999999996e-06 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410854+0000", 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9810000000000003e-06 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410963+0000", 2023-09-23T15:35:33.383 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000109166 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410970+0000", 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.5649999999999998e-06 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422375+0000", 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.01140536 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.384 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422395+0000", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.9913000000000001e-05 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306676+0000", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88428063000000001 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306827+0000", 2023-09-23T15:35:33.385 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00015183300000000001 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.306965+0000", 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000137427 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313079+0000", 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0061140869999999998 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.386 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313118+0000", 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.8633000000000001e-05 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316281+0000", 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031639160000000001 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.387 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316328+0000", 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.6897000000000002e-05 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316461+0000", 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013290900000000001 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.388 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88045 9.d 9:b617d99b:::benchmark_data_smithi017_543597_object4254:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.85887769700003, 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.91186458299999995, 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88045 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.389 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410858+0000", 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.390 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410860+0000", 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5600000000000001e-06 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410866+0000", 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.6860000000000003e-06 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.391 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410871+0000", 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.9980000000000003e-06 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410993+0000", 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000122359 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411000+0000", 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.0950000000000001e-06 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.392 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422421+0000", 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.011421093 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.422430+0000", 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.1200000000000008e-06 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.393 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309805+0000", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88737535099999998 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.309933+0000", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012809900000000001 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 5,7", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310023+0000", 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.9247999999999997e-05 2023-09-23T15:35:33.394 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313160+0000", 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0031370759999999999 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.316471+0000", 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.003311278 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.395 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322477+0000", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0060065279999999997 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322536+0000", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.8792000000000002e-05 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.322723+0000", 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018669199999999999 2023-09-23T15:35:33.396 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88046 9.3 9:c0e328f8:::benchmark_data_smithi017_543597_object4255:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.85886091700002, 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89982923699999995, 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.397 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88046 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.398 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410875+0000", 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410877+0000", 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.4669999999999998e-06 2023-09-23T15:35:33.399 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410885+0000", 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 7.841e-06 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410890+0000", 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0069999999999997e-06 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.400 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411148+0000", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00025797 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op must wait for map", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411157+0000", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.5380000000000002e-06 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413770+0000", 2023-09-23T15:35:33.401 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026130089999999999 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413782+0000", 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.289e-05 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303621+0000", 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.88983906499999998 2023-09-23T15:35:33.402 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303734+0000", 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011293099999999999 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.303859+0000", 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012465600000000001 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.403 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310319+0000", 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.006459752 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310340+0000", 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.1526999999999999e-05 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310470+0000", 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00012976200000000001 2023-09-23T15:35:33.404 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310519+0000", 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4.8906999999999998e-05 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.310704+0000", 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018528199999999999 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.405 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46157.0:88047 9.3 9:c39718c3:::benchmark_data_smithi017_543597_object4256:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 471.85883341700003, 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.90486523799999996, 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46157", 2023-09-23T15:35:33.406 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 88047 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410903+0000", 2023-09-23T15:35:33.407 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410902+0000", 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410905+0000", 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6520000000000002e-06 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.408 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410911+0000", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.4119999999999998e-06 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.410917+0000", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.5759999999999996e-06 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411181+0000", 2023-09-23T15:35:33.409 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00026399799999999999 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting_for_map not empty", 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.411187+0000", 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.6200000000000001e-06 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413808+0000", 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0026202399999999998 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.410 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for peered", 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:41.413816+0000", 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.2530000000000004e-06 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307084+0000", 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.89326843099999997 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.411 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307210+0000", 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000125623 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.307340+0000", 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013041499999999999 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.311327+0000", 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0039864119999999999 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.412 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.313609+0000", 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.002282315 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315515+0000", 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0019060640000000001 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.413 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315579+0000", 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3367999999999994e-05 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:27:42.315768+0000", 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00018929 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.414 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18484 10.2 10:46869aa5:::benchmark_data_smithi017_604517_object18483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 329.199502908, 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40252255100000001, 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18484 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.415 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070233+0000", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070231+0000", 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:33.416 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070257+0000", 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.6016e-05 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070274+0000", 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.6682999999999999e-05 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.417 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070283+0000", 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0580000000000004e-06 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070333+0000", 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.0442999999999998e-05 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070465+0000", 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00013140100000000001 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.418 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070591+0000", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.000126329 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072125+0000", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.001533855 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077175+0000", 2023-09-23T15:35:33.419 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0050495369999999998 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472642+0000", 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39546752299999999 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472704+0000", 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.2088999999999995e-05 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.420 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.472756+0000", 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 5.1493999999999999e-05 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "description": "osd_op(client.46520.0:18486 10.2 10:43afa650:::benchmark_data_smithi017_604517_object18485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "initiated_at": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "age": 329.199076021, 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.40280343299999999, 2023-09-23T15:35:33.421 INFO:teuthology.orchestra.run.smithi017.stdout: "type_data": { 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client_info": { 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client": "client.46520", 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "tid": 18486 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "events": [ 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "initiated", 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.422 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "throttled", 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070660+0000", 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "header_read", 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070659+0000", 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 4294967295.999999 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "all_read", 2023-09-23T15:35:33.423 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070685+0000", 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 2.5811000000000001e-05 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "dispatched", 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070697+0000", 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 1.1596999999999999e-05 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "queued_for_pg", 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.070705+0000", 2023-09-23T15:35:33.424 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 8.1359999999999997e-06 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "reached_pg", 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071435+0000", 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00072993699999999995 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "started", 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071545+0000", 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00011006999999999999 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.425 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.071635+0000", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 9.0004000000000001e-05 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "op_commit", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.072295+0000", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.00066030600000000004 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.077270+0000", 2023-09-23T15:35:33.426 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.0049749349999999998 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473366+0000", 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 0.39609567099999998 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "commit_sent", 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473429+0000", 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 6.3528999999999994e-05 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: }, 2023-09-23T15:35:33.427 INFO:teuthology.orchestra.run.smithi017.stdout: { 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: "event": "done", 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: "time": "2023-09-23T15:30:04.473463+0000", 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: "duration": 3.4245999999999997e-05 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: } 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout: ] 2023-09-23T15:35:33.428 INFO:teuthology.orchestra.run.smithi017.stdout:} 2023-09-23T15:35:34.470 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T15:35:34.843 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:34 smithi027 ceph-mon[195233]: pgmap v1465: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.1 KiB/s, 1 objects/s recovering 2023-09-23T15:35:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:34 smithi027 ceph-mon[195233]: 1.1b deep-scrub starts 2023-09-23T15:35:34.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:34 smithi027 ceph-mon[195233]: 1.1b deep-scrub ok 2023-09-23T15:35:34.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[215646]: pgmap v1465: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.1 KiB/s, 1 objects/s recovering 2023-09-23T15:35:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[215646]: 1.1b deep-scrub starts 2023-09-23T15:35:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[215646]: 1.1b deep-scrub ok 2023-09-23T15:35:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[222878]: pgmap v1465: 103 pgs: 1 peering, 1 activating+undersized+degraded+remapped, 2 activating+undersized+remapped, 1 active+undersized+remapped, 8 remapped+peering, 90 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 8.4 KiB/s wr, 0 op/s; 20/87 objects degraded (22.989%); 6.1 KiB/s, 1 objects/s recovering 2023-09-23T15:35:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[222878]: 1.1b deep-scrub starts 2023-09-23T15:35:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:34 smithi017 ceph-mon[222878]: 1.1b deep-scrub ok 2023-09-23T15:35:35.706 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T15:35:35.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:35 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:36.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:35 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:36.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:35 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:36.482 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T15:35:36.521 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:36.522 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:36.522 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:35:36.522 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:36.858 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:36.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: pgmap v1466: 103 pgs: 1 peering, 102 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded) 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: Cluster is now healthy 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: failed to encode map e502 with expected crc 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: failed to encode map e502 with expected crc 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[215646]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T15:35:36.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: pgmap v1466: 103 pgs: 1 peering, 102 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded) 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: Cluster is now healthy 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: failed to encode map e502 with expected crc 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: failed to encode map e502 with expected crc 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:36.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:36 smithi017 ceph-mon[222878]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T15:35:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: pgmap v1466: 103 pgs: 1 peering, 102 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T15:35:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T15:35:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 20/87 objects degraded (22.989%), 1 pg degraded) 2023-09-23T15:35:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: Cluster is now healthy 2023-09-23T15:35:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: failed to encode map e502 with expected crc 2023-09-23T15:35:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: failed to encode map e502 with expected crc 2023-09-23T15:35:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/248350594' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:37.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:36 smithi027 ceph-mon[195233]: osdmap e502: 8 total, 8 up, 8 in 2023-09-23T15:35:37.170 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T15:35:37.527 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:38.220 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:38] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.43.0" 2023-09-23T15:35:38.704 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T15:35:38.816 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:38 smithi027 ceph-mon[195233]: pgmap v1468: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 60 KiB/s, 6 objects/s recovering 2023-09-23T15:35:38.816 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:38 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3524357114' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:38.816 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:38 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:39.037 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:39.037 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:39.037 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:35:39.037 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:35:39.037 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:39.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[215646]: pgmap v1468: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 60 KiB/s, 6 objects/s recovering 2023-09-23T15:35:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3524357114' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:39.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[222878]: pgmap v1468: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 60 KiB/s, 6 objects/s recovering 2023-09-23T15:35:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3524357114' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:39.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:38 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T15:35:39.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T15:35:39.653 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T15:35:39.747 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:39.973 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[215646]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.975 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:39.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T15:35:39.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: failed to encode map e503 with expected crc 2023-09-23T15:35:39.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:39 smithi017 ceph-mon[222878]: failed to encode map e503 with expected crc 2023-09-23T15:35:40.014 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: failed to encode map e503 with expected crc 2023-09-23T15:35:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: failed to encode map e503 with expected crc 2023-09-23T15:35:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T15:35:40.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: osdmap e503: 8 total, 8 up, 8 in 2023-09-23T15:35:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: failed to encode map e503 with expected crc 2023-09-23T15:35:40.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:39 smithi027 ceph-mon[195233]: failed to encode map e503 with expected crc 2023-09-23T15:35:40.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:40 smithi027 ceph-mon[195233]: pgmap v1470: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s, 5 objects/s recovering 2023-09-23T15:35:40.985 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:40 smithi017 ceph-mon[215646]: pgmap v1470: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s, 5 objects/s recovering 2023-09-23T15:35:40.985 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:40 smithi017 ceph-mon[222878]: pgmap v1470: 103 pgs: 103 active+clean; 8.5 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 52 KiB/s, 5 objects/s recovering 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.489 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:83113 9.1 9:84c8439c:::benchmark_data_smithi017_543597_object83112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e358)", 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 525.59338011700004, 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.067041744, 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 83113 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.490 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892815+0000", 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.491 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892814+0000", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892915+0000", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000101517 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892934+0000", 2023-09-23T15:35:41.492 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8862999999999999e-05 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892943+0000", 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.6300000000000004e-06 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.892994+0000", 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1109000000000001e-05 2023-09-23T15:35:41.493 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893124+0000", 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013067299999999999 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 3,7", 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.893274+0000", 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000149534 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.494 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.896402+0000", 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0031281579999999998 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906247+0000", 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0098445379999999999 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.495 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:55.906253+0000", 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7000000000000002e-06 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.947718+0000", 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0414645709999999 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.496 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for degraded object", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.948393+0000", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00067537000000000001 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959773+0000", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01137928 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:26:56.959857+0000", 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4637999999999998e-05 2023-09-23T15:35:41.497 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18474 10.d 10:b2d26cad:::benchmark_data_smithi017_604517_object18473:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 337.41773198800001, 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40456172899999998, 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.498 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18474 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.499 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068464+0000", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068463+0000", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068810+0000", 2023-09-23T15:35:41.500 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034735199999999999 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068821+0000", 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0264e-05 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068827+0000", 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.443e-06 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.501 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069014+0000", 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018699400000000001 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069120+0000", 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010654000000000001 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.502 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.069251+0000", 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013087999999999999 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072104+0000", 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0028525260000000002 2023-09-23T15:35:41.503 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.504 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.504 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.504 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077080+0000", 2023-09-23T15:35:41.504 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0049762950000000004 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472922+0000", 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39584164300000002 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472980+0000", 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8023000000000003e-05 2023-09-23T15:35:41.505 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.473025+0000", 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5479000000000003e-05 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18476 10.d 10:b8c42a96:::benchmark_data_smithi017_604517_object18475:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:35:41.506 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 337.41735541600002, 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.40577544799999998, 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18476 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.507 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068840+0000", 2023-09-23T15:35:41.508 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068872+0000", 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2694000000000002e-05 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068881+0000", 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.371e-06 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.509 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068885+0000", 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3229999999999999e-06 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070067+0000", 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0011816400000000001 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.510 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070195+0000", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012812699999999999 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,4", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.070349+0000", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015427600000000001 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072111+0000", 2023-09-23T15:35:41.511 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0017617259999999999 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077110+0000", 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004998794 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474512+0000", 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39740209999999998 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.512 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474569+0000", 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7841999999999999e-05 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.474616+0000", 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6134000000000002e-05 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.513 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:18477 10.a 10:55f810cd:::benchmark_data_smithi017_604517_object18476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 337.41730572199998, 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.403843171, 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.514 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 18477 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068890+0000", 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.515 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068889+0000", 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068920+0000", 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0133000000000001e-05 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.516 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068927+0000", 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5780000000000001e-06 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.068932+0000", 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7840000000000003e-06 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072819+0000", 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.003887129 2023-09-23T15:35:41.517 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.072970+0000", 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015059700000000001 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 0,5", 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.073147+0000", 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017778 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.518 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.075739+0000", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0025920600000000002 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.077221+0000", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0014813630000000001 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472627+0000", 2023-09-23T15:35:41.519 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.39540625000000001 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472672+0000", 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5337999999999999e-05 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:30:04.472733+0000", 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0619999999999999e-05 2023-09-23T15:35:41.520 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195331 10.7 10:e31d95f4:::benchmark_data_smithi017_604517_object89220:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.7484671, 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1400244129999999, 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.521 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195331 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.522 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737728+0000", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737727+0000", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737734+0000", 2023-09-23T15:35:41.523 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9539999999999998e-06 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737745+0000", 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.132e-05 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737750+0000", 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3329999999999998e-06 2023-09-23T15:35:41.524 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737809+0000", 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9676000000000003e-05 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737892+0000", 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2812999999999995e-05 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:35:41.525 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.737966+0000", 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4040000000000003e-05 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.742332+0000", 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043653770000000001 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.750884+0000", 2023-09-23T15:35:41.526 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0085524480000000007 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758674+0000", 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0077902680000000004 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758686+0000", 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1745e-05 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.527 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759654+0000", 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0009683540000001 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759672+0000", 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7527e-05 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.877695+0000", 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.118023293 2023-09-23T15:35:41.528 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.877753+0000", 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7645999999999997e-05 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.529 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195336 10.7 10:e0c5d133:::benchmark_data_smithi017_604517_object89225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e459)", 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.742748718, 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.1346623220000001, 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195336 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.530 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743447+0000", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743446+0000", 2023-09-23T15:35:41.531 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743451+0000", 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8010000000000003e-06 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743468+0000", 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6775000000000001e-05 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.532 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743473+0000", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743503+0000", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0386e-05 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743624+0000", 2023-09-23T15:35:41.533 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120671 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3", 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.743739+0000", 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000115364 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.750339+0000", 2023-09-23T15:35:41.534 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066004480000000001 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.753689+0000", 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033490400000000002 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758831+0000", 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0051428940000000003 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.535 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.758840+0000", 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2239999999999995e-06 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759707+0000", 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0008677020000001 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.536 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759712+0000", 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9830000000000004e-06 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878063+0000", 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1183501629999999 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878109+0000", 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6693999999999997e-05 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.537 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195402 10.7 10:e8a13b56:::benchmark_data_smithi017_604517_object89291:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.65467551399999, 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0597443800000002, 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.538 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195402 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831520+0000", 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.539 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831518+0000", 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831527+0000", 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.2549999999999997e-06 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.540 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831544+0000", 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7051e-05 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831552+0000", 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0600000000000008e-06 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.541 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831615+0000", 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3020000000000003e-05 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.831626+0000", 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1319e-05 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759726+0000", 2023-09-23T15:35:41.542 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92809969400000003 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759733+0000", 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7889999999999997e-06 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878161+0000", 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1184288579999999 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.543 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878312+0000", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000150459 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.878488+0000", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017655899999999999 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884902+0000", 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0064132140000000004 2023-09-23T15:35:41.544 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884913+0000", 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1184e-05 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885002+0000", 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9128000000000005e-05 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.545 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891051+0000", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060486730000000001 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891118+0000", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7461999999999999e-05 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891264+0000", 2023-09-23T15:35:41.546 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014637299999999999 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195408 10.7 10:e55c1ca3:::benchmark_data_smithi017_604517_object89297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.652648062, 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0579925349999999, 2023-09-23T15:35:41.547 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195408 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.548 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833548+0000", 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833547+0000", 2023-09-23T15:35:41.549 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833549+0000", 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9970000000000001e-06 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833554+0000", 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1050000000000003e-06 2023-09-23T15:35:41.550 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833558+0000", 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4790000000000001e-06 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833601+0000", 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3269999999999997e-05 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.551 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.833610+0000", 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.2490000000000007e-06 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759748+0000", 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92613742399999999 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.552 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759755+0000", 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.148e-06 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881083+0000", 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1213284610000001 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881184+0000", 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010103300000000001 2023-09-23T15:35:41.553 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.881595+0000", 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00041013399999999999 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.884953+0000", 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033583160000000001 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.554 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888041+0000", 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0030881319999999999 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.890997+0000", 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029561489999999999 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.555 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891415+0000", 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000418102 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891454+0000", 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8875999999999998e-05 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.556 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891540+0000", 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5929999999999999e-05 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195418 10.7 10:eae79fec:::benchmark_data_smithi017_604517_object89307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:35:41.557 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.64065839, 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0517118569999999, 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195418 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.558 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845537+0000", 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.559 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845540+0000", 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3699999999999999e-06 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845547+0000", 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3669999999999999e-06 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.560 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845552+0000", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7520000000000004e-06 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845607+0000", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.524e-05 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.845614+0000", 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3149999999999997e-06 2023-09-23T15:35:41.561 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759809+0000", 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91419547400000001 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759818+0000", 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.5359999999999992e-06 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.562 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885028+0000", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1252105480000001 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885118+0000", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9884999999999997e-05 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.885252+0000", 2023-09-23T15:35:41.563 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013337099999999999 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891282+0000", 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0060303270000000003 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891357+0000", 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5458999999999997e-05 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.564 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891396+0000", 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9125000000000002e-05 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897023+0000", 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0056263069999999997 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.565 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897087+0000", 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4036999999999997e-05 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897249+0000", 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00016222400000000001 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.566 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195467 10.7 10:eb064bcf:::benchmark_data_smithi017_604517_object89356:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.59116062699999, 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0025557250000001, 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195467 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.567 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895035+0000", 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.568 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895034+0000", 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895039+0000", 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6689999999999997e-06 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.569 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895048+0000", 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8470000000000002e-06 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895053+0000", 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3600000000000004e-06 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895104+0000", 2023-09-23T15:35:41.570 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1248000000000001e-05 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.895116+0000", 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1756000000000001e-05 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759849+0000", 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86473266500000001 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.571 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759854+0000", 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1200000000000001e-06 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888086+0000", 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1282324240000001 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.572 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888206+0000", 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000120252 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.888298+0000", 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1280999999999996e-05 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891292+0000", 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.002994118 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.573 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891563+0000", 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00027159800000000001 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891584+0000", 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0157e-05 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.574 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897377+0000", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0057933710000000003 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897440+0000", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3437999999999998e-05 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897591+0000", 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015026699999999999 2023-09-23T15:35:41.575 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195469 10.7 10:e6a6a234:::benchmark_data_smithi017_604517_object89358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.58872225299999, 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0175167780000001, 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.576 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195469 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897473+0000", 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.577 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897472+0000", 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897478+0000", 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2070000000000003e-06 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.578 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897493+0000", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4875000000000001e-05 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897499+0000", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4740000000000002e-06 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897555+0000", 2023-09-23T15:35:41.579 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5829999999999999e-05 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897562+0000", 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8290000000000002e-06 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759879+0000", 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86231730600000001 2023-09-23T15:35:41.580 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759883+0000", 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4379999999999997e-06 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891597+0000", 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.131713433 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.581 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891688+0000", 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1167000000000003e-05 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.891777+0000", 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8654999999999997e-05 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.582 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.896977+0000", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0052007900000000003 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897336+0000", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035870999999999999 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907873+0000", 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010536581 2023-09-23T15:35:41.583 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914774+0000", 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0069013700000000004 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914840+0000", 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5710999999999995e-05 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.584 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.914990+0000", 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015026699999999999 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195470 10.7 10:e10e854e:::benchmark_data_smithi017_604517_object89359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.58868458399999, 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.0178075889999998, 2023-09-23T15:35:41.585 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195470 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:35:41.586 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897511+0000", 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.587 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897513+0000", 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6280000000000001e-06 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897523+0000", 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3109999999999995e-06 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.588 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897527+0000", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3950000000000003e-06 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897595+0000", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.8285000000000001e-05 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897605+0000", 2023-09-23T15:35:41.589 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3160000000000003e-06 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759906+0000", 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86230155399999997 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759910+0000", 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.177e-06 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.590 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894172+0000", 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1342621820000001 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894290+0000", 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000117631 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.591 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.894379+0000", 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9316999999999999e-05 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897322+0000", 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0029428710000000001 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.900796+0000", 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0034740050000000001 2023-09-23T15:35:41.592 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915068+0000", 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.014271855999999999 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915113+0000", 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4650000000000001e-05 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.593 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915181+0000", 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7730000000000004e-05 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915319+0000", 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013808700000000001 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.594 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195472 10.7 10:e3ee2323:::benchmark_data_smithi017_604517_object89361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.58862041500001, 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.01809517, 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195472 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.595 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.596 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897575+0000", 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897578+0000", 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.9160000000000001e-06 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.597 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897586+0000", 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0099999999999995e-06 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897590+0000", 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7450000000000003e-06 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.598 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897626+0000", 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5605000000000002e-05 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.897630+0000", 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3059999999999999e-06 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759930+0000", 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.86229939200000005 2023-09-23T15:35:41.599 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759934+0000", 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1810000000000003e-06 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897641+0000", 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1377075400000001 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.600 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897743+0000", 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102106 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.897827+0000", 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3660999999999997e-05 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.601 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907809+0000", 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0099816060000000005 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915336+0000", 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075277720000000003 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.602 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915439+0000", 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000102908 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915482+0000", 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2416999999999997e-05 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915538+0000", 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.6577999999999999e-05 2023-09-23T15:35:41.603 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915670+0000", 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013201599999999999 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195497 10.7 10:ec423fb3:::benchmark_data_smithi017_604517_object89386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.55711134500001, 2023-09-23T15:35:41.604 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9868509519999999, 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195497 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.605 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929084+0000", 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929083+0000", 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.606 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929089+0000", 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3180000000000003e-06 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929102+0000", 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3128e-05 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.607 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929109+0000", 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3690000000000003e-06 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929174+0000", 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4955000000000004e-05 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:29.929187+0000", 2023-09-23T15:35:41.608 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3665e-05 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759962+0000", 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.83077460400000003 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759970+0000", 2023-09-23T15:35:41.609 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.8480000000000001e-06 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.900848+0000", 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.140877777 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.901004+0000", 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000156088 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.610 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.901181+0000", 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000177646 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907842+0000", 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0066605249999999996 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.611 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915348+0000", 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0075065030000000003 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915697+0000", 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00034851400000000002 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.612 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915793+0000", 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6348000000000007e-05 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915847+0000", 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3866999999999998e-05 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915935+0000", 2023-09-23T15:35:41.613 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8084999999999994e-05 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195557 10.7 10:ebbb3f94:::benchmark_data_smithi017_604517_object89446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.47674618100001, 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.9068306000000002, 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.614 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195557 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.615 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009449+0000", 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009454+0000", 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9219999999999996e-06 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.616 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009465+0000", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1234e-05 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.009471+0000", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9429999999999997e-06 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.011481+0000", 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0020101619999999998 2023-09-23T15:35:41.617 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.011487+0000", 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6270000000000002e-06 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759994+0000", 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74850652900000003 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.618 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.759998+0000", 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4290000000000003e-06 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904399+0000", 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1444002740000001 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904540+0000", 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014094500000000001 2023-09-23T15:35:41.619 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.904696+0000", 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015625900000000001 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907907+0000", 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032108760000000001 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.620 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915360+0000", 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0074533530000000002 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915989+0000", 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00062900200000000003 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916072+0000", 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.3102999999999994e-05 2023-09-23T15:35:41.621 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916127+0000", 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5126e-05 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916280+0000", 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015262300000000001 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.622 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195584 10.7 10:eb9b89c9:::benchmark_data_smithi017_604517_object89473:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.42822406499999, 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8769677090000001, 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195584 2023-09-23T15:35:41.623 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057972+0000", 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.624 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057969+0000", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.057981+0000", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.146e-05 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058014+0000", 2023-09-23T15:35:41.625 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3043000000000002e-05 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058027+0000", 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3183000000000001e-05 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058163+0000", 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013621299999999999 2023-09-23T15:35:41.626 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.058190+0000", 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6449e-05 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760021+0000", 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.70183132199999998 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.627 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760026+0000", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6330000000000004e-06 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.907929+0000", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1479031609999999 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.908041+0000", 2023-09-23T15:35:41.628 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011233 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.908178+0000", 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013667 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915372+0000", 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071940249999999997 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.629 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915745+0000", 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00037352199999999999 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916331+0000", 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00058567700000000005 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.630 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934743+0000", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018411927000000002 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934801+0000", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7664999999999998e-05 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934939+0000", 2023-09-23T15:35:41.631 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000138589 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195610 10.7 10:efb219c1:::benchmark_data_smithi017_604517_object89499:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.37683008799999, 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8259069960000001, 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.632 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195610 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.633 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109366+0000", 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109364+0000", 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109371+0000", 2023-09-23T15:35:41.634 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2230000000000001e-06 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109389+0000", 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7427000000000001e-05 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109395+0000", 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8730000000000004e-06 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.635 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109446+0000", 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0974999999999999e-05 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.109456+0000", 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0292000000000001e-05 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760047+0000", 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.65059110499999995 2023-09-23T15:35:41.636 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760051+0000", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0589999999999996e-06 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911246+0000", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1511953669999999 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911420+0000", 2023-09-23T15:35:41.637 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017322099999999999 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.911571+0000", 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00015136699999999999 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.915383+0000", 2023-09-23T15:35:41.638 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0038122500000000001 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916028+0000", 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00064446299999999996 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935004+0000", 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.01897654 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.639 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935042+0000", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8118999999999999e-05 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935087+0000", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4762999999999998e-05 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935273+0000", 2023-09-23T15:35:41.640 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00018532100000000001 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195653 10.7 10:e4163779:::benchmark_data_smithi017_604517_object89542:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.294375551, 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7437647190000001, 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.641 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195653 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.642 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191820+0000", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191818+0000", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191827+0000", 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7690000000000002e-06 2023-09-23T15:35:41.643 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191846+0000", 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8978999999999999e-05 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191856+0000", 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7820000000000005e-06 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.644 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191919+0000", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.2704000000000001e-05 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.191929+0000", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.081e-05 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760070+0000", 2023-09-23T15:35:41.645 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.56814065000000002 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760074+0000", 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0910000000000003e-06 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916357+0000", 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1562823070000001 2023-09-23T15:35:41.646 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916461+0000", 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000104323 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.916550+0000", 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9177000000000004e-05 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.647 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.922092+0000", 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0055422589999999999 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934700+0000", 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012608199 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935356+0000", 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00065563000000000004 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.648 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935390+0000", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3699999999999999e-05 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935443+0000", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3597000000000002e-05 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935585+0000", 2023-09-23T15:35:41.649 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141349 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195677 10.7 10:e92f236a:::benchmark_data_smithi017_604517_object89566:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.246786714, 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.6964536799999999, 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.650 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195677 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.651 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239409+0000", 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239408+0000", 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.652 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239413+0000", 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7550000000000002e-06 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239423+0000", 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8660000000000001e-06 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.653 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239427+0000", 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.262e-06 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239483+0000", 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5686000000000002e-05 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.654 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.239492+0000", 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.3719999999999994e-06 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760096+0000", 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.52060388099999999 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760100+0000", 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0749999999999999e-06 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.655 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919203+0000", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1591034630000001 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919314+0000", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000110086 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.919411+0000", 2023-09-23T15:35:41.656 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7120999999999994e-05 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.922156+0000", 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0027455940000000001 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934951+0000", 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.012794244999999999 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.657 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935636+0000", 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00068527400000000004 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935675+0000", 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9638000000000002e-05 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.658 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935724+0000", 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8804999999999998e-05 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935863+0000", 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013832199999999999 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.659 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46520.0:195695 10.7 10:e3e7c11c:::benchmark_data_smithi017_604517_object89584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e460)", 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 131.113193623, 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5632770379999998, 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46520", 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/798647905", 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 195695 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.660 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373002+0000", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373001+0000", 2023-09-23T15:35:41.661 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.999999 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373006+0000", 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6999999999999999e-06 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373016+0000", 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9699999999999994e-06 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.662 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373020+0000", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1790000000000001e-06 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373064+0000", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3591000000000003e-05 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.373073+0000", 2023-09-23T15:35:41.663 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1689999999999998e-06 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760112+0000", 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.38703970300000001 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:30.760118+0000", 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2089999999999996e-06 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.664 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923633+0000", 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1635158809999999 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923768+0000", 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013468099999999999 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 2,3,4", 2023-09-23T15:35:41.665 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.923910+0000", 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000141428 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.934966+0000", 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.011056015000000001 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.935321+0000", 2023-09-23T15:35:41.666 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00035499899999999999 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936057+0000", 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00073604400000000004 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936096+0000", 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8983999999999998e-05 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.667 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936147+0000", 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.1069000000000002e-05 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:33:31.936279+0000", 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013231199999999999 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:41.668 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:41.669 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:41.717 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T15:35:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:41 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/420935912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:41.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:41 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:42.127 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T15:35:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:41 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/420935912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:42.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:41 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:42.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:41 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/420935912' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:42.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:41 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:42.400 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T15:35:42.488 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:42.768 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:42.999 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[215646]: pgmap v1471: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s, 5 objects/s recovering 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[215646]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[215646]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[215646]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[222878]: pgmap v1471: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s, 5 objects/s recovering 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[222878]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[222878]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:43.000 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:42 smithi017 ceph-mon[222878]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T15:35:43.070 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:42 smithi027 ceph-mon[195233]: pgmap v1471: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 45 KiB/s, 5 objects/s recovering 2023-09-23T15:35:43.070 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:42 smithi027 ceph-mon[195233]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.070 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:42 smithi027 ceph-mon[195233]: failed to encode map e504 with expected crc 2023-09-23T15:35:43.070 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:42 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:43.070 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:42 smithi027 ceph-mon[195233]: osdmap e504: 8 total, 8 up, 8 in 2023-09-23T15:35:43.747 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:43.971 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:43.972 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:43.972 INFO:teuthology.orchestra.run.smithi027.stdout: "num_ops": 0 2023-09-23T15:35:43.972 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:44.094 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:44.095 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:44.518 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T15:35:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:44 smithi027 ceph-mon[195233]: pgmap v1473: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:44.849 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:44 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:44 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2658422125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:44.850 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:44 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:44.880 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:45.020 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T15:35:45.042 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[215646]: pgmap v1473: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2658422125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[222878]: pgmap v1473: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2658422125' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:45.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:44 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T15:35:45.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T15:35:46.035 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: pgmap v1475: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: 1.18 scrub starts 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:46 smithi027 ceph-mon[195233]: 1.18 scrub ok 2023-09-23T15:35:46.362 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:46.362 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [], 2023-09-23T15:35:46.362 INFO:teuthology.orchestra.run.smithi027.stdout: "complaint_time": 30, 2023-09-23T15:35:46.362 INFO:teuthology.orchestra.run.smithi027.stdout: "num_blocked_ops": 0 2023-09-23T15:35:46.362 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:46.398 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: pgmap v1475: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: 1.18 scrub starts 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[215646]: 1.18 scrub ok 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.399 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T15:35:46.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: osdmap e505: 8 total, 8 up, 8 in 2023-09-23T15:35:46.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: pgmap v1475: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:46.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: 1.18 scrub starts 2023-09-23T15:35:46.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: failed to encode map e505 with expected crc 2023-09-23T15:35:46.400 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:46 smithi017 ceph-mon[222878]: 1.18 scrub ok 2023-09-23T15:35:46.952 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T15:35:47.308 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:47 smithi017 ceph-mon[215646]: failed to encode map e505 with expected crc 2023-09-23T15:35:47.308 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:47 smithi017 ceph-mon[222878]: failed to encode map e505 with expected crc 2023-09-23T15:35:47.311 INFO:teuthology.orchestra.run.smithi027.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T15:35:47.349 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:47 smithi027 ceph-mon[195233]: failed to encode map e505 with expected crc 2023-09-23T15:35:48.443 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:48] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.43.0" 2023-09-23T15:35:48.670 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T15:35:48.746 INFO:teuthology.orchestra.run.smithi027.stdout:{ 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "size": 20, 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 600, 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "ops": [ 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88039 9.d 9.b6231ccd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.334408935, 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515619899999996, 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.747 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88039 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.748 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409931+0000", 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409929+0000", 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409936+0000", 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9029999999999998e-06 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.749 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409948+0000", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1756999999999999e-05 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409957+0000", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1209999999999996e-06 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:35:48.750 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3381999999999998e-05 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410021+0000", 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.101e-05 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405075+0000", 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505363800000002 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.751 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405087+0000", 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1863e-05 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88041 9.d 9.a4d7176d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.33436833600001, 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99514826899999997, 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.752 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88041 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.753 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409971+0000", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409973+0000", 2023-09-23T15:35:48.754 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.6620000000000001e-06 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409980+0000", 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7689999999999999e-06 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.409985+0000", 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6720000000000003e-06 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.755 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410051+0000", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5931000000000003e-05 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410058+0000", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9380000000000003e-06 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405110+0000", 2023-09-23T15:35:48.756 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99505194299999999 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405119+0000", 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7710000000000001e-06 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88044 9.d 9.5d4f13fd (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.33433182900001, 2023-09-23T15:35:48.757 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515036099999998, 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88044 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.758 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410008+0000", 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410007+0000", 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.759 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410010+0000", 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.5220000000000002e-06 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410018+0000", 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6599999999999995e-06 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410023+0000", 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3789999999999997e-06 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.760 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410129+0000", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010582300000000001 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410141+0000", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2234e-05 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405148+0000", 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.995007164 2023-09-23T15:35:48.761 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405158+0000", 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9410000000000002e-06 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88045 9.d 9.d99be86d (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.33430880999998, 2023-09-23T15:35:48.762 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99515891099999998, 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88045 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.763 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410031+0000", 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410030+0000", 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:48.765 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410033+0000", 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.463e-06 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410039+0000", 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0480000000000004e-06 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.766 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410043+0000", 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4689999999999999e-06 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410178+0000", 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000134657 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410185+0000", 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7519999999999999e-06 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.767 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405180+0000", 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.994995097 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.405190+0000", 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7569999999999999e-06 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.768 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88046 9.3 9.1f14c703 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.33429140800001, 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99103617399999999, 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88046 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.769 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410048+0000", 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.770 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410050+0000", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8989999999999999e-06 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410055+0000", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4700000000000001e-06 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410061+0000", 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7860000000000002e-06 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.771 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410100+0000", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8804999999999999e-05 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410109+0000", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.8459999999999997e-06 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401069+0000", 2023-09-23T15:35:48.772 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99096060100000005 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.401084+0000", 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.5098e-05 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:88047 9.3 9.c318e9c3 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e374)", 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:35:48.773 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 488.33426337399999, 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99202290000000004, 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "reached pg", 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 88047 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:35:48.774 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410076+0000", 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999995 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.775 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410078+0000", 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.4839999999999998e-06 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410085+0000", 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.7649999999999996e-06 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410090+0000", 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.899e-06 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.776 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410149+0000", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.9074999999999999e-05 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:40.410159+0000", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0394000000000001e-05 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402086+0000", 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.99192703699999996 2023-09-23T15:35:48.777 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:27:41.402099+0000", 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2700999999999999e-05 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118583 9.d 9:b22e1d7c:::benchmark_data_smithi017_543597_object34792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.38007186599998, 2023-09-23T15:35:48.778 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.944761042, 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118583 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.779 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364268+0000", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364266+0000", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364271+0000", 2023-09-23T15:35:48.780 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0320000000000003e-06 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364281+0000", 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8560000000000002e-06 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364286+0000", 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3299999999999998e-06 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.781 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364340+0000", 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3464999999999997e-05 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.364349+0000", 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.1099999999999992e-06 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298613+0000", 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93426368199999998 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.782 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298716+0000", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010302999999999999 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.298806+0000", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.056e-05 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305553+0000", 2023-09-23T15:35:48.783 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0067469649999999997 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305618+0000", 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5029000000000004e-05 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308931+0000", 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033124449999999998 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.784 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308986+0000", 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4809999999999999e-05 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309029+0000", 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.3118000000000003e-05 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.785 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118587 9.d 9:b46673d5:::benchmark_data_smithi017_543597_object34796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.37350622100001, 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.93829474099999999, 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118587 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.786 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370833+0000", 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370832+0000", 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.787 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370837+0000", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5280000000000003e-06 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370850+0000", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2576e-05 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370856+0000", 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5419999999999996e-06 2023-09-23T15:35:48.788 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370909+0000", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3155999999999997e-05 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.370915+0000", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5300000000000002e-06 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301114+0000", 2023-09-23T15:35:48.789 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.930198525 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301253+0000", 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013897099999999999 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.301389+0000", 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000135814 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.790 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305605+0000", 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0042168529999999996 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308882+0000", 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032768129999999999 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309075+0000", 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000192296 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.791 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309105+0000", 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.0505e-05 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309128+0000", 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 2.2966999999999999e-05 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118593 9.d 9:b3ef2821:::benchmark_data_smithi017_543597_object34802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.792 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.36480770899999, 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.934446412, 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118593 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.793 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379532+0000", 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379530+0000", 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.794 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379535+0000", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.4610000000000001e-06 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379545+0000", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.9029999999999999e-06 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379550+0000", 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2059999999999998e-06 2023-09-23T15:35:48.795 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379683+0000", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013354299999999999 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.379695+0000", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1833e-05 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303454+0000", 2023-09-23T15:35:48.796 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.92375848400000005 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303581+0000", 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000127126 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.303717+0000", 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000136534 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.797 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305643+0000", 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0019254490000000001 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.308885+0000", 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0032424519999999998 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313893+0000", 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0050071999999999998 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.798 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313940+0000", 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6974000000000001e-05 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313978+0000", 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8679e-05 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.799 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118617 9.d 9:b0bafb86:::benchmark_data_smithi017_543597_object34826:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.345043584, 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.91924088900000001, 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118617 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.800 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399296+0000", 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399295+0000", 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.801 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399305+0000", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0543999999999999e-05 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399318+0000", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.325e-05 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399326+0000", 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.9670000000000001e-06 2023-09-23T15:35:48.802 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399399+0000", 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2344999999999999e-05 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.399412+0000", 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3570999999999999e-05 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.803 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305695+0000", 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.90628282500000001 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305769+0000", 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3960000000000003e-05 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.305840+0000", 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0525999999999995e-05 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.804 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309148+0000", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0033085340000000001 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309185+0000", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.6806e-05 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318447+0000", 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0092622610000000008 2023-09-23T15:35:48.805 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318496+0000", 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.8952999999999997e-05 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318537+0000", 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.0771e-05 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.806 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118636 9.d 9:be5e9593:::benchmark_data_smithi017_543597_object34845:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.32027697299998, 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.89944873700000005, 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118636 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.807 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424063+0000", 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.808 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424061+0000", 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424066+0000", 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7740000000000004e-06 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424078+0000", 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2053e-05 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.809 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424084+0000", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.7379999999999999e-06 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424159+0000", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.5166999999999994e-05 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.424170+0000", 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1090000000000001e-05 2023-09-23T15:35:48.810 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307472+0000", 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.88330215099999998 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307554+0000", 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.1878000000000004e-05 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.811 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.307626+0000", 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1649000000000005e-05 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309193+0000", 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.001566858 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313853+0000", 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0046608609999999997 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.812 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323430+0000", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095760589999999996 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323476+0000", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.6399000000000001e-05 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323511+0000", 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5370000000000002e-05 2023-09-23T15:35:48.813 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118669 9.d 9:b914bc60:::benchmark_data_smithi017_543597_object34878:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.27908982500003, 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.85838970000000003, 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.814 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118669 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465250+0000", 2023-09-23T15:35:48.815 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465248+0000", 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465255+0000", 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.9920000000000001e-06 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.816 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465273+0000", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.8357999999999999e-05 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465282+0000", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465349+0000", 2023-09-23T15:35:48.817 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.6883000000000003e-05 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.465360+0000", 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.1181e-05 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309219+0000", 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.84385860999999995 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.818 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309293+0000", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3689000000000005e-05 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.309370+0000", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7571999999999995e-05 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.313995+0000", 2023-09-23T15:35:48.819 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.004624548 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314027+0000", 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2759999999999998e-05 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323570+0000", 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0095423469999999996 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.820 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323608+0000", 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.8599000000000001e-05 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323639+0000", 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.1059999999999997e-05 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.821 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118722 9.d 9:bde41141:::benchmark_data_smithi017_543597_object34931:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.212903332, 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.80511723000000002, 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118722 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.822 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531436+0000", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531435+0000", 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999981 2023-09-23T15:35:48.823 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531441+0000", 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5289999999999997e-06 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531458+0000", 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7099e-05 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.824 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.531466+0000", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3810000000000001e-06 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532385+0000", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00091978700000000004 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.532394+0000", 2023-09-23T15:35:48.825 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7830000000000004e-06 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311491+0000", 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77909717099999998 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311622+0000", 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013045400000000001 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.826 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.311748+0000", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012667700000000001 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314036+0000", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0022880510000000001 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318410+0000", 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0043731940000000004 2023-09-23T15:35:48.827 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336468+0000", 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.018058586000000001 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336516+0000", 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.7856999999999999e-05 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.828 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336553+0000", 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.7351999999999997e-05 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118729 9.d 9:b8e345b2:::benchmark_data_smithi017_543597_object34938:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.20658240400002, 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.799062563, 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.829 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118729 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.830 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537757+0000", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537756+0000", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537763+0000", 2023-09-23T15:35:48.831 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.0600000000000002e-06 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537777+0000", 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4175999999999999e-05 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537782+0000", 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9740000000000001e-06 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.832 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537847+0000", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5366999999999999e-05 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.537854+0000", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.5139999999999998e-06 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314070+0000", 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77621656999999999 2023-09-23T15:35:48.833 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314182+0000", 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00011181200000000001 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.314270+0000", 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.7640999999999997e-05 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.834 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323357+0000", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.009087108 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323363+0000", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.3090000000000001e-06 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336742+0000", 2023-09-23T15:35:48.835 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013378594000000001 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336784+0000", 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2333000000000003e-05 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336820+0000", 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.5672e-05 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.836 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118744 9.d 9:b54a00db:::benchmark_data_smithi017_543597_object34953:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.19133774199997, 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.78398909900000002, 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.837 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118744 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553002+0000", 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.838 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553000+0000", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553006+0000", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.4979999999999997e-06 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553019+0000", 2023-09-23T15:35:48.839 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.2855e-05 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553024+0000", 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.271e-06 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553069+0000", 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.494e-05 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.840 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.553077+0000", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.6000000000000001e-06 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.315967+0000", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76288994200000004 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316067+0000", 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00010011 2023-09-23T15:35:48.841 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.316183+0000", 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0001165 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323368+0000", 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0071846490000000004 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.842 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323401+0000", 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2699e-05 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336915+0000", 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.013514884 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336958+0000", 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.2064000000000002e-05 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.843 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336991+0000", 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.3380999999999998e-05 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118748 9.d 9:b5a57405:::benchmark_data_smithi017_543597_object34957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.18949299299999, 2023-09-23T15:35:48.844 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.79383968999999999, 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118748 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.845 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554847+0000", 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554845+0000", 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.846 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554853+0000", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.7870000000000002e-06 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554866+0000", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.3534e-05 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554874+0000", 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3470000000000001e-06 2023-09-23T15:35:48.847 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554929+0000", 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5269999999999998e-05 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.554938+0000", 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.9709999999999993e-06 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.848 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318565+0000", 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76362680500000002 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318662+0000", 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.6918999999999998e-05 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.318774+0000", 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000112495 2023-09-23T15:35:48.849 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323373+0000", 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0045984939999999998 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323543+0000", 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00017058199999999999 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.850 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348594+0000", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.025050833000000002 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348644+0000", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.0266000000000003e-05 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348686+0000", 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.1925999999999998e-05 2023-09-23T15:35:48.851 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118765 9.d 9:b25beba1:::benchmark_data_smithi017_543597_object34974:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.171014093, 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.77546686300000001, 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.852 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118765 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573325+0000", 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.853 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573324+0000", 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573331+0000", 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.4259999999999997e-06 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.854 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573342+0000", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0403e-05 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573348+0000", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.1800000000000001e-06 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573421+0000", 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.3468999999999997e-05 2023-09-23T15:35:48.855 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.573431+0000", 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5540000000000003e-06 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321037+0000", 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.747606306 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.856 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321179+0000", 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00014166 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.321251+0000", 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.2272000000000002e-05 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.857 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323671+0000", 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0024195829999999999 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327602+0000", 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0039315790000000002 2023-09-23T15:35:48.858 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348718+0000", 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.021115821999999999 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.859 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348757+0000", 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9100999999999997e-05 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.348792+0000", 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4980000000000001e-05 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.860 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118776 9.d 9:b496d689:::benchmark_data_smithi017_543597_object34985:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.15266479000002, 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.76259006200000001, 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.861 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118776 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.862 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591675+0000", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591673+0000", 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.863 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591678+0000", 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.2259999999999996e-06 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591693+0000", 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4647000000000001e-05 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.864 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591699+0000", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4659999999999996e-06 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591758+0000", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.8990000000000003e-05 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.591766+0000", 2023-09-23T15:35:48.865 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 7.1250000000000004e-06 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323691+0000", 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.73192517599999996 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323786+0000", 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.5377000000000005e-05 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.866 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.323892+0000", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000105882 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.332104+0000", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0082115420000000005 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.333996+0000", 2023-09-23T15:35:48.867 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0018925509999999999 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354164+0000", 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.020168175 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354220+0000", 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.5327000000000001e-05 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.868 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354265+0000", 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.5158999999999999e-05 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118785 9.d 9:b48c076d:::benchmark_data_smithi017_543597_object34994:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.13986719600001, 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.74991449799999998, 2023-09-23T15:35:48.869 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118785 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.870 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604472+0000", 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604471+0000", 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999986 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.871 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604477+0000", 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.806e-06 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604488+0000", 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.0818999999999999e-05 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604494+0000", 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.0739999999999998e-06 2023-09-23T15:35:48.872 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604547+0000", 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 5.3557999999999998e-05 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.604557+0000", 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.7880000000000001e-06 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.873 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325681+0000", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72112381699999994 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325761+0000", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.0216000000000006e-05 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.325859+0000", 2023-09-23T15:35:48.874 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 9.8181000000000001e-05 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336439+0000", 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.010580001 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336654+0000", 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.000215183 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.875 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354304+0000", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017649116999999999 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354353+0000", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4.9005000000000003e-05 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354387+0000", 2023-09-23T15:35:48.876 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.4282999999999997e-05 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "description": "osd_op(client.46157.0:118797 9.d 9:b088cc93:::benchmark_data_smithi017_543597_object35006:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e387)", 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "initiated_at": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "age": 445.11176149800002, 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.72191496499999996, 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "type_data": { 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T15:35:48.877 INFO:teuthology.orchestra.run.smithi027.stdout: "client_info": { 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client": "client.46157", 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "client_addr": "172.21.15.17:0/348348112", 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "tid": 118797 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "events": [ 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "initiated", 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.878 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "throttled", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632578+0000", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "header_read", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632576+0000", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 4294967295.9999976 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "all_read", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632584+0000", 2023-09-23T15:35:48.879 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4419999999999998e-06 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "dispatched", 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632601+0000", 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.7013000000000001e-05 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "queued_for_pg", 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632610+0000", 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 8.4749999999999993e-06 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.880 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632675+0000", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 6.4850000000000004e-05 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for peered", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:23.632689+0000", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.4800000000000001e-05 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "reached_pg", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327685+0000", 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.69499560800000004 2023-09-23T15:35:48.881 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "started", 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327806+0000", 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00012095999999999999 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "waiting for subops from 1,5", 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.327937+0000", 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.00013145299999999999 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.882 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336643+0000", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.0087055399999999995 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "op_commit", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.336662+0000", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 1.933e-05 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "sub_op_commit_rec", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354421+0000", 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 0.017759161999999998 2023-09-23T15:35:48.883 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "commit_sent", 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354461+0000", 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.9267000000000002e-05 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: }, 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: { 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "event": "done", 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "time": "2023-09-23T15:28:24.354493+0000", 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: "duration": 3.2258999999999997e-05 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.884 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.885 INFO:teuthology.orchestra.run.smithi027.stdout: } 2023-09-23T15:35:48.885 INFO:teuthology.orchestra.run.smithi027.stdout: ] 2023-09-23T15:35:48.885 INFO:teuthology.orchestra.run.smithi027.stdout:} 2023-09-23T15:35:48.885 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:48 smithi027 ceph-mon[195233]: pgmap v1476: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:48.885 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:48.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:48 smithi017 ceph-mon[215646]: pgmap v1476: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:48.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:48 smithi017 ceph-mon[222878]: pgmap v1476: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:48.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:49.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[215646]: failed to encode map e506 with expected crc 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[215646]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[222878]: failed to encode map e506 with expected crc 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:49.669 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:49 smithi017 ceph-mon[222878]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T15:35:49.681 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:49 smithi027 ceph-mon[195233]: failed to encode map e506 with expected crc 2023-09-23T15:35:50.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:49 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3441174866' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:50.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:49 smithi027 ceph-mon[195233]: osdmap e506: 8 total, 8 up, 8 in 2023-09-23T15:35:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[215646]: pgmap v1478: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[215646]: failed to encode map e506 with expected crc 2023-09-23T15:35:50.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[215646]: failed to encode map e506 with expected crc 2023-09-23T15:35:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[222878]: pgmap v1478: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[222878]: failed to encode map e506 with expected crc 2023-09-23T15:35:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:50 smithi017 ceph-mon[222878]: failed to encode map e506 with expected crc 2023-09-23T15:35:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:50 smithi027 ceph-mon[195233]: pgmap v1478: 103 pgs: 103 active+clean; 8.4 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:50 smithi027 ceph-mon[195233]: failed to encode map e506 with expected crc 2023-09-23T15:35:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:50 smithi027 ceph-mon[195233]: failed to encode map e506 with expected crc 2023-09-23T15:35:51.686 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T15:35:51.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3713720855' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:51.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:51 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3713720855' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:51.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:51 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3713720855' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:52.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:51 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T15:35:52.441 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T15:35:52.821 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: pgmap v1479: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[215646]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: pgmap v1479: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: failed to encode map e507 with expected crc 2023-09-23T15:35:52.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:52 smithi017 ceph-mon[222878]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: pgmap v1479: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: osdmap e507: 8 total, 8 up, 8 in 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:52 smithi027 ceph-mon[195233]: failed to encode map e507 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[215646]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[215646]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[215646]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[215646]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[215646]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[222878]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[222878]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T15:35:53.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[222878]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[222878]: failed to encode map e508 with expected crc 2023-09-23T15:35:53.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:53 smithi017 ceph-mon[222878]: failed to encode map e508 with expected crc 2023-09-23T15:35:54.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:53 smithi027 ceph-mon[195233]: failed to encode map e508 with expected crc 2023-09-23T15:35:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:53 smithi027 ceph-mon[195233]: osdmap e508: 8 total, 8 up, 8 in 2023-09-23T15:35:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:53 smithi027 ceph-mon[195233]: failed to encode map e508 with expected crc 2023-09-23T15:35:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:53 smithi027 ceph-mon[195233]: failed to encode map e508 with expected crc 2023-09-23T15:35:54.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:53 smithi027 ceph-mon[195233]: failed to encode map e508 with expected crc 2023-09-23T15:35:54.693 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T15:35:54.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[222878]: pgmap v1482: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:54.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/680173475' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:54.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[215646]: pgmap v1482: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/680173475' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:54.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:54 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:54 smithi027 ceph-mon[195233]: pgmap v1482: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/680173475' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:54 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:55.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T15:35:55.756 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:55.947 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[215646]: failed to encode map e509 with expected crc 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[215646]: failed to encode map e509 with expected crc 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[215646]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[215646]: failed to encode map e509 with expected crc 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[222878]: failed to encode map e509 with expected crc 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[222878]: failed to encode map e509 with expected crc 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[222878]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T15:35:55.948 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:55 smithi017 ceph-mon[222878]: failed to encode map e509 with expected crc 2023-09-23T15:35:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:55 smithi027 ceph-mon[195233]: failed to encode map e509 with expected crc 2023-09-23T15:35:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:55 smithi027 ceph-mon[195233]: failed to encode map e509 with expected crc 2023-09-23T15:35:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:55 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:56.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:55 smithi027 ceph-mon[195233]: osdmap e509: 8 total, 8 up, 8 in 2023-09-23T15:35:56.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:55 smithi027 ceph-mon[195233]: failed to encode map e509 with expected crc 2023-09-23T15:35:56.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:56 smithi017 ceph-mon[222878]: pgmap v1484: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:56.976 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:56 smithi017 ceph-mon[222878]: failed to encode map e509 with expected crc 2023-09-23T15:35:56.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:56 smithi017 ceph-mon[215646]: pgmap v1484: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:56.976 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:56 smithi017 ceph-mon[215646]: failed to encode map e509 with expected crc 2023-09-23T15:35:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:56 smithi027 ceph-mon[195233]: pgmap v1484: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:35:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:56 smithi027 ceph-mon[195233]: failed to encode map e509 with expected crc 2023-09-23T15:35:57.707 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T15:35:58.021 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:57 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:58.022 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:57 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:58.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:57 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T15:35:58.363 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:35:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:35:58] "GET /metrics HTTP/1.1" 200 34006 "" "Prometheus/2.43.0" 2023-09-23T15:35:58.381 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T15:35:58.750 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: pgmap v1485: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: pgmap v1485: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:35:58.964 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:58.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T15:35:58.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: failed to encode map e510 with expected crc 2023-09-23T15:35:58.965 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:35:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:35:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: pgmap v1485: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:35:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: failed to encode map e510 with expected crc 2023-09-23T15:35:59.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: failed to encode map e510 with expected crc 2023-09-23T15:35:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/953678032' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T15:35:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: osdmap e510: 8 total, 8 up, 8 in 2023-09-23T15:35:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: failed to encode map e510 with expected crc 2023-09-23T15:35:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:35:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:00.712 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T15:36:01.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:00 smithi027 ceph-mon[195233]: pgmap v1487: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:01.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:00 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:01.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:00 smithi017 ceph-mon[215646]: pgmap v1487: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:01.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:00 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:00 smithi017 ceph-mon[222878]: pgmap v1487: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:01.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:00 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:01.357 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T15:36:01.718 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:01.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[215646]: failed to encode map e511 with expected crc 2023-09-23T15:36:01.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[215646]: failed to encode map e511 with expected crc 2023-09-23T15:36:01.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:01.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[215646]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T15:36:01.982 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[215646]: failed to encode map e511 with expected crc 2023-09-23T15:36:01.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[222878]: failed to encode map e511 with expected crc 2023-09-23T15:36:01.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[222878]: failed to encode map e511 with expected crc 2023-09-23T15:36:01.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:01.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[222878]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T15:36:01.983 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:01 smithi017 ceph-mon[222878]: failed to encode map e511 with expected crc 2023-09-23T15:36:02.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:01 smithi027 ceph-mon[195233]: failed to encode map e511 with expected crc 2023-09-23T15:36:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:01 smithi027 ceph-mon[195233]: failed to encode map e511 with expected crc 2023-09-23T15:36:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:01 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1600234354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:01 smithi027 ceph-mon[195233]: osdmap e511: 8 total, 8 up, 8 in 2023-09-23T15:36:02.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:01 smithi027 ceph-mon[195233]: failed to encode map e511 with expected crc 2023-09-23T15:36:03.040 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:02 smithi017 ceph-mon[222878]: pgmap v1489: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:03.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:02 smithi017 ceph-mon[215646]: pgmap v1489: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:02 smithi027 ceph-mon[195233]: pgmap v1489: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T15:36:03.726 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T15:36:04.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:03 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/160757486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:03 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:03 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/160757486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:03 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:03 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/160757486' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:03 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T15:36:04.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T15:36:04.747 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:04.972 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[215646]: pgmap v1490: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:04.972 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[215646]: failed to encode map e512 with expected crc 2023-09-23T15:36:04.972 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:04.972 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[215646]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T15:36:04.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[222878]: pgmap v1490: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:04.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[222878]: failed to encode map e512 with expected crc 2023-09-23T15:36:04.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:04.973 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:04 smithi017 ceph-mon[222878]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T15:36:05.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:04 smithi027 ceph-mon[195233]: pgmap v1490: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:04 smithi027 ceph-mon[195233]: failed to encode map e512 with expected crc 2023-09-23T15:36:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:04 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T15:36:05.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:04 smithi027 ceph-mon[195233]: osdmap e512: 8 total, 8 up, 8 in 2023-09-23T15:36:06.730 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T15:36:07.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:06 smithi027 ceph-mon[195233]: pgmap v1492: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:06 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1914758288' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:06 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[215646]: pgmap v1492: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1914758288' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[222878]: pgmap v1492: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1914758288' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:06 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:07.446 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T15:36:07.814 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:08.012 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[215646]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.012 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[215646]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[215646]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[222878]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[222878]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:08.013 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:07 smithi017 ceph-mon[222878]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T15:36:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:07 smithi027 ceph-mon[195233]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:07 smithi027 ceph-mon[195233]: failed to encode map e513 with expected crc 2023-09-23T15:36:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:07 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:08.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:07 smithi027 ceph-mon[195233]: osdmap e513: 8 total, 8 up, 8 in 2023-09-23T15:36:08.313 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:08 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:08] "GET /metrics HTTP/1.1" 200 34002 "" "Prometheus/2.43.0" 2023-09-23T15:36:09.083 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:08 smithi017 ceph-mon[215646]: pgmap v1494: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:09.083 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:08 smithi017 ceph-mon[222878]: pgmap v1494: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:09.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:08 smithi027 ceph-mon[195233]: pgmap v1494: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:09.744 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T15:36:10.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:09 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1490334691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:09 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:09 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1490334691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:09 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:09 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1490334691' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:09 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T15:36:10.432 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T15:36:10.797 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:11.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[215646]: pgmap v1495: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:11.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[215646]: failed to encode map e514 with expected crc 2023-09-23T15:36:11.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:11.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[215646]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T15:36:11.046 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[215646]: failed to encode map e514 with expected crc 2023-09-23T15:36:11.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[222878]: pgmap v1495: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:11.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[222878]: failed to encode map e514 with expected crc 2023-09-23T15:36:11.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:11.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[222878]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T15:36:11.047 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:10 smithi017 ceph-mon[222878]: failed to encode map e514 with expected crc 2023-09-23T15:36:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:10 smithi027 ceph-mon[195233]: pgmap v1495: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:10 smithi027 ceph-mon[195233]: failed to encode map e514 with expected crc 2023-09-23T15:36:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:10 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T15:36:11.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:10 smithi027 ceph-mon[195233]: osdmap e514: 8 total, 8 up, 8 in 2023-09-23T15:36:11.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:10 smithi027 ceph-mon[195233]: failed to encode map e514 with expected crc 2023-09-23T15:36:12.754 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T15:36:13.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:12 smithi027 ceph-mon[195233]: pgmap v1497: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:12 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/799631250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:12 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[222878]: pgmap v1497: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:13.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/799631250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[215646]: pgmap v1497: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:13.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/799631250' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:12 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:13.412 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T15:36:13.780 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:14.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.013 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.014 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:13 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: osdmap e515: 8 total, 8 up, 8 in 2023-09-23T15:36:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: failed to encode map e515 with expected crc 2023-09-23T15:36:14.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:13 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:15.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:14 smithi027 ceph-mon[195233]: pgmap v1499: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:15.105 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:14 smithi017 ceph-mon[222878]: pgmap v1499: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:15.106 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:14 smithi017 ceph-mon[215646]: pgmap v1499: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:15.765 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T15:36:16.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:15 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1399093646' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:15 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:15 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1399093646' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:15 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:15 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1399093646' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:15 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T15:36:16.404 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T15:36:16.774 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:17.016 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: pgmap v1500: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:17.016 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[215646]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: pgmap v1500: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T15:36:17.017 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:16 smithi017 ceph-mon[222878]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: pgmap v1500: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: failed to encode map e516 with expected crc 2023-09-23T15:36:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T15:36:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: osdmap e516: 8 total, 8 up, 8 in 2023-09-23T15:36:17.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:16 smithi027 ceph-mon[195233]: failed to encode map e516 with expected crc 2023-09-23T15:36:18.140 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:18 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:18] "GET /metrics HTTP/1.1" 200 34007 "" "Prometheus/2.43.0" 2023-09-23T15:36:18.769 INFO:teuthology.orchestra.run.smithi017.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T15:36:19.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:18 smithi027 ceph-mon[195233]: pgmap v1502: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:19.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:18 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:19.192 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:18 smithi017 ceph-mon[215646]: pgmap v1502: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:19.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:18 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:18 smithi017 ceph-mon[222878]: pgmap v1502: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:19.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:18 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:19.437 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T15:36:19.799 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:20.037 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[215646]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[215646]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[215646]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[222878]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[222878]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:20.038 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:19 smithi017 ceph-mon[222878]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T15:36:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:19 smithi027 ceph-mon[195233]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:19 smithi027 ceph-mon[195233]: failed to encode map e517 with expected crc 2023-09-23T15:36:20.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:19 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1939764922' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:20.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:19 smithi027 ceph-mon[195233]: osdmap e517: 8 total, 8 up, 8 in 2023-09-23T15:36:21.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:20 smithi027 ceph-mon[195233]: pgmap v1504: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:21.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:20 smithi027 ceph-mon[195233]: failed to encode map e517 with expected crc 2023-09-23T15:36:21.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:20 smithi017 ceph-mon[215646]: pgmap v1504: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:21.114 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:20 smithi017 ceph-mon[215646]: failed to encode map e517 with expected crc 2023-09-23T15:36:21.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:20 smithi017 ceph-mon[222878]: pgmap v1504: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:21.114 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:20 smithi017 ceph-mon[222878]: failed to encode map e517 with expected crc 2023-09-23T15:36:21.783 INFO:teuthology.orchestra.run.smithi017.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T15:36:22.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:21 smithi027 ceph-mon[195233]: failed to encode map e517 with expected crc 2023-09-23T15:36:22.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:21 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:22.192 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:21 smithi017 ceph-mon[222878]: failed to encode map e517 with expected crc 2023-09-23T15:36:22.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:21 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:21 smithi017 ceph-mon[215646]: failed to encode map e517 with expected crc 2023-09-23T15:36:22.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:21 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T15:36:22.444 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T15:36:22.444 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T15:36:22.445 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T15:36:22.445 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T15:36:22.445 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T15:36:22.445 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T15:36:22.808 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:23.042 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: pgmap v1505: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[215646]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: pgmap v1505: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.043 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:23.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T15:36:23.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.044 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:22 smithi017 ceph-mon[222878]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: pgmap v1505: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/461675017' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T15:36:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: osdmap e518: 8 total, 8 up, 8 in 2023-09-23T15:36:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: failed to encode map e518 with expected crc 2023-09-23T15:36:23.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:22 smithi027 ceph-mon[195233]: failed to encode map e518 with expected crc 2023-09-23T15:36:24.562 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:36:24.562 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":518,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T15:36:21.772721+0000","last_up_change":"2023-09-23T15:27:40.376810+0000","last_in_change":"2023-09-23T15:35:27.657216+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"398","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"357","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"488","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":488,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":500,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6803","nonce":1593815239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6805","nonce":1593815239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6809","nonce":1593815239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6807","nonce":1593815239}]},"public_addr":"172.21.15.17:6803/1593815239","cluster_addr":"172.21.15.17:6805/1593815239","heartbeat_back_addr":"172.21.15.17:6809/1593815239","heartbeat_front_addr":"172.21.15.17:6807/1593815239","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":100,"last_clean_end":131,"up_from":183,"up_thru":500,"down_at":132,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6811","nonce":689323320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6813","nonce":689323320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6817","nonce":689323320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6815","nonce":689323320}]},"public_addr":"172.21.15.17:6811/689323320","cluster_addr":"172.21.15.17:6813/689323320","heartbeat_back_addr":"172.21.15.17:6817/689323320","heartbeat_front_addr":"172.21.15.17:6815/689323320","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":499,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6819","nonce":1172679298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6821","nonce":1172679298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6825","nonce":1172679298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6823","nonce":1172679298}]},"public_addr":"172.21.15.17:6819/1172679298","cluster_addr":"172.21.15.17:6821/1172679298","heartbeat_back_addr":"172.21.15.17:6825/1172679298","heartbeat_front_addr":"172.21.15.17:6823/1172679298","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":507,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6827","nonce":3586551743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6829","nonce":3586551743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6833","nonce":3586551743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6831","nonce":3586551743}]},"public_addr":"172.21.15.17:6827/3586551743","cluster_addr":"172.21.15.17:6829/3586551743","heartbeat_back_addr":"172.21.15.17:6833/3586551743","heartbeat_front_addr":"172.21.15.17:6831/3586551743","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":499,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":500,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":280,"up_from":304,"up_thru":500,"down_at":281,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6817","nonce":3452389963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6819","nonce":3452389963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6823","nonce":3452389963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6821","nonce":3452389963}]},"public_addr":"172.21.15.27:6817/3452389963","cluster_addr":"172.21.15.27:6819/3452389963","heartbeat_back_addr":"172.21.15.27:6823/3452389963","heartbeat_front_addr":"172.21.15.27:6821/3452389963","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":358,"up_from":374,"up_thru":500,"down_at":359,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6825","nonce":1440800272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6827","nonce":1440800272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6831","nonce":1440800272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6829","nonce":1440800272}]},"public_addr":"172.21.15.27:6825/1440800272","cluster_addr":"172.21.15.27:6827/1440800272","heartbeat_back_addr":"172.21.15.27:6831/1440800272","heartbeat_front_addr":"172.21.15.27:6829/1440800272","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":131},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":280},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":358}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.7","mappings":[{"from":3,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.9","mappings":[{"from":1,"to":7}]},{"pgid":"4.13","mappings":[{"from":1,"to":0}]},{"pgid":"4.14","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.17:0/623737292":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/2369426862":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6800/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2670521468":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2135416630":"2023-09-24T14:59:04.520407+0000","172.21.15.27:6834/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/707576150":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/244258131":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6801/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/1467411231":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.27:0/816410812":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.27:6835/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.27:0/1374246236":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/1851970177":"2023-09-24T14:59:04.520407+0000","172.21.15.27:0/986573237":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/774095134":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1666581637":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.27:0/79453409":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/201191039":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/3598198038":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6800/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/1926022044":"2023-09-24T15:00:28.003544+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T15:36:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[215646]: pgmap v1507: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:36:24.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2824793749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:24.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[222878]: pgmap v1507: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:24.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:36:24.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:24 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2824793749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:24 smithi027 ceph-mon[195233]: pgmap v1507: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:24 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T15:36:25.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:24 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2824793749' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:25.387 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T15:36:25.388 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T15:36:26.025 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:27.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:26 smithi027 ceph-mon[195233]: pgmap v1508: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:27.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:26 smithi017 ceph-mon[215646]: pgmap v1508: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:27.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:26 smithi017 ceph-mon[222878]: pgmap v1508: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:28.175 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:36:28.175 INFO:teuthology.orchestra.run.smithi017.stdout:{"epoch":518,"fsid":"546d6668-5a20-11ee-8db4-212e2dc638e7","created":"2023-09-23T14:50:02.735782+0000","modified":"2023-09-23T15:36:21.772721+0000","last_up_change":"2023-09-23T15:27:40.376810+0000","last_in_change":"2023-09-23T15:35:27.657216+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":25,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T14:53:07.440399+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"398","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"357","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T14:55:41.150126+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"488","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":488,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T14:56:30.265228+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T14:56:30.763278+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"0ab537e0-788a-404a-aee3-04d5c85f3ea8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":91,"up_from":95,"up_thru":500,"down_at":92,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6802","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6803","nonce":1593815239}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6804","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6805","nonce":1593815239}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6808","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6809","nonce":1593815239}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6806","nonce":1593815239},{"type":"v1","addr":"172.21.15.17:6807","nonce":1593815239}]},"public_addr":"172.21.15.17:6803/1593815239","cluster_addr":"172.21.15.17:6805/1593815239","heartbeat_back_addr":"172.21.15.17:6809/1593815239","heartbeat_front_addr":"172.21.15.17:6807/1593815239","state":["exists","up"]},{"osd":1,"uuid":"2d26e6ec-8564-4b09-a9e8-f7fe77e36a65","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":100,"last_clean_end":131,"up_from":183,"up_thru":500,"down_at":132,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6810","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6811","nonce":689323320}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6812","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6813","nonce":689323320}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6816","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6817","nonce":689323320}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6814","nonce":689323320},{"type":"v1","addr":"172.21.15.17:6815","nonce":689323320}]},"public_addr":"172.21.15.17:6811/689323320","cluster_addr":"172.21.15.17:6813/689323320","heartbeat_back_addr":"172.21.15.17:6817/689323320","heartbeat_front_addr":"172.21.15.17:6815/689323320","state":["exists","up"]},{"osd":2,"uuid":"509065ed-f964-449d-8ed3-bb7c0b529dd8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":101,"up_from":105,"up_thru":499,"down_at":102,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6818","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6819","nonce":1172679298}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6820","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6821","nonce":1172679298}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6824","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6825","nonce":1172679298}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6822","nonce":1172679298},{"type":"v1","addr":"172.21.15.17:6823","nonce":1172679298}]},"public_addr":"172.21.15.17:6819/1172679298","cluster_addr":"172.21.15.17:6821/1172679298","heartbeat_back_addr":"172.21.15.17:6825/1172679298","heartbeat_front_addr":"172.21.15.17:6823/1172679298","state":["exists","up"]},{"osd":3,"uuid":"1430d849-5f40-43a2-9ebd-c82ddaaa262b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":106,"up_from":110,"up_thru":507,"down_at":107,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6826","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6827","nonce":3586551743}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6828","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6829","nonce":3586551743}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6832","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6833","nonce":3586551743}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.17:6830","nonce":3586551743},{"type":"v1","addr":"172.21.15.17:6831","nonce":3586551743}]},"public_addr":"172.21.15.17:6827/3586551743","cluster_addr":"172.21.15.17:6829/3586551743","heartbeat_back_addr":"172.21.15.17:6833/3586551743","heartbeat_front_addr":"172.21.15.17:6831/3586551743","state":["exists","up"]},{"osd":4,"uuid":"5b9137a6-fdb7-4150-813f-c72707df3509","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":499,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6800","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6801","nonce":2107894574}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6802","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6803","nonce":2107894574}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6806","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6807","nonce":2107894574}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6804","nonce":2107894574},{"type":"v1","addr":"172.21.15.27:6805","nonce":2107894574}]},"public_addr":"172.21.15.27:6801/2107894574","cluster_addr":"172.21.15.27:6803/2107894574","heartbeat_back_addr":"172.21.15.27:6807/2107894574","heartbeat_front_addr":"172.21.15.27:6805/2107894574","state":["exists","up"]},{"osd":5,"uuid":"06d868f4-810f-45ef-be6a-0c6edcea5ad7","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":500,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6808","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6809","nonce":634894946}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6810","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6811","nonce":634894946}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6814","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6815","nonce":634894946}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6812","nonce":634894946},{"type":"v1","addr":"172.21.15.27:6813","nonce":634894946}]},"public_addr":"172.21.15.27:6809/634894946","cluster_addr":"172.21.15.27:6811/634894946","heartbeat_back_addr":"172.21.15.27:6815/634894946","heartbeat_front_addr":"172.21.15.27:6813/634894946","state":["exists","up"]},{"osd":6,"uuid":"06f02713-faf7-456c-bd0d-52eb13680742","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":47,"last_clean_end":280,"up_from":304,"up_thru":500,"down_at":281,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6816","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6817","nonce":3452389963}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6818","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6819","nonce":3452389963}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6822","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6823","nonce":3452389963}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6820","nonce":3452389963},{"type":"v1","addr":"172.21.15.27:6821","nonce":3452389963}]},"public_addr":"172.21.15.27:6817/3452389963","cluster_addr":"172.21.15.27:6819/3452389963","heartbeat_back_addr":"172.21.15.27:6823/3452389963","heartbeat_front_addr":"172.21.15.27:6821/3452389963","state":["exists","up"]},{"osd":7,"uuid":"d3cb8c99-f1ea-446e-9b8c-7d916ac62bc4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":55,"last_clean_end":358,"up_from":374,"up_thru":500,"down_at":359,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6824","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6825","nonce":1440800272}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6826","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6827","nonce":1440800272}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6830","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6831","nonce":1440800272}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.27:6828","nonce":1440800272},{"type":"v1","addr":"172.21.15.27:6829","nonce":1440800272}]},"public_addr":"172.21.15.27:6825/1440800272","cluster_addr":"172.21.15.27:6827/1440800272","heartbeat_back_addr":"172.21.15.27:6831/1440800272","heartbeat_front_addr":"172.21.15.27:6829/1440800272","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:17.030682+0000","dead_epoch":91},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:52:40.353617+0000","dead_epoch":131},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:03.642734+0000","dead_epoch":101},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:27.063567+0000","dead_epoch":106},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:53:48.562577+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:09.851500+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:31.456940+0000","dead_epoch":280},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T14:54:53.581286+0000","dead_epoch":358}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"3.7","mappings":[{"from":3,"to":2}]},{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.9","mappings":[{"from":1,"to":7}]},{"pgid":"4.13","mappings":[{"from":1,"to":0}]},{"pgid":"4.14","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.27:0/79453409":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1503872993":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1791820912":"2023-09-24T14:50:31.828392+0000","172.21.15.17:6800/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2670521468":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/2135416630":"2023-09-24T14:59:04.520407+0000","172.21.15.27:6834/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/707576150":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/244258131":"2023-09-24T14:59:45.132154+0000","172.21.15.27:0/1374246236":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6801/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.27:0/816410812":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/814296392":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/4217628619":"2023-09-24T14:55:28.581618+0000","172.21.15.27:6835/1216874840":"2023-09-24T14:59:45.132154+0000","172.21.15.17:6800/1159417105":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/1467411231":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/625940860":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6801/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:6801/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/2463078697":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/1605426391":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/2823536185":"2023-09-24T14:50:58.900461+0000","172.21.15.17:6800/3040364158":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/201191039":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/1851970177":"2023-09-24T14:59:04.520407+0000","172.21.15.17:0/1666581637":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/774095134":"2023-09-24T14:59:04.520407+0000","172.21.15.17:6800/3796759750":"2023-09-24T14:55:28.581618+0000","172.21.15.17:0/3598198038":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6800/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:6801/1926022044":"2023-09-24T15:00:28.003544+0000","172.21.15.17:0/2369426862":"2023-09-24T15:00:28.003544+0000","172.21.15.27:0/986573237":"2023-09-24T14:59:45.132154+0000","172.21.15.17:0/1446163903":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/577227578":"2023-09-24T14:50:31.828392+0000","172.21.15.17:0/3013219017":"2023-09-24T14:50:58.900461+0000","172.21.15.17:0/623737292":"2023-09-24T15:00:28.003544+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T15:36:28.352 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:28 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:28] "GET /metrics HTTP/1.1" 200 34007 "" "Prometheus/2.43.0" 2023-09-23T15:36:28.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T15:36:28.987 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T15:36:28.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T15:36:28.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T15:36:28.988 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T15:36:28.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T15:36:28.989 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T15:36:28.990 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[215646]: pgmap v1509: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1667597286' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:29.040 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[222878]: pgmap v1509: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:29.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1667597286' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:29.041 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:28 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:28 smithi027 ceph-mon[195233]: pgmap v1509: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:29.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:28 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1667597286' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T15:36:29.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:28 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:30.132 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:30.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:30.304 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:30.338 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:30.557 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:31.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:30 smithi027 ceph-mon[195233]: pgmap v1510: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:30 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[215646]: pgmap v1510: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[222878]: pgmap v1510: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:30 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:31.541 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:31.607 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:32.003 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:31 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:32.003 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:31 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:32.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:31 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:32.244 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:33.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:32 smithi027 ceph-mon[195233]: pgmap v1511: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:33.169 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:32 smithi017 ceph-mon[222878]: pgmap v1511: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:33.169 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:32 smithi017 ceph-mon[215646]: pgmap v1511: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:34.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:34 smithi017 ceph-mon[215646]: pgmap v1512: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:34.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:34 smithi017 ceph-mon[222878]: pgmap v1512: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:34.984 INFO:teuthology.orchestra.run.smithi017.stdout:1305670058155 2023-09-23T15:36:34.985 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T15:36:35.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:34 smithi027 ceph-mon[195233]: pgmap v1512: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:37.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:36 smithi027 ceph-mon[195233]: pgmap v1513: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:37.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:36 smithi017 ceph-mon[215646]: pgmap v1513: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:37.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:36 smithi017 ceph-mon[222878]: pgmap v1513: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:38.315 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:38 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:38] "GET /metrics HTTP/1.1" 200 34005 "" "Prometheus/2.43.0" 2023-09-23T15:36:38.395 INFO:teuthology.orchestra.run.smithi017.stdout:408021893503 2023-09-23T15:36:38.396 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T15:36:38.876 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:38 smithi017 ceph-mon[215646]: pgmap v1514: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:38.876 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:38 smithi017 ceph-mon[222878]: pgmap v1514: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:39.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:38 smithi027 ceph-mon[195233]: pgmap v1514: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:40.116 INFO:teuthology.orchestra.run.smithi017.stdout:167503725055 2023-09-23T15:36:40.116 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T15:36:40.717 INFO:teuthology.orchestra.run.smithi017.stdout:133143986692 2023-09-23T15:36:40.717 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T15:36:40.741 INFO:teuthology.orchestra.run.smithi017.stdout:472446402925 2023-09-23T15:36:40.741 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T15:36:40.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:40 smithi017 ceph-mon[215646]: pgmap v1515: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:40.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:40 smithi017 ceph-mon[222878]: pgmap v1515: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:41.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:40 smithi027 ceph-mon[195233]: pgmap v1515: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:41.151 INFO:teuthology.orchestra.run.smithi017.stdout:785979015457 2023-09-23T15:36:41.153 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T15:36:41.372 INFO:teuthology.orchestra.run.smithi017.stdout:450971566454 2023-09-23T15:36:41.372 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T15:36:41.499 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:41.868 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:42.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:42 smithi017 ceph-mon[215646]: pgmap v1516: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:42.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:42 smithi017 ceph-mon[222878]: pgmap v1516: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:43.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:42 smithi027 ceph-mon[195233]: pgmap v1516: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:43.269 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:43.269 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:43.363 INFO:teuthology.orchestra.run.smithi017.stdout:1606317768814 2023-09-23T15:36:43.363 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T15:36:43.515 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:43.893 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:43 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:43.893 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:43 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:44.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:43 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:44.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:44 smithi017 ceph-mon[215646]: pgmap v1517: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:44.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:44 smithi017 ceph-mon[222878]: pgmap v1517: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:45.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:44 smithi027 ceph-mon[195233]: pgmap v1517: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:46.656 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:46.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:46 smithi017 ceph-mon[215646]: pgmap v1518: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:46.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:46 smithi017 ceph-mon[222878]: pgmap v1518: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:47.041 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:47.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:46 smithi027 ceph-mon[195233]: pgmap v1518: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:47.815 INFO:teuthology.orchestra.run.smithi017.stdout:408021893505 2023-09-23T15:36:48.224 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:48 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:48] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.43.0" 2023-09-23T15:36:48.709 INFO:teuthology.orchestra.run.smithi017.stdout:785979015459 2023-09-23T15:36:48.924 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[215646]: pgmap v1519: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:48.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/2488583144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:36:48.925 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/392510778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:36:48.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[222878]: pgmap v1519: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:48.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/2488583144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:36:48.925 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:48 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/392510778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:36:49.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:48 smithi027 ceph-mon[195233]: pgmap v1519: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/2488583144' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T15:36:49.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:48 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/392510778' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T15:36:50.056 INFO:teuthology.orchestra.run.smithi017.stdout:133143986694 2023-09-23T15:36:50.297 INFO:teuthology.orchestra.run.smithi017.stdout:167503725057 2023-09-23T15:36:50.777 INFO:teuthology.orchestra.run.smithi017.stdout:1305670058158 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: pgmap v1520: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/849224739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/4182362522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:50.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: pgmap v1520: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/849224739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/4182362522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:36:50.944 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:50 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: pgmap v1520: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:51.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/849224739' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/4182362522' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T15:36:51.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:50 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' 2023-09-23T15:36:51.312 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986692 got 133143986694 for osd.4 2023-09-23T15:36:51.313 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:51.864 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:51 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3485797184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:36:51.864 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:51 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3485797184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:36:52.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:51 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3485797184' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T15:36:52.401 INFO:tasks.cephadm.ceph_manager.ceph:need seq 785979015457 got 785979015459 for osd.1 2023-09-23T15:36:52.401 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:52.446 INFO:tasks.cephadm.ceph_manager.ceph:need seq 408021893503 got 408021893505 for osd.0 2023-09-23T15:36:52.446 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:52.835 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:52.942 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:52 smithi017 ceph-mon[215646]: pgmap v1521: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:52.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:52 smithi017 ceph-mon[222878]: pgmap v1521: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:53.013 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725055 got 167503725057 for osd.5 2023-09-23T15:36:53.013 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:53.031 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1305670058155 got 1305670058158 for osd.6 2023-09-23T15:36:53.032 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:53.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:52 smithi027 ceph-mon[195233]: pgmap v1521: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:53.793 INFO:teuthology.orchestra.run.smithi017.stdout:472446402928 2023-09-23T15:36:54.089 INFO:teuthology.orchestra.run.smithi017.stdout:450971566457 2023-09-23T15:36:55.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:54 smithi027 ceph-mon[195233]: pgmap v1522: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3597364626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:36:55.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:54 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/3875312430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[215646]: pgmap v1522: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3597364626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/3875312430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[222878]: pgmap v1522: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3597364626' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T15:36:55.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:54 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/3875312430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T15:36:55.210 INFO:tasks.cephadm.ceph_manager.ceph:need seq 472446402925 got 472446402928 for osd.3 2023-09-23T15:36:55.210 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:55.410 INFO:tasks.cephadm.ceph_manager.ceph:need seq 450971566454 got 450971566457 for osd.2 2023-09-23T15:36:55.410 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:55.903 INFO:teuthology.orchestra.run.smithi017.stdout:1606317768817 2023-09-23T15:36:56.581 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1606317768814 got 1606317768817 for osd.7 2023-09-23T15:36:56.582 DEBUG:teuthology.parallel:result is None 2023-09-23T15:36:56.582 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T15:36:56.583 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:36:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:56 smithi017 ceph-mon[215646]: pgmap v1523: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:56.943 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:56 smithi017 ceph-mon[215646]: from='client.? 172.21.15.17:0/1104146347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:36:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:56 smithi017 ceph-mon[222878]: pgmap v1523: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:56.943 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:56 smithi017 ceph-mon[222878]: from='client.? 172.21.15.17:0/1104146347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:36:56.953 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:57.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:56 smithi027 ceph-mon[195233]: pgmap v1523: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:57.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:56 smithi027 ceph-mon[195233]: from='client.? 172.21.15.17:0/1104146347' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T15:36:58.309 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:36:58 smithi017 conmon[202315]: ::ffff:172.21.15.27 - - [23/Sep/2023:15:36:58] "GET /metrics HTTP/1.1" 200 34004 "" "Prometheus/2.43.0" 2023-09-23T15:36:58.735 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:36:58.737 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:36:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:58 smithi027 ceph-mon[195233]: pgmap v1524: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:59.100 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:58 smithi027 ceph-mon[195233]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:58 smithi017 ceph-mon[215646]: pgmap v1524: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:59.193 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:58 smithi017 ceph-mon[215646]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:58 smithi017 ceph-mon[222878]: pgmap v1524: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:36:59.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:58 smithi017 ceph-mon[222878]: from='mgr.34107 172.21.15.17:0/3810089709' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T15:36:59.367 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1524,"stamp":"2023-09-23T15:36:57.102893+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11944,"num_read_kb":26378,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":4205364,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6998,"ondisk_log_size":6998,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4750220,"kb_used_data":73420,"kb_used_omap":57,"kb_used_meta":4676678,"kb_avail":745210996,"statfs":{"total":767960285184,"available":763096059904,"internally_reserved":0,"allocated":75182080,"data_stored":59433305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4788918297},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003715"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":563,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766953+0000","last_change":"2023-09-23T15:35:45.127684+0000","last_active":"2023-09-23T15:36:18.766953+0000","last_peered":"2023-09-23T15:36:18.766953+0000","last_clean":"2023-09-23T15:36:18.766953+0000","last_became_active":"2023-09-23T15:35:31.704743+0000","last_became_peered":"2023-09-23T15:35:31.704743+0000","last_unstale":"2023-09-23T15:36:18.766953+0000","last_undegraded":"2023-09-23T15:36:18.766953+0000","last_fullsized":"2023-09-23T15:36:18.766953+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:35:45.127621+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:35:45.127621+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:44.864740+0000","scrub_duration":0.0017349170000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258188+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:36:20.258188+0000","last_peered":"2023-09-23T15:36:20.258188+0000","last_clean":"2023-09-23T15:36:20.258188+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:36:20.258188+0000","last_undegraded":"2023-09-23T15:36:20.258188+0000","last_fullsized":"2023-09-23T15:36:20.258188+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":395,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920248+0000","last_change":"2023-09-23T15:35:52.691279+0000","last_active":"2023-09-23T15:36:18.920248+0000","last_peered":"2023-09-23T15:36:18.920248+0000","last_clean":"2023-09-23T15:36:18.920248+0000","last_became_active":"2023-09-23T15:35:52.690933+0000","last_became_peered":"2023-09-23T15:35:52.690933+0000","last_unstale":"2023-09-23T15:36:18.920248+0000","last_undegraded":"2023-09-23T15:36:18.920248+0000","last_fullsized":"2023-09-23T15:36:18.920248+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:42.453633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":541,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766977+0000","last_change":"2023-09-23T15:35:34.128191+0000","last_active":"2023-09-23T15:36:18.766977+0000","last_peered":"2023-09-23T15:36:18.766977+0000","last_clean":"2023-09-23T15:36:18.766977+0000","last_became_active":"2023-09-23T15:35:31.708660+0000","last_became_peered":"2023-09-23T15:35:31.708660+0000","last_unstale":"2023-09-23T15:36:18.766977+0000","last_undegraded":"2023-09-23T15:36:18.766977+0000","last_fullsized":"2023-09-23T15:36:18.766977+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:35:34.128097+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:35:34.128097+0000","last_clean_scrub_stamp":"2023-09-23T15:35:34.128097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:25.416633+0000","scrub_duration":0.0018547609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":445,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.389991+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:36:19.389991+0000","last_peered":"2023-09-23T15:36:19.389991+0000","last_clean":"2023-09-23T15:36:19.389991+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:36:19.389991+0000","last_undegraded":"2023-09-23T15:36:19.389991+0000","last_fullsized":"2023-09-23T15:36:19.389991+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":517,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.117949+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:36:22.117949+0000","last_peered":"2023-09-23T15:36:22.117949+0000","last_clean":"2023-09-23T15:36:22.117949+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:36:22.117949+0000","last_undegraded":"2023-09-23T15:36:22.117949+0000","last_fullsized":"2023-09-23T15:36:22.117949+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":653,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259771+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:36:20.259771+0000","last_peered":"2023-09-23T15:36:20.259771+0000","last_clean":"2023-09-23T15:36:20.259771+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:36:20.259771+0000","last_undegraded":"2023-09-23T15:36:20.259771+0000","last_fullsized":"2023-09-23T15:36:20.259771+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":432,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919896+0000","last_change":"2023-09-23T15:35:52.689532+0000","last_active":"2023-09-23T15:36:18.919896+0000","last_peered":"2023-09-23T15:36:18.919896+0000","last_clean":"2023-09-23T15:36:18.919896+0000","last_became_active":"2023-09-23T15:35:52.689189+0000","last_became_peered":"2023-09-23T15:35:52.689189+0000","last_unstale":"2023-09-23T15:36:18.919896+0000","last_undegraded":"2023-09-23T15:36:18.919896+0000","last_fullsized":"2023-09-23T15:36:18.919896+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:10.452807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":488,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771717+0000","last_change":"2023-09-23T15:35:28.686386+0000","last_active":"2023-09-23T15:36:18.771717+0000","last_peered":"2023-09-23T15:36:18.771717+0000","last_clean":"2023-09-23T15:36:18.771717+0000","last_became_active":"2023-09-23T15:35:28.685718+0000","last_became_peered":"2023-09-23T15:35:28.685718+0000","last_unstale":"2023-09-23T15:36:18.771717+0000","last_undegraded":"2023-09-23T15:36:18.771717+0000","last_fullsized":"2023-09-23T15:36:18.771717+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:46.720477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":826,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257825+0000","last_change":"2023-09-23T15:35:31.710915+0000","last_active":"2023-09-23T15:36:20.257825+0000","last_peered":"2023-09-23T15:36:20.257825+0000","last_clean":"2023-09-23T15:36:20.257825+0000","last_became_active":"2023-09-23T15:35:31.710539+0000","last_became_peered":"2023-09-23T15:35:31.710539+0000","last_unstale":"2023-09-23T15:36:20.257825+0000","last_undegraded":"2023-09-23T15:36:20.257825+0000","last_fullsized":"2023-09-23T15:36:20.257825+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:24.282583+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765155+0000","last_change":"2023-09-23T15:35:28.678928+0000","last_active":"2023-09-23T15:36:18.765155+0000","last_peered":"2023-09-23T15:36:18.765155+0000","last_clean":"2023-09-23T15:36:18.765155+0000","last_became_active":"2023-09-23T15:35:28.678376+0000","last_became_peered":"2023-09-23T15:35:28.678376+0000","last_unstale":"2023-09-23T15:36:18.765155+0000","last_undegraded":"2023-09-23T15:36:18.765155+0000","last_fullsized":"2023-09-23T15:36:18.765155+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:13.220709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":429,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919913+0000","last_change":"2023-09-23T15:35:52.691449+0000","last_active":"2023-09-23T15:36:18.919913+0000","last_peered":"2023-09-23T15:36:18.919913+0000","last_clean":"2023-09-23T15:36:18.919913+0000","last_became_active":"2023-09-23T15:35:52.690736+0000","last_became_peered":"2023-09-23T15:35:52.690736+0000","last_unstale":"2023-09-23T15:36:18.919913+0000","last_undegraded":"2023-09-23T15:36:18.919913+0000","last_fullsized":"2023-09-23T15:36:18.919913+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:41.247555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":929,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.038991+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:36:19.038991+0000","last_peered":"2023-09-23T15:36:19.038991+0000","last_clean":"2023-09-23T15:36:19.038991+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:36:19.038991+0000","last_undegraded":"2023-09-23T15:36:19.038991+0000","last_fullsized":"2023-09-23T15:36:19.038991+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767078+0000","last_change":"2023-09-23T15:35:28.690198+0000","last_active":"2023-09-23T15:36:18.767078+0000","last_peered":"2023-09-23T15:36:18.767078+0000","last_clean":"2023-09-23T15:36:18.767078+0000","last_became_active":"2023-09-23T15:35:28.689773+0000","last_became_peered":"2023-09-23T15:35:28.689773+0000","last_unstale":"2023-09-23T15:36:18.767078+0000","last_undegraded":"2023-09-23T15:36:18.767078+0000","last_fullsized":"2023-09-23T15:36:18.767078+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:20.156779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":389,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920023+0000","last_change":"2023-09-23T15:35:52.694201+0000","last_active":"2023-09-23T15:36:18.920023+0000","last_peered":"2023-09-23T15:36:18.920023+0000","last_clean":"2023-09-23T15:36:18.920023+0000","last_became_active":"2023-09-23T15:35:52.694015+0000","last_became_peered":"2023-09-23T15:35:52.694015+0000","last_unstale":"2023-09-23T15:36:18.920023+0000","last_undegraded":"2023-09-23T15:36:18.920023+0000","last_fullsized":"2023-09-23T15:36:18.920023+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:36:21.761252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":602,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767263+0000","last_change":"2023-09-23T15:35:31.709507+0000","last_active":"2023-09-23T15:36:18.767263+0000","last_peered":"2023-09-23T15:36:18.767263+0000","last_clean":"2023-09-23T15:36:18.767263+0000","last_became_active":"2023-09-23T15:35:31.708061+0000","last_became_peered":"2023-09-23T15:35:31.708061+0000","last_unstale":"2023-09-23T15:36:18.767263+0000","last_undegraded":"2023-09-23T15:36:18.767263+0000","last_fullsized":"2023-09-23T15:36:18.767263+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:06.824864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:06.824864+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:44.536644+0000","scrub_duration":0.0065159160000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":263,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259081+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:36:20.259081+0000","last_peered":"2023-09-23T15:36:20.259081+0000","last_clean":"2023-09-23T15:36:20.259081+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:36:20.259081+0000","last_undegraded":"2023-09-23T15:36:20.259081+0000","last_fullsized":"2023-09-23T15:36:20.259081+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":414,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390209+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:36:19.390209+0000","last_peered":"2023-09-23T15:36:19.390209+0000","last_clean":"2023-09-23T15:36:19.390209+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:36:19.390209+0000","last_undegraded":"2023-09-23T15:36:19.390209+0000","last_fullsized":"2023-09-23T15:36:19.390209+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":671,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390467+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:36:19.390467+0000","last_peered":"2023-09-23T15:36:19.390467+0000","last_clean":"2023-09-23T15:36:19.390467+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:36:19.390467+0000","last_undegraded":"2023-09-23T15:36:19.390467+0000","last_fullsized":"2023-09-23T15:36:19.390467+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":184,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259992+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:36:20.259992+0000","last_peered":"2023-09-23T15:36:20.259992+0000","last_clean":"2023-09-23T15:36:20.259992+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:36:20.259992+0000","last_undegraded":"2023-09-23T15:36:20.259992+0000","last_fullsized":"2023-09-23T15:36:20.259992+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258012+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:36:20.258012+0000","last_peered":"2023-09-23T15:36:20.258012+0000","last_clean":"2023-09-23T15:36:20.258012+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:36:20.258012+0000","last_undegraded":"2023-09-23T15:36:20.258012+0000","last_fullsized":"2023-09-23T15:36:20.258012+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":865,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763939+0000","last_change":"2023-09-23T15:35:31.707371+0000","last_active":"2023-09-23T15:36:18.763939+0000","last_peered":"2023-09-23T15:36:18.763939+0000","last_clean":"2023-09-23T15:36:18.763939+0000","last_became_active":"2023-09-23T15:35:31.707047+0000","last_became_peered":"2023-09-23T15:35:31.707047+0000","last_unstale":"2023-09-23T15:36:18.763939+0000","last_undegraded":"2023-09-23T15:36:18.763939+0000","last_fullsized":"2023-09-23T15:36:18.763939+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:04.181094+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":267,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259078+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:36:20.259078+0000","last_peered":"2023-09-23T15:36:20.259078+0000","last_clean":"2023-09-23T15:36:20.259078+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:36:20.259078+0000","last_undegraded":"2023-09-23T15:36:20.259078+0000","last_fullsized":"2023-09-23T15:36:20.259078+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039240+0000","last_change":"2023-09-23T15:33:30.765277+0000","last_active":"2023-09-23T15:36:19.039240+0000","last_peered":"2023-09-23T15:36:19.039240+0000","last_clean":"2023-09-23T15:36:19.039240+0000","last_became_active":"2023-09-23T15:33:30.764903+0000","last_became_peered":"2023-09-23T15:33:30.764903+0000","last_unstale":"2023-09-23T15:36:19.039240+0000","last_undegraded":"2023-09-23T15:36:19.039240+0000","last_fullsized":"2023-09-23T15:36:19.039240+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:24:08.782060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":660,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766815+0000","last_change":"2023-09-23T15:35:31.704842+0000","last_active":"2023-09-23T15:36:18.766815+0000","last_peered":"2023-09-23T15:36:18.766815+0000","last_clean":"2023-09-23T15:36:18.766815+0000","last_became_active":"2023-09-23T15:35:31.704369+0000","last_became_peered":"2023-09-23T15:35:31.704369+0000","last_unstale":"2023-09-23T15:36:18.766815+0000","last_undegraded":"2023-09-23T15:36:18.766815+0000","last_fullsized":"2023-09-23T15:36:18.766815+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:56.396764+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":514,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763984+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:36:18.763984+0000","last_peered":"2023-09-23T15:36:18.763984+0000","last_clean":"2023-09-23T15:36:18.763984+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:36:18.763984+0000","last_undegraded":"2023-09-23T15:36:18.763984+0000","last_fullsized":"2023-09-23T15:36:18.763984+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767499+0000","last_change":"2023-09-23T15:35:28.690359+0000","last_active":"2023-09-23T15:36:18.767499+0000","last_peered":"2023-09-23T15:36:18.767499+0000","last_clean":"2023-09-23T15:36:18.767499+0000","last_became_active":"2023-09-23T15:35:28.689948+0000","last_became_peered":"2023-09-23T15:35:28.689948+0000","last_unstale":"2023-09-23T15:36:18.767499+0000","last_undegraded":"2023-09-23T15:36:18.767499+0000","last_fullsized":"2023-09-23T15:36:18.767499+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:25.640800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":847,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390237+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:36:19.390237+0000","last_peered":"2023-09-23T15:36:19.390237+0000","last_clean":"2023-09-23T15:36:19.390237+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:36:19.390237+0000","last_undegraded":"2023-09-23T15:36:19.390237+0000","last_fullsized":"2023-09-23T15:36:19.390237+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258733+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:36:20.258733+0000","last_peered":"2023-09-23T15:36:20.258733+0000","last_clean":"2023-09-23T15:36:20.258733+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:36:20.258733+0000","last_undegraded":"2023-09-23T15:36:20.258733+0000","last_fullsized":"2023-09-23T15:36:20.258733+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765004+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:36:18.765004+0000","last_peered":"2023-09-23T15:36:18.765004+0000","last_clean":"2023-09-23T15:36:18.765004+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:36:18.765004+0000","last_undegraded":"2023-09-23T15:36:18.765004+0000","last_fullsized":"2023-09-23T15:36:18.765004+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":562,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766918+0000","last_change":"2023-09-23T15:35:31.710854+0000","last_active":"2023-09-23T15:36:18.766918+0000","last_peered":"2023-09-23T15:36:18.766918+0000","last_clean":"2023-09-23T15:36:18.766918+0000","last_became_active":"2023-09-23T15:35:31.710465+0000","last_became_peered":"2023-09-23T15:35:31.710465+0000","last_unstale":"2023-09-23T15:36:18.766918+0000","last_undegraded":"2023-09-23T15:36:18.766918+0000","last_fullsized":"2023-09-23T15:36:18.766918+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:09:24.612228+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":494,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.770883+0000","last_change":"2023-09-23T15:35:28.699648+0000","last_active":"2023-09-23T15:36:18.770883+0000","last_peered":"2023-09-23T15:36:18.770883+0000","last_clean":"2023-09-23T15:36:18.770883+0000","last_became_active":"2023-09-23T15:35:28.686841+0000","last_became_peered":"2023-09-23T15:35:28.686841+0000","last_unstale":"2023-09-23T15:36:18.770883+0000","last_undegraded":"2023-09-23T15:36:18.770883+0000","last_fullsized":"2023-09-23T15:36:18.770883+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:20.963381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":440,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920558+0000","last_change":"2023-09-23T15:35:52.697013+0000","last_active":"2023-09-23T15:36:18.920558+0000","last_peered":"2023-09-23T15:36:18.920558+0000","last_clean":"2023-09-23T15:36:18.920558+0000","last_became_active":"2023-09-23T15:35:52.696731+0000","last_became_peered":"2023-09-23T15:35:52.696731+0000","last_unstale":"2023-09-23T15:36:18.920558+0000","last_undegraded":"2023-09-23T15:36:18.920558+0000","last_fullsized":"2023-09-23T15:36:18.920558+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:32:19.949603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":814,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259102+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:36:20.259102+0000","last_peered":"2023-09-23T15:36:20.259102+0000","last_clean":"2023-09-23T15:36:20.259102+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:36:20.259102+0000","last_undegraded":"2023-09-23T15:36:20.259102+0000","last_fullsized":"2023-09-23T15:36:20.259102+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":257,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259471+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:36:20.259471+0000","last_peered":"2023-09-23T15:36:20.259471+0000","last_clean":"2023-09-23T15:36:20.259471+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:36:20.259471+0000","last_undegraded":"2023-09-23T15:36:20.259471+0000","last_fullsized":"2023-09-23T15:36:20.259471+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766577+0000","last_change":"2023-09-23T15:35:28.691240+0000","last_active":"2023-09-23T15:36:18.766577+0000","last_peered":"2023-09-23T15:36:18.766577+0000","last_clean":"2023-09-23T15:36:18.766577+0000","last_became_active":"2023-09-23T15:35:28.690862+0000","last_became_peered":"2023-09-23T15:35:28.690862+0000","last_unstale":"2023-09-23T15:36:18.766577+0000","last_undegraded":"2023-09-23T15:36:18.766577+0000","last_fullsized":"2023-09-23T15:36:18.766577+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:41.052218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":761,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920257+0000","last_change":"2023-09-23T15:35:52.691576+0000","last_active":"2023-09-23T15:36:18.920257+0000","last_peered":"2023-09-23T15:36:18.920257+0000","last_clean":"2023-09-23T15:36:18.920257+0000","last_became_active":"2023-09-23T15:35:52.691182+0000","last_became_peered":"2023-09-23T15:35:52.691182+0000","last_unstale":"2023-09-23T15:36:18.920257+0000","last_undegraded":"2023-09-23T15:36:18.920257+0000","last_fullsized":"2023-09-23T15:36:18.920257+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:32.354840+0000","scrub_duration":0.08250528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":257,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258802+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:36:20.258802+0000","last_peered":"2023-09-23T15:36:20.258802+0000","last_clean":"2023-09-23T15:36:20.258802+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:36:20.258802+0000","last_undegraded":"2023-09-23T15:36:20.258802+0000","last_fullsized":"2023-09-23T15:36:20.258802+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258788+0000","last_change":"2023-09-23T15:33:30.765181+0000","last_active":"2023-09-23T15:36:20.258788+0000","last_peered":"2023-09-23T15:36:20.258788+0000","last_clean":"2023-09-23T15:36:20.258788+0000","last_became_active":"2023-09-23T15:33:30.764802+0000","last_became_peered":"2023-09-23T15:33:30.764802+0000","last_unstale":"2023-09-23T15:36:20.258788+0000","last_undegraded":"2023-09-23T15:36:20.258788+0000","last_fullsized":"2023-09-23T15:36:20.258788+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:36:15.536014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":805,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771619+0000","last_change":"2023-09-23T15:35:31.703608+0000","last_active":"2023-09-23T15:36:18.771619+0000","last_peered":"2023-09-23T15:36:18.771619+0000","last_clean":"2023-09-23T15:36:18.771619+0000","last_became_active":"2023-09-23T15:35:31.703159+0000","last_became_peered":"2023-09-23T15:35:31.703159+0000","last_unstale":"2023-09-23T15:36:18.771619+0000","last_undegraded":"2023-09-23T15:36:18.771619+0000","last_fullsized":"2023-09-23T15:36:18.771619+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:46.705865+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":273,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258897+0000","last_change":"2023-09-23T15:35:28.702405+0000","last_active":"2023-09-23T15:36:20.258897+0000","last_peered":"2023-09-23T15:36:20.258897+0000","last_clean":"2023-09-23T15:36:20.258897+0000","last_became_active":"2023-09-23T15:35:28.689620+0000","last_became_peered":"2023-09-23T15:35:28.689620+0000","last_unstale":"2023-09-23T15:36:20.258897+0000","last_undegraded":"2023-09-23T15:36:20.258897+0000","last_fullsized":"2023-09-23T15:36:20.258897+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:04.339529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":498,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771072+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:36:18.771072+0000","last_peered":"2023-09-23T15:36:18.771072+0000","last_clean":"2023-09-23T15:36:18.771072+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:36:18.771072+0000","last_undegraded":"2023-09-23T15:36:18.771072+0000","last_fullsized":"2023-09-23T15:36:18.771072+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":735,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260255+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:36:20.260255+0000","last_peered":"2023-09-23T15:36:20.260255+0000","last_clean":"2023-09-23T15:36:20.260255+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:36:20.260255+0000","last_undegraded":"2023-09-23T15:36:20.260255+0000","last_fullsized":"2023-09-23T15:36:20.260255+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767307+0000","last_change":"2023-09-23T15:35:28.685014+0000","last_active":"2023-09-23T15:36:18.767307+0000","last_peered":"2023-09-23T15:36:18.767307+0000","last_clean":"2023-09-23T15:36:18.767307+0000","last_became_active":"2023-09-23T15:35:28.684585+0000","last_became_peered":"2023-09-23T15:35:28.684585+0000","last_unstale":"2023-09-23T15:36:18.767307+0000","last_undegraded":"2023-09-23T15:36:18.767307+0000","last_fullsized":"2023-09-23T15:36:18.767307+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:52.512630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":522,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764395+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:36:18.764395+0000","last_peered":"2023-09-23T15:36:18.764395+0000","last_clean":"2023-09-23T15:36:18.764395+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:36:18.764395+0000","last_undegraded":"2023-09-23T15:36:18.764395+0000","last_fullsized":"2023-09-23T15:36:18.764395+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":770,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258549+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:36:20.258549+0000","last_peered":"2023-09-23T15:36:20.258549+0000","last_clean":"2023-09-23T15:36:20.258549+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:36:20.258549+0000","last_undegraded":"2023-09-23T15:36:20.258549+0000","last_fullsized":"2023-09-23T15:36:20.258549+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":436,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919963+0000","last_change":"2023-09-23T15:35:52.692066+0000","last_active":"2023-09-23T15:36:18.919963+0000","last_peered":"2023-09-23T15:36:18.919963+0000","last_clean":"2023-09-23T15:36:18.919963+0000","last_became_active":"2023-09-23T15:35:52.691748+0000","last_became_peered":"2023-09-23T15:35:52.691748+0000","last_unstale":"2023-09-23T15:36:18.919963+0000","last_undegraded":"2023-09-23T15:36:18.919963+0000","last_fullsized":"2023-09-23T15:36:18.919963+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:42.388322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257975+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:36:20.257975+0000","last_peered":"2023-09-23T15:36:20.257975+0000","last_clean":"2023-09-23T15:36:20.257975+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:36:20.257975+0000","last_undegraded":"2023-09-23T15:36:20.257975+0000","last_fullsized":"2023-09-23T15:36:20.257975+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":633,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766903+0000","last_change":"2023-09-23T15:35:31.701840+0000","last_active":"2023-09-23T15:36:18.766903+0000","last_peered":"2023-09-23T15:36:18.766903+0000","last_clean":"2023-09-23T15:36:18.766903+0000","last_became_active":"2023-09-23T15:35:31.701439+0000","last_became_peered":"2023-09-23T15:35:31.701439+0000","last_unstale":"2023-09-23T15:36:18.766903+0000","last_undegraded":"2023-09-23T15:36:18.766903+0000","last_fullsized":"2023-09-23T15:36:18.766903+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:05.479226+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.479226+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:16.177908+0000","scrub_duration":0.0055363749999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":485,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771115+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:36:18.771115+0000","last_peered":"2023-09-23T15:36:18.771115+0000","last_clean":"2023-09-23T15:36:18.771115+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:36:18.771115+0000","last_undegraded":"2023-09-23T15:36:18.771115+0000","last_fullsized":"2023-09-23T15:36:18.771115+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258006+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:36:20.258006+0000","last_peered":"2023-09-23T15:36:20.258006+0000","last_clean":"2023-09-23T15:36:20.258006+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:36:20.258006+0000","last_undegraded":"2023-09-23T15:36:20.258006+0000","last_fullsized":"2023-09-23T15:36:20.258006+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":834,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390139+0000","last_change":"2023-09-23T15:35:31.709834+0000","last_active":"2023-09-23T15:36:19.390139+0000","last_peered":"2023-09-23T15:36:19.390139+0000","last_clean":"2023-09-23T15:36:19.390139+0000","last_became_active":"2023-09-23T15:35:31.709429+0000","last_became_peered":"2023-09-23T15:35:31.709429+0000","last_unstale":"2023-09-23T15:36:19.390139+0000","last_undegraded":"2023-09-23T15:36:19.390139+0000","last_fullsized":"2023-09-23T15:36:19.390139+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:06.567918+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258881+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:36:20.258881+0000","last_peered":"2023-09-23T15:36:20.258881+0000","last_clean":"2023-09-23T15:36:20.258881+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:36:20.258881+0000","last_undegraded":"2023-09-23T15:36:20.258881+0000","last_fullsized":"2023-09-23T15:36:20.258881+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260275+0000","last_change":"2023-09-23T15:32:16.253887+0000","last_active":"2023-09-23T15:36:20.260275+0000","last_peered":"2023-09-23T15:36:20.260275+0000","last_clean":"2023-09-23T15:36:20.260275+0000","last_became_active":"2023-09-23T15:32:16.253556+0000","last_became_peered":"2023-09-23T15:32:16.253556+0000","last_unstale":"2023-09-23T15:36:20.260275+0000","last_undegraded":"2023-09-23T15:36:20.260275+0000","last_fullsized":"2023-09-23T15:36:20.260275+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:16.930439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":628,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766808+0000","last_change":"2023-09-23T15:35:31.705105+0000","last_active":"2023-09-23T15:36:18.766808+0000","last_peered":"2023-09-23T15:36:18.766808+0000","last_clean":"2023-09-23T15:36:18.766808+0000","last_became_active":"2023-09-23T15:35:31.704697+0000","last_became_peered":"2023-09-23T15:35:31.704697+0000","last_unstale":"2023-09-23T15:36:18.766808+0000","last_undegraded":"2023-09-23T15:36:18.766808+0000","last_fullsized":"2023-09-23T15:36:18.766808+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:30.357860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:30.357860+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:20.163569+0000","scrub_duration":0.0015641730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919814+0000","last_change":"2023-09-23T15:35:52.695833+0000","last_active":"2023-09-23T15:36:18.919814+0000","last_peered":"2023-09-23T15:36:18.919814+0000","last_clean":"2023-09-23T15:36:18.919814+0000","last_became_active":"2023-09-23T15:35:52.695109+0000","last_became_peered":"2023-09-23T15:35:52.695109+0000","last_unstale":"2023-09-23T15:36:18.919814+0000","last_undegraded":"2023-09-23T15:36:18.919814+0000","last_fullsized":"2023-09-23T15:36:18.919814+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:39.595373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390622+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:36:19.390622+0000","last_peered":"2023-09-23T15:36:19.390622+0000","last_clean":"2023-09-23T15:36:19.390622+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:36:19.390622+0000","last_undegraded":"2023-09-23T15:36:19.390622+0000","last_fullsized":"2023-09-23T15:36:19.390622+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":758,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771636+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:36:18.771636+0000","last_peered":"2023-09-23T15:36:18.771636+0000","last_clean":"2023-09-23T15:36:18.771636+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:36:18.771636+0000","last_undegraded":"2023-09-23T15:36:18.771636+0000","last_fullsized":"2023-09-23T15:36:18.771636+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"488'163","reported_seq":1039,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390584+0000","last_change":"2023-09-23T15:34:54.709256+0000","last_active":"2023-09-23T15:36:19.390584+0000","last_peered":"2023-09-23T15:36:19.390584+0000","last_clean":"2023-09-23T15:36:19.390584+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:36:19.390584+0000","last_undegraded":"2023-09-23T15:36:19.390584+0000","last_fullsized":"2023-09-23T15:36:19.390584+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021979399999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":726,"num_read_kb":1976,"num_write":174,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767382+0000","last_change":"2023-09-23T15:35:28.684619+0000","last_active":"2023-09-23T15:36:18.767382+0000","last_peered":"2023-09-23T15:36:18.767382+0000","last_clean":"2023-09-23T15:36:18.767382+0000","last_became_active":"2023-09-23T15:35:28.684056+0000","last_became_peered":"2023-09-23T15:35:28.684056+0000","last_unstale":"2023-09-23T15:36:18.767382+0000","last_undegraded":"2023-09-23T15:36:18.767382+0000","last_fullsized":"2023-09-23T15:36:18.767382+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:46.348436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":501,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039013+0000","last_change":"2023-09-23T15:35:28.684478+0000","last_active":"2023-09-23T15:36:19.039013+0000","last_peered":"2023-09-23T15:36:19.039013+0000","last_clean":"2023-09-23T15:36:19.039013+0000","last_became_active":"2023-09-23T15:35:28.684083+0000","last_became_peered":"2023-09-23T15:35:28.684083+0000","last_unstale":"2023-09-23T15:36:19.039013+0000","last_undegraded":"2023-09-23T15:36:19.039013+0000","last_fullsized":"2023-09-23T15:36:19.039013+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.700819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":626,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766778+0000","last_change":"2023-09-23T15:35:31.708912+0000","last_active":"2023-09-23T15:36:18.766778+0000","last_peered":"2023-09-23T15:36:18.766778+0000","last_clean":"2023-09-23T15:36:18.766778+0000","last_became_active":"2023-09-23T15:35:31.708400+0000","last_became_peered":"2023-09-23T15:35:31.708400+0000","last_unstale":"2023-09-23T15:36:18.766778+0000","last_undegraded":"2023-09-23T15:36:18.766778+0000","last_fullsized":"2023-09-23T15:36:18.766778+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.797446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.797446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:22.876441+0000","scrub_duration":0.0090775909999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"488'157","reported_seq":1115,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766829+0000","last_change":"2023-09-23T15:35:31.711290+0000","last_active":"2023-09-23T15:36:18.766829+0000","last_peered":"2023-09-23T15:36:18.766829+0000","last_clean":"2023-09-23T15:36:18.766829+0000","last_became_active":"2023-09-23T15:35:31.710579+0000","last_became_peered":"2023-09-23T15:35:31.710579+0000","last_unstale":"2023-09-23T15:36:18.766829+0000","last_undegraded":"2023-09-23T15:36:18.766829+0000","last_fullsized":"2023-09-23T15:36:18.766829+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:22.204523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033387800000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1377,"num_read_kb":7709,"num_write":130,"num_write_kb":5863,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258349+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:36:20.258349+0000","last_peered":"2023-09-23T15:36:20.258349+0000","last_clean":"2023-09-23T15:36:20.258349+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:36:20.258349+0000","last_undegraded":"2023-09-23T15:36:20.258349+0000","last_fullsized":"2023-09-23T15:36:20.258349+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":420,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390395+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:36:19.390395+0000","last_peered":"2023-09-23T15:36:19.390395+0000","last_clean":"2023-09-23T15:36:19.390395+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:36:19.390395+0000","last_undegraded":"2023-09-23T15:36:19.390395+0000","last_fullsized":"2023-09-23T15:36:19.390395+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":881,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258518+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:36:20.258518+0000","last_peered":"2023-09-23T15:36:20.258518+0000","last_clean":"2023-09-23T15:36:20.258518+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:36:20.258518+0000","last_undegraded":"2023-09-23T15:36:20.258518+0000","last_fullsized":"2023-09-23T15:36:20.258518+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"488'400","reported_seq":1944,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.038836+0000","last_change":"2023-09-23T15:34:54.705587+0000","last_active":"2023-09-23T15:36:19.038836+0000","last_peered":"2023-09-23T15:36:19.038836+0000","last_clean":"2023-09-23T15:36:19.038836+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:36:19.038836+0000","last_undegraded":"2023-09-23T15:36:19.038836+0000","last_fullsized":"2023-09-23T15:36:19.038836+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047212799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2483,"num_read_kb":3714,"num_write":351,"num_write_kb":5627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":416,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779043+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:36:21.779043+0000","last_peered":"2023-09-23T15:36:21.779043+0000","last_clean":"2023-09-23T15:36:21.779043+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:36:21.779043+0000","last_undegraded":"2023-09-23T15:36:21.779043+0000","last_fullsized":"2023-09-23T15:36:21.779043+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":442,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779190+0000","last_change":"2023-09-23T15:35:28.685371+0000","last_active":"2023-09-23T15:36:21.779190+0000","last_peered":"2023-09-23T15:36:21.779190+0000","last_clean":"2023-09-23T15:36:21.779190+0000","last_became_active":"2023-09-23T15:35:28.685039+0000","last_became_peered":"2023-09-23T15:35:28.685039+0000","last_unstale":"2023-09-23T15:36:21.779190+0000","last_undegraded":"2023-09-23T15:36:21.779190+0000","last_fullsized":"2023-09-23T15:36:21.779190+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:35.106519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"488'306","reported_seq":1008,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766858+0000","last_change":"2023-09-23T15:35:31.706131+0000","last_active":"2023-09-23T15:36:18.766858+0000","last_peered":"2023-09-23T15:36:18.766858+0000","last_clean":"2023-09-23T15:36:18.766858+0000","last_became_active":"2023-09-23T15:35:31.705360+0000","last_became_peered":"2023-09-23T15:35:31.705360+0000","last_unstale":"2023-09-23T15:36:18.766858+0000","last_undegraded":"2023-09-23T15:36:18.766858+0000","last_fullsized":"2023-09-23T15:36:18.766858+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:37.831256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022811199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1196,"num_read_kb":2267,"num_write":287,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":824,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258479+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:36:20.258479+0000","last_peered":"2023-09-23T15:36:20.258479+0000","last_clean":"2023-09-23T15:36:20.258479+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:36:20.258479+0000","last_undegraded":"2023-09-23T15:36:20.258479+0000","last_fullsized":"2023-09-23T15:36:20.258479+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":364,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919952+0000","last_change":"2023-09-23T15:35:52.695806+0000","last_active":"2023-09-23T15:36:18.919952+0000","last_peered":"2023-09-23T15:36:18.919952+0000","last_clean":"2023-09-23T15:36:18.919952+0000","last_became_active":"2023-09-23T15:35:52.695503+0000","last_became_peered":"2023-09-23T15:35:52.695503+0000","last_unstale":"2023-09-23T15:36:18.919952+0000","last_undegraded":"2023-09-23T15:36:18.919952+0000","last_fullsized":"2023-09-23T15:36:18.919952+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:22.363061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":510,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771475+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:36:18.771475+0000","last_peered":"2023-09-23T15:36:18.771475+0000","last_clean":"2023-09-23T15:36:18.771475+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:36:18.771475+0000","last_undegraded":"2023-09-23T15:36:18.771475+0000","last_fullsized":"2023-09-23T15:36:18.771475+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":864,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390272+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:36:19.390272+0000","last_peered":"2023-09-23T15:36:19.390272+0000","last_clean":"2023-09-23T15:36:19.390272+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:36:19.390272+0000","last_undegraded":"2023-09-23T15:36:19.390272+0000","last_fullsized":"2023-09-23T15:36:19.390272+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"488'240","reported_seq":648,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258128+0000","last_change":"2023-09-23T15:34:54.708597+0000","last_active":"2023-09-23T15:36:20.258128+0000","last_peered":"2023-09-23T15:36:20.258128+0000","last_clean":"2023-09-23T15:36:20.258128+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:36:20.258128+0000","last_undegraded":"2023-09-23T15:36:20.258128+0000","last_fullsized":"2023-09-23T15:36:20.258128+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013449600000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1148,"num_read_kb":1983,"num_write":238,"num_write_kb":6264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":526,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763939+0000","last_change":"2023-09-23T15:35:10.809299+0000","last_active":"2023-09-23T15:36:18.763939+0000","last_peered":"2023-09-23T15:36:18.763939+0000","last_clean":"2023-09-23T15:36:18.763939+0000","last_became_active":"2023-09-23T15:35:10.808955+0000","last_became_peered":"2023-09-23T15:35:10.808955+0000","last_unstale":"2023-09-23T15:36:18.763939+0000","last_undegraded":"2023-09-23T15:36:18.763939+0000","last_fullsized":"2023-09-23T15:36:18.763939+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:53.650771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":501,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764845+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:36:18.764845+0000","last_peered":"2023-09-23T15:36:18.764845+0000","last_clean":"2023-09-23T15:36:18.764845+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:36:18.764845+0000","last_undegraded":"2023-09-23T15:36:18.764845+0000","last_fullsized":"2023-09-23T15:36:18.764845+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":844,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771484+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:36:18.771484+0000","last_peered":"2023-09-23T15:36:18.771484+0000","last_clean":"2023-09-23T15:36:18.771484+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:36:18.771484+0000","last_undegraded":"2023-09-23T15:36:18.771484+0000","last_fullsized":"2023-09-23T15:36:18.771484+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"488'207","reported_seq":1472,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258952+0000","last_change":"2023-09-23T15:34:54.709815+0000","last_active":"2023-09-23T15:36:20.258952+0000","last_peered":"2023-09-23T15:36:20.258952+0000","last_clean":"2023-09-23T15:36:20.258952+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:36:20.258952+0000","last_undegraded":"2023-09-23T15:36:20.258952+0000","last_fullsized":"2023-09-23T15:36:20.258952+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":207,"log_dups_size":0,"ondisk_log_size":207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038879700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1613,"num_read_kb":2033,"num_write":161,"num_write_kb":409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":418,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390516+0000","last_change":"2023-09-23T15:35:28.685172+0000","last_active":"2023-09-23T15:36:19.390516+0000","last_peered":"2023-09-23T15:36:19.390516+0000","last_clean":"2023-09-23T15:36:19.390516+0000","last_became_active":"2023-09-23T15:35:28.684751+0000","last_became_peered":"2023-09-23T15:35:28.684751+0000","last_unstale":"2023-09-23T15:36:19.390516+0000","last_undegraded":"2023-09-23T15:36:19.390516+0000","last_fullsized":"2023-09-23T15:36:19.390516+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.436320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764621+0000","last_change":"2023-09-23T15:35:28.690179+0000","last_active":"2023-09-23T15:36:18.764621+0000","last_peered":"2023-09-23T15:36:18.764621+0000","last_clean":"2023-09-23T15:36:18.764621+0000","last_became_active":"2023-09-23T15:35:28.689871+0000","last_became_peered":"2023-09-23T15:35:28.689871+0000","last_unstale":"2023-09-23T15:36:18.764621+0000","last_undegraded":"2023-09-23T15:36:18.764621+0000","last_fullsized":"2023-09-23T15:36:18.764621+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:27.075241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":809,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260233+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:36:20.260233+0000","last_peered":"2023-09-23T15:36:20.260233+0000","last_clean":"2023-09-23T15:36:20.260233+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:36:20.260233+0000","last_undegraded":"2023-09-23T15:36:20.260233+0000","last_fullsized":"2023-09-23T15:36:20.260233+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":292,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039201+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:36:19.039201+0000","last_peered":"2023-09-23T15:36:19.039201+0000","last_clean":"2023-09-23T15:36:19.039201+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:36:19.039201+0000","last_undegraded":"2023-09-23T15:36:19.039201+0000","last_fullsized":"2023-09-23T15:36:19.039201+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"488'320","reported_seq":1857,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779452+0000","last_change":"2023-09-23T15:34:54.710828+0000","last_active":"2023-09-23T15:36:21.779452+0000","last_peered":"2023-09-23T15:36:21.779452+0000","last_clean":"2023-09-23T15:36:21.779452+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:36:21.779452+0000","last_undegraded":"2023-09-23T15:36:21.779452+0000","last_fullsized":"2023-09-23T15:36:21.779452+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":320,"log_dups_size":0,"ondisk_log_size":320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011064320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2324,"num_read_kb":4368,"num_write":216,"num_write_kb":2345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":430,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920063+0000","last_change":"2023-09-23T15:35:52.696068+0000","last_active":"2023-09-23T15:36:18.920063+0000","last_peered":"2023-09-23T15:36:18.920063+0000","last_clean":"2023-09-23T15:36:18.920063+0000","last_became_active":"2023-09-23T15:35:52.695534+0000","last_became_peered":"2023-09-23T15:35:52.695534+0000","last_unstale":"2023-09-23T15:36:18.920063+0000","last_undegraded":"2023-09-23T15:36:18.920063+0000","last_fullsized":"2023-09-23T15:36:18.920063+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:08.843615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":768,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920163+0000","last_change":"2023-09-23T15:35:52.695854+0000","last_active":"2023-09-23T15:36:18.920163+0000","last_peered":"2023-09-23T15:36:18.920163+0000","last_clean":"2023-09-23T15:36:18.920163+0000","last_became_active":"2023-09-23T15:35:52.695506+0000","last_became_peered":"2023-09-23T15:35:52.695506+0000","last_unstale":"2023-09-23T15:36:18.920163+0000","last_undegraded":"2023-09-23T15:36:18.920163+0000","last_fullsized":"2023-09-23T15:36:18.920163+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:33.677784+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"488'157","reported_seq":1130,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771724+0000","last_change":"2023-09-23T15:34:54.714657+0000","last_active":"2023-09-23T15:36:18.771724+0000","last_peered":"2023-09-23T15:36:18.771724+0000","last_clean":"2023-09-23T15:36:18.771724+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:36:18.771724+0000","last_undegraded":"2023-09-23T15:36:18.771724+0000","last_fullsized":"2023-09-23T15:36:18.771724+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031768699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1077,"num_read_kb":2328,"num_write":133,"num_write_kb":2319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":560,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764916+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:36:18.764916+0000","last_peered":"2023-09-23T15:36:18.764916+0000","last_clean":"2023-09-23T15:36:18.764916+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:36:18.764916+0000","last_undegraded":"2023-09-23T15:36:18.764916+0000","last_fullsized":"2023-09-23T15:36:18.764916+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258967+0000","last_change":"2023-09-23T15:35:28.678721+0000","last_active":"2023-09-23T15:36:20.258967+0000","last_peered":"2023-09-23T15:36:20.258967+0000","last_clean":"2023-09-23T15:36:20.258967+0000","last_became_active":"2023-09-23T15:35:28.678381+0000","last_became_peered":"2023-09-23T15:35:28.678381+0000","last_unstale":"2023-09-23T15:36:20.258967+0000","last_undegraded":"2023-09-23T15:36:20.258967+0000","last_fullsized":"2023-09-23T15:36:20.258967+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:45.972251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":675,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919838+0000","last_change":"2023-09-23T15:35:52.691580+0000","last_active":"2023-09-23T15:36:18.919838+0000","last_peered":"2023-09-23T15:36:18.919838+0000","last_clean":"2023-09-23T15:36:18.919838+0000","last_became_active":"2023-09-23T15:35:52.691171+0000","last_became_peered":"2023-09-23T15:35:52.691171+0000","last_unstale":"2023-09-23T15:36:18.919838+0000","last_undegraded":"2023-09-23T15:36:18.919838+0000","last_fullsized":"2023-09-23T15:36:18.919838+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:32:33.268462+0000","scrub_duration":0.017140743999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":58,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.117788+0000","last_change":"2023-09-23T15:34:12.777336+0000","last_active":"2023-09-23T15:36:22.117788+0000","last_peered":"2023-09-23T15:36:22.117788+0000","last_clean":"2023-09-23T15:36:22.117788+0000","last_became_active":"2023-09-23T15:34:12.777101+0000","last_became_peered":"2023-09-23T15:34:12.777101+0000","last_unstale":"2023-09-23T15:36:22.117788+0000","last_undegraded":"2023-09-23T15:36:22.117788+0000","last_fullsized":"2023-09-23T15:36:22.117788+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:13:25.546836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":49,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039077+0000","last_change":"2023-09-23T15:34:44.672844+0000","last_active":"2023-09-23T15:36:19.039077+0000","last_peered":"2023-09-23T15:36:19.039077+0000","last_clean":"2023-09-23T15:36:19.039077+0000","last_became_active":"2023-09-23T15:34:44.672523+0000","last_became_peered":"2023-09-23T15:34:44.672523+0000","last_unstale":"2023-09-23T15:36:19.039077+0000","last_undegraded":"2023-09-23T15:36:19.039077+0000","last_fullsized":"2023-09-23T15:36:19.039077+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:26.341913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":644,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920282+0000","last_change":"2023-09-23T15:35:52.694927+0000","last_active":"2023-09-23T15:36:18.920282+0000","last_peered":"2023-09-23T15:36:18.920282+0000","last_clean":"2023-09-23T15:36:18.920282+0000","last_became_active":"2023-09-23T15:35:52.694716+0000","last_became_peered":"2023-09-23T15:35:52.694716+0000","last_unstale":"2023-09-23T15:36:18.920282+0000","last_undegraded":"2023-09-23T15:36:18.920282+0000","last_fullsized":"2023-09-23T15:36:18.920282+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:12.464049+0000","scrub_duration":0.011035615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":34,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767205+0000","last_change":"2023-09-23T15:35:28.693370+0000","last_active":"2023-09-23T15:36:18.767205+0000","last_peered":"2023-09-23T15:36:18.767205+0000","last_clean":"2023-09-23T15:36:18.767205+0000","last_became_active":"2023-09-23T15:35:28.686120+0000","last_became_peered":"2023-09-23T15:35:28.686120+0000","last_unstale":"2023-09-23T15:36:18.767205+0000","last_undegraded":"2023-09-23T15:36:18.767205+0000","last_fullsized":"2023-09-23T15:36:18.767205+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:19:19.106941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258223+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:36:20.258223+0000","last_peered":"2023-09-23T15:36:20.258223+0000","last_clean":"2023-09-23T15:36:20.258223+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:36:20.258223+0000","last_undegraded":"2023-09-23T15:36:20.258223+0000","last_fullsized":"2023-09-23T15:36:20.258223+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":34,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767625+0000","last_change":"2023-09-23T15:35:28.696024+0000","last_active":"2023-09-23T15:36:18.767625+0000","last_peered":"2023-09-23T15:36:18.767625+0000","last_clean":"2023-09-23T15:36:18.767625+0000","last_became_active":"2023-09-23T15:35:28.690465+0000","last_became_peered":"2023-09-23T15:35:28.690465+0000","last_unstale":"2023-09-23T15:36:18.767625+0000","last_undegraded":"2023-09-23T15:36:18.767625+0000","last_fullsized":"2023-09-23T15:36:18.767625+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:20.906445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259089+0000","last_change":"2023-09-23T15:35:10.809180+0000","last_active":"2023-09-23T15:36:20.259089+0000","last_peered":"2023-09-23T15:36:20.259089+0000","last_clean":"2023-09-23T15:36:20.259089+0000","last_became_active":"2023-09-23T15:35:10.808819+0000","last_became_peered":"2023-09-23T15:35:10.808819+0000","last_unstale":"2023-09-23T15:36:20.259089+0000","last_undegraded":"2023-09-23T15:36:20.259089+0000","last_fullsized":"2023-09-23T15:36:20.259089+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:22.677054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":705,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258804+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:36:20.258804+0000","last_peered":"2023-09-23T15:36:20.258804+0000","last_clean":"2023-09-23T15:36:20.258804+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:36:20.258804+0000","last_undegraded":"2023-09-23T15:36:20.258804+0000","last_fullsized":"2023-09-23T15:36:20.258804+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":421,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920492+0000","last_change":"2023-09-23T15:35:52.693192+0000","last_active":"2023-09-23T15:36:18.920492+0000","last_peered":"2023-09-23T15:36:18.920492+0000","last_clean":"2023-09-23T15:36:18.920492+0000","last_became_active":"2023-09-23T15:35:52.692840+0000","last_became_peered":"2023-09-23T15:35:52.692840+0000","last_unstale":"2023-09-23T15:36:18.920492+0000","last_undegraded":"2023-09-23T15:36:18.920492+0000","last_fullsized":"2023-09-23T15:36:18.920492+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:19.258799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":419,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920011+0000","last_change":"2023-09-23T15:35:52.692698+0000","last_active":"2023-09-23T15:36:18.920011+0000","last_peered":"2023-09-23T15:36:18.920011+0000","last_clean":"2023-09-23T15:36:18.920011+0000","last_became_active":"2023-09-23T15:35:52.692407+0000","last_became_peered":"2023-09-23T15:35:52.692407+0000","last_unstale":"2023-09-23T15:36:18.920011+0000","last_undegraded":"2023-09-23T15:36:18.920011+0000","last_fullsized":"2023-09-23T15:36:18.920011+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:44.027672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":700,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390321+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:36:19.390321+0000","last_peered":"2023-09-23T15:36:19.390321+0000","last_clean":"2023-09-23T15:36:19.390321+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:36:19.390321+0000","last_undegraded":"2023-09-23T15:36:19.390321+0000","last_fullsized":"2023-09-23T15:36:19.390321+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":505,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765199+0000","last_change":"2023-09-23T15:35:28.693557+0000","last_active":"2023-09-23T15:36:18.765199+0000","last_peered":"2023-09-23T15:36:18.765199+0000","last_clean":"2023-09-23T15:36:18.765199+0000","last_became_active":"2023-09-23T15:35:28.685966+0000","last_became_peered":"2023-09-23T15:35:28.685966+0000","last_unstale":"2023-09-23T15:36:18.765199+0000","last_undegraded":"2023-09-23T15:36:18.765199+0000","last_fullsized":"2023-09-23T15:36:18.765199+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:45.228139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257819+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:36:20.257819+0000","last_peered":"2023-09-23T15:36:20.257819+0000","last_clean":"2023-09-23T15:36:20.257819+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:36:20.257819+0000","last_undegraded":"2023-09-23T15:36:20.257819+0000","last_fullsized":"2023-09-23T15:36:20.257819+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":11022,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11944,"num_read_kb":26378,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":161,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768817,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":150444,"kb_used_data":9764,"kb_used_omap":13,"kb_used_meta":140658,"kb_avail":93594708,"statfs":{"total":95995035648,"available":95840980992,"internally_reserved":0,"allocated":9998336,"data_stored":8451230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":144033886},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":4.6799999999999997,"15min":5.3410000000000002},"min":{"1min":0.29999999999999999,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":0.70399999999999996,"5min":121.012,"15min":121.012},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.51200000000000001,"5min":4.4299999999999997,"15min":5.2939999999999996},"min":{"1min":0.34499999999999997,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.66800000000000004,"5min":120.97,"15min":120.97},"last":0.53200000000000003}]},{"osd":1,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":1.552,"15min":12.551},"min":{"1min":0.29799999999999999,"5min":0.26700000000000002,"15min":0.24399999999999999},"max":{"1min":0.59999999999999998,"5min":10.093999999999999,"15min":157.096},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":1.238,"15min":12.478},"min":{"1min":0.30299999999999999,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.748,"5min":9.9740000000000002,"15min":157.011},"last":0.61799999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":7.6769999999999996,"15min":3.3570000000000002},"min":{"1min":0.312,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.68899999999999995,"5min":220.875,"15min":220.875},"last":0.5},{"interface":"front","average":{"1min":0.49299999999999999,"5min":7.4340000000000002,"15min":3.3140000000000001},"min":{"1min":0.33500000000000002,"5min":0.247,"15min":0.247},"max":{"1min":0.71099999999999997,"5min":220.84399999999999,"15min":220.84399999999999},"last":0.72299999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.5760000000000001,"15min":3.1589999999999998},"min":{"1min":0.32700000000000001,"5min":0.30099999999999999,"15min":0.25900000000000001},"max":{"1min":0.72499999999999998,"5min":10.061,"15min":76.019000000000005},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.792,"15min":3.5009999999999999},"min":{"1min":0.34799999999999998,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":0.66000000000000003,"5min":12.606999999999999,"15min":76.317999999999998},"last":0.64300000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.6419999999999999,"15min":1.619},"min":{"1min":0.28299999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65400000000000003,"5min":43.737000000000002,"15min":170.15799999999999},"last":0.63200000000000001},{"interface":"front","average":{"1min":0.504,"5min":1.8089999999999999,"15min":1.649},"min":{"1min":0.30399999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.74299999999999999,"5min":43.844000000000001,"15min":170.017},"last":0.70299999999999996}]},{"osd":5,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.92100000000000004,"15min":0.84699999999999998},"min":{"1min":0.35299999999999998,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":0.75900000000000001,"5min":7.2910000000000004,"15min":7.2910000000000004},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.58399999999999996,"5min":1.379,"15min":0.99299999999999999},"min":{"1min":0.375,"5min":0.35399999999999998,"15min":0.23300000000000001},"max":{"1min":0.76800000000000002,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":0.79100000000000004}]},{"osd":6,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":1.6000000000000001,"15min":2.0609999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.78100000000000003,"5min":45.136000000000003,"15min":262.38099999999997},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.57099999999999995,"5min":1.7210000000000001,"15min":2.0910000000000002},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.74299999999999999,"5min":44.945,"15min":261.488},"last":0.83899999999999997}]}]},{"osd":6,"up_from":304,"seq":1305670058160,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213728,"kb_used_data":5720,"kb_used_omap":13,"kb_used_meta":207986,"kb_avail":93531424,"statfs":{"total":95995035648,"available":95776178176,"internally_reserved":0,"allocated":5857280,"data_stored":4321791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":212978019},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.8839999999999999,"15min":3.2559999999999998},"min":{"1min":0.22500000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.88200000000000001,"5min":44.853999999999999,"15min":184.53200000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":3.4940000000000002,"15min":3.2480000000000002},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.97299999999999998,"5min":40.959000000000003,"15min":86.227999999999994},"last":0.625}]},{"osd":1,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.1970000000000001,"15min":2.2269999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.221},"max":{"1min":0.95099999999999996,"5min":7.907,"15min":86.534000000000006},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.1200000000000001,"15min":2.0510000000000002},"min":{"1min":0.318,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.95499999999999996,"5min":6.4909999999999997,"15min":86.144000000000005},"last":0.54100000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6779999999999999,"15min":2.0699999999999998},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":1.03,"5min":63.512,"15min":63.512},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.972,"15min":2.181},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":0.98799999999999999,"5min":63.526000000000003,"15min":63.526000000000003},"last":0.61799999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.3160000000000001,"15min":2.343},"min":{"1min":0.36099999999999999,"5min":0.30299999999999999,"15min":0.26800000000000002},"max":{"1min":1.006,"5min":11.176,"15min":142.67599999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.8280000000000001,"15min":2.7429999999999999},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":11.795,"15min":142.541},"last":0.55300000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.68700000000000006,"15min":1.421},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.23999999999999999},"max":{"1min":0.69499999999999995,"5min":4.3710000000000004,"15min":128.11600000000001},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.254,"15min":1.9319999999999999},"min":{"1min":0.36799999999999999,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.91800000000000004,"5min":8.2449999999999992,"15min":128.095},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.214,"15min":2.6190000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.248},"max":{"1min":0.74399999999999999,"5min":9.2560000000000002,"15min":130.91300000000001},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.73299999999999998,"15min":2.2149999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":1.0580000000000001,"5min":4.4400000000000004,"15min":130.92599999999999},"last":0.73999999999999999}]},{"osd":7,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.339,"15min":1.2809999999999999},"min":{"1min":0.373,"5min":0.34699999999999998,"15min":0.28000000000000003},"max":{"1min":0.93899999999999995,"5min":10.087,"15min":10.087},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.88300000000000001,"15min":1.2549999999999999},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.308},"max":{"1min":1.091,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.69899999999999995}]}]},{"osd":1,"up_from":183,"seq":785979015461,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700544,"kb_used_data":10616,"kb_used_omap":6,"kb_used_meta":689913,"kb_avail":93044608,"statfs":{"total":95995035648,"available":95277678592,"internally_reserved":0,"allocated":10870784,"data_stored":8342958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":706470987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":2.8759999999999999,"15min":3.3079999999999998},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.19400000000000001},"max":{"1min":0.44,"5min":67.236000000000004,"15min":101.27800000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.33100000000000002,"5min":2.9449999999999998,"15min":3.6059999999999999},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.221},"max":{"1min":0.45600000000000002,"5min":64.620999999999995,"15min":101.051},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":3.5099999999999998,"15min":3.5499999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.502,"5min":125.94,"15min":186.304},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.36699999999999999,"5min":3.1389999999999998,"15min":3.1840000000000002},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":0.46500000000000002,"5min":126.54600000000001,"15min":186.541},"last":0.41999999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.268,"15min":2.1160000000000001},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.22700000000000001},"max":{"1min":0.54300000000000004,"5min":11.896000000000001,"15min":64.540999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.2090000000000001,"15min":2.1360000000000001},"min":{"1min":0.35999999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.52600000000000002,"5min":13.98,"15min":67.465000000000003},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.367,"15min":2.2850000000000001},"min":{"1min":0.40600000000000003,"5min":0.374,"15min":0.32100000000000001},"max":{"1min":0.88100000000000001,"5min":8.5289999999999999,"15min":72.602000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.165,"15min":1.9139999999999999},"min":{"1min":0.41099999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77700000000000002,"5min":8.7729999999999997,"15min":73.454999999999998},"last":0.56299999999999994}]},{"osd":5,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.3979999999999999,"15min":2.984},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.312},"max":{"1min":0.94799999999999995,"5min":30.009,"15min":173.18799999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.764,"15min":3.3130000000000002},"min":{"1min":0.42299999999999999,"5min":0.34100000000000003,"15min":0.29099999999999998},"max":{"1min":0.79400000000000004,"5min":29.983000000000001,"15min":172.822},"last":0.50800000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":1.3759999999999999,"15min":2.4340000000000002},"min":{"1min":0.497,"5min":0.38100000000000001,"15min":0.34899999999999998},"max":{"1min":1.052,"5min":8.7170000000000005,"15min":54.198},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.2010000000000001,"15min":1.657},"min":{"1min":0.61199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82899999999999996,"5min":10.996,"15min":15.605},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":1.345,"15min":1.8029999999999999},"min":{"1min":0.48099999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.76500000000000001,"5min":11.032999999999999,"15min":11.032999999999999},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.65800000000000003,"5min":1.74,"15min":3.6139999999999999},"min":{"1min":0.47099999999999997,"5min":0.41899999999999998,"15min":0.40899999999999997},"max":{"1min":0.94899999999999995,"5min":8.5730000000000004,"15min":54.223999999999997},"last":0.61799999999999999}]}]},{"osd":0,"up_from":95,"seq":408021893507,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898684,"kb_used_data":14836,"kb_used_omap":8,"kb_used_meta":883831,"kb_avail":92846468,"statfs":{"total":95995035648,"available":95074783232,"internally_reserved":0,"allocated":15192064,"data_stored":12841351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":905043540},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.056,"15min":3.9609999999999999},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":0.61799999999999999,"5min":63.393000000000001,"15min":141.52199999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.376,"5min":1.9750000000000001,"15min":3.8879999999999999},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":0.66400000000000003,"5min":63.417999999999999,"15min":141.62799999999999},"last":0.46300000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":2.907,"15min":2.3639999999999999},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":0.48799999999999999,"5min":66.132999999999996,"15min":66.132999999999996},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.38700000000000001,"5min":3.258,"15min":2.3650000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.20300000000000001},"max":{"1min":0.68600000000000005,"5min":68.296999999999997,"15min":68.296999999999997},"last":0.36899999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.6269999999999998,"15min":4.2489999999999997},"min":{"1min":0.29399999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.59099999999999997,"5min":65.251999999999995,"15min":197.78899999999999},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":2.847,"15min":4.1779999999999999},"min":{"1min":0.36399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68999999999999995,"5min":65.269999999999996,"15min":200.58799999999999},"last":0.41299999999999998}]},{"osd":4,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":2.6080000000000001,"15min":2.2200000000000002},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.84899999999999998,"5min":60.771999999999998,"15min":60.771999999999998},"last":0.78800000000000003},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.512,"15min":2.1880000000000002},"min":{"1min":0.40899999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.746,"5min":60.725999999999999,"15min":60.725999999999999},"last":0.73099999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.915,"15min":2.056},"min":{"1min":0.44900000000000001,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":0.83699999999999997,"5min":33.158999999999999,"15min":44.57},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.016,"15min":2.3250000000000002},"min":{"1min":0.43099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":33.140000000000001,"15min":44.588000000000001},"last":0.81299999999999994}]},{"osd":6,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":2.6640000000000001,"15min":2.536},"min":{"1min":0.42999999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.91200000000000003,"5min":62.168999999999997,"15min":62.168999999999997},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.624,"5min":2.6880000000000002,"15min":2.7719999999999998},"min":{"1min":0.45100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81899999999999995,"5min":61.972000000000001,"15min":61.972000000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.9339999999999999,"15min":2.1829999999999998},"min":{"1min":0.51300000000000001,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.88800000000000001,"5min":12.954000000000001,"15min":36.848999999999997},"last":0.80100000000000005},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.952,"15min":3.2090000000000001},"min":{"1min":0.46899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83299999999999996,"5min":11.445,"15min":36.947000000000003},"last":0.71799999999999997}]}]},{"osd":2,"up_from":105,"seq":450971566457,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854840,"kb_used_data":10672,"kb_used_omap":11,"kb_used_meta":844148,"kb_avail":92890312,"statfs":{"total":95995035648,"available":95119679488,"internally_reserved":0,"allocated":10928128,"data_stored":8581201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":864408336},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":2.883,"15min":4.5609999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":0.56999999999999995,"5min":82.878,"15min":235.517},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.40200000000000002,"5min":2.855,"15min":4.484},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.64900000000000002,"5min":82.762,"15min":235.33799999999999},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:36:35 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.5589999999999999,"15min":2.7919999999999998},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.47999999999999998,"5min":11.468,"15min":71.819000000000003},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.38200000000000001,"5min":1.4810000000000001,"15min":2.77},"min":{"1min":0.28299999999999997,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.54600000000000004,"5min":20.048999999999999,"15min":71.805000000000007},"last":0.40400000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:36:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.355,"15min":3.0659999999999998},"min":{"1min":0.26400000000000001,"5min":0.22,"15min":0.216},"max":{"1min":0.93700000000000006,"5min":11.324999999999999,"15min":77.031999999999996},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.41199999999999998,"5min":1.5269999999999999,"15min":3.1200000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":0.66300000000000003,"5min":11.420999999999999,"15min":76.742000000000004},"last":0.39200000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.242,"15min":1.6719999999999999},"min":{"1min":0.39900000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79700000000000004,"5min":9.6210000000000004,"15min":63.747999999999998},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.355,"15min":1.831},"min":{"1min":0.40600000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.79100000000000004,"5min":9.3480000000000008,"15min":63.954000000000001},"last":0.46999999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.175,"15min":2.0739999999999998},"min":{"1min":0.45300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.92600000000000005,"5min":9.9559999999999995,"15min":107.021},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.204,"15min":2.0699999999999998},"min":{"1min":0.46600000000000003,"5min":0.31,"15min":0.31},"max":{"1min":0.77600000000000002,"5min":9.577,"15min":107.136},"last":0.44600000000000001}]},{"osd":6,"last update":"Sat Sep 23 15:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.266,"15min":2.2610000000000001},"min":{"1min":0.41699999999999998,"5min":0.318,"15min":0.318},"max":{"1min":1.006,"5min":10.51,"15min":156.16},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.1459999999999999,"15min":2.0710000000000002},"min":{"1min":0.48599999999999999,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":0.84299999999999997,"5min":9.2680000000000007,"15min":155.226},"last":0.53000000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:35:55 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.7270000000000001,"15min":2.556},"min":{"1min":0.45700000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.4199999999999999,"5min":12.282,"15min":112.70699999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":1.671,"15min":2.948},"min":{"1min":0.47499999999999998,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.3169999999999999,"5min":9.9280000000000008,"15min":109.402},"last":0.55900000000000005}]}]},{"osd":3,"up_from":110,"seq":472446402929,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866316,"kb_used_data":10756,"kb_used_omap":3,"kb_used_meta":855548,"kb_avail":92878836,"statfs":{"total":95995035648,"available":95107928064,"internally_reserved":0,"allocated":11014144,"data_stored":8647987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":876081321},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":2.6760000000000002,"15min":3.8170000000000002},"min":{"1min":0.26200000000000001,"5min":0.187,"15min":0.183},"max":{"1min":0.51300000000000001,"5min":51.253,"15min":141.934},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.375,"5min":3.2280000000000002,"15min":4.3179999999999996},"min":{"1min":0.29099999999999998,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":0.56499999999999995,"5min":53.731000000000002,"15min":141.72800000000001},"last":0.371}]},{"osd":1,"last update":"Sat Sep 23 15:36:34 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.7290000000000001,"15min":3.1949999999999998},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":0.76300000000000001,"5min":16.623000000000001,"15min":126.875},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.381,"15min":3.4289999999999998},"min":{"1min":0.28299999999999997,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":2.641,"5min":12.243,"15min":128.249},"last":0.48199999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":3.0089999999999999,"15min":2.8919999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.002,"5min":76.468999999999994,"15min":144.35599999999999},"last":0.443},{"interface":"front","average":{"1min":0.47299999999999998,"5min":3.5150000000000001,"15min":3.367},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.246},"max":{"1min":1.081,"5min":85.108000000000004,"15min":144.494},"last":0.45800000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":2.2160000000000002,"15min":1.8819999999999999},"min":{"1min":0.41099999999999998,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":0.69999999999999996,"5min":44.289000000000001,"15min":44.289000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.097,"15min":1.8049999999999999},"min":{"1min":0.38700000000000001,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.85599999999999998,"5min":44.055,"15min":44.055},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.9419999999999999,"15min":3.1320000000000001},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":0.82999999999999996,"5min":14.083,"15min":148.619},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.627,"5min":1.8620000000000001,"15min":2.9329999999999998},"min":{"1min":0.46899999999999997,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":0.81100000000000005,"5min":16.506,"15min":148.56999999999999},"last":0.68700000000000006}]},{"osd":6,"last update":"Sat Sep 23 15:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4550000000000001,"15min":2.714},"min":{"1min":0.436,"5min":0.373,"15min":0.32700000000000001},"max":{"1min":0.82499999999999996,"5min":44.356000000000002,"15min":84.665000000000006},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.64200000000000002,"5min":2.5369999999999999,"15min":2.988},"min":{"1min":0.5,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77000000000000002,"5min":44.453000000000003,"15min":84.897999999999996},"last":0.72599999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.514,"15min":7.5990000000000002},"min":{"1min":0.45100000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1160000000000001,"5min":9.3729999999999993,"15min":216.56899999999999},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":2.0670000000000002,"15min":8.5},"min":{"1min":0.51000000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81599999999999995,"5min":14.222,"15min":216.548},"last":0.70399999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986696,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525284,"kb_used_data":5404,"kb_used_omap":0,"kb_used_meta":519872,"kb_avail":93219868,"statfs":{"total":95995035648,"available":95457144832,"internally_reserved":0,"allocated":5533696,"data_stored":3994168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":532348928},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:40 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.3640000000000001,"15min":3.2280000000000002},"min":{"1min":0.32400000000000001,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":0.76900000000000002,"5min":28.805,"15min":162.75999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6120000000000001,"15min":3.2309999999999999},"min":{"1min":0.40000000000000002,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":1.0009999999999999,"5min":29.167000000000002,"15min":162.34800000000001},"last":0.54400000000000004}]},{"osd":1,"last update":"Sat Sep 23 15:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6160000000000001,"15min":3.5449999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":0.752,"5min":41.354999999999997,"15min":144.76900000000001},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.54500000000000004,"5min":1.478,"15min":3.609},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.25},"max":{"1min":0.86399999999999999,"5min":41.343000000000004,"15min":144.58799999999999},"last":0.47899999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:36:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":3.2909999999999999,"15min":2.419},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":0.94799999999999995,"5min":89.980000000000004,"15min":89.980000000000004},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.65500000000000003,"5min":3.278,"15min":2.492},"min":{"1min":0.48399999999999999,"5min":0.36499999999999999,"15min":0.30599999999999999},"max":{"1min":0.874,"5min":90.016999999999996,"15min":90.016999999999996},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 15:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.3109999999999999,"15min":2.3069999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":0.82299999999999995,"5min":9.7550000000000008,"15min":112.681},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.343,"15min":2.3069999999999999},"min":{"1min":0.39100000000000001,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":0.83599999999999997,"5min":7.6500000000000004,"15min":112.621},"last":0.435}]},{"osd":5,"last update":"Sat Sep 23 15:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.0289999999999999,"15min":1.222},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.221},"max":{"1min":0.82799999999999996,"5min":11.904999999999999,"15min":40.685000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":1.3320000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.246},"max":{"1min":0.81599999999999995,"5min":6.4089999999999998,"15min":38.439},"last":0.627}]},{"osd":6,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1439999999999999,"15min":2.0179999999999998},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.84499999999999997,"5min":25.437999999999999,"15min":228.93199999999999},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.345,"15min":2.2770000000000001},"min":{"1min":0.378,"5min":0.36499999999999999,"15min":0.29499999999999998},"max":{"1min":0.875,"5min":25.768999999999998,"15min":232.54900000000001},"last":0.71999999999999997}]},{"osd":7,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.93200000000000005,"15min":1.8440000000000001},"min":{"1min":0.52000000000000002,"5min":0.36199999999999999,"15min":0.36099999999999999},"max":{"1min":0.90200000000000002,"5min":5.2539999999999996,"15min":183.94800000000001},"last":0.67600000000000005},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.1100000000000001,"15min":1.742},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":0.871,"5min":7.5060000000000002,"15min":183.72399999999999},"last":0.72899999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725059,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":540380,"kb_used_data":5652,"kb_used_omap":0,"kb_used_meta":534720,"kb_avail":93204772,"statfs":{"total":95995035648,"available":95441686528,"internally_reserved":0,"allocated":5787648,"data_stored":4252619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":547553280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":2.8759999999999999,"15min":5.7649999999999997},"min":{"1min":0.35499999999999998,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":0.91100000000000003,"5min":116.307,"15min":397.84199999999998},"last":0.628},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.669,"15min":5.7279999999999998},"min":{"1min":0.33500000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0469999999999999,"5min":113.917,"15min":397.28100000000001},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.613,"15min":3.2639999999999998},"min":{"1min":0.33600000000000002,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":0.88400000000000001,"5min":30.427,"15min":193.66499999999999},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.54,"15min":2.879},"min":{"1min":0.33100000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":1.075,"5min":30.382999999999999,"15min":193.52099999999999},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":3.3820000000000001,"15min":2.8660000000000001},"min":{"1min":0.36899999999999999,"5min":0.35399999999999998,"15min":0.25},"max":{"1min":2.3199999999999998,"5min":107.907,"15min":107.907},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.65800000000000003,"5min":2.9249999999999998,"15min":2.843},"min":{"1min":0.376,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.0940000000000001,"5min":107.938,"15min":107.938},"last":0.58199999999999996}]},{"osd":3,"last update":"Sat Sep 23 15:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.506,"15min":3.726},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":0.92700000000000005,"5min":10.000999999999999,"15min":266.70299999999997},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.466,"15min":3.944},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.028,"5min":8.5129999999999999,"15min":266.80399999999997},"last":0.80300000000000005}]},{"osd":4,"last update":"Sat Sep 23 15:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.98599999999999999,"15min":1.337},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.249},"max":{"1min":0.754,"5min":8.2560000000000002,"15min":14.068},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.0609999999999999,"15min":1.3640000000000001},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.23599999999999999},"max":{"1min":0.93400000000000005,"5min":8.9550000000000001,"15min":19.879999999999999},"last":0.66000000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.101,"15min":1.6739999999999999},"min":{"1min":0.378,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.92900000000000005,"5min":8.4139999999999997,"15min":35.667000000000002},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.198,"15min":1.724},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.30499999999999999},"max":{"1min":0.84299999999999997,"5min":7.5039999999999996,"15min":36.140999999999998},"last":0.60499999999999998}]},{"osd":7,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.127,"15min":1.2989999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":0.88200000000000001,"5min":9.7390000000000008,"15min":16.065999999999999},"last":0.75},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.173,"15min":1.464},"min":{"1min":0.439,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":0.95699999999999996,"5min":8.3320000000000007,"15min":19.908000000000001},"last":0.78900000000000003}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:36:59.375 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T15:36:59.747 INFO:teuthology.orchestra.run.smithi017.stderr:Inferring config /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T15:36:59.958 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:36:59 smithi017 ceph-mon[215646]: from='client.47903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:36:59.958 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:36:59 smithi017 ceph-mon[222878]: from='client.47903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:37:00.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:36:59 smithi027 ceph-mon[195233]: from='client.47903 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:37:00.992 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:00 smithi017 ceph-mon[215646]: pgmap v1525: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:00.993 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:00 smithi017 ceph-mon[222878]: pgmap v1525: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:01.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:00 smithi027 ceph-mon[195233]: pgmap v1525: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:01.395 INFO:teuthology.orchestra.run.smithi017.stdout: 2023-09-23T15:37:01.397 INFO:teuthology.orchestra.run.smithi017.stderr:dumped all 2023-09-23T15:37:02.231 INFO:teuthology.orchestra.run.smithi017.stdout:{"pg_ready":true,"pg_map":{"version":1526,"stamp":"2023-09-23T15:37:01.103650+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11944,"num_read_kb":26378,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":4205364,"num_keys_recovered":10,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6998,"ondisk_log_size":6998,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4750220,"kb_used_data":73420,"kb_used_omap":57,"kb_used_meta":4676678,"kb_avail":745210996,"statfs":{"total":767960285184,"available":763096059904,"internally_reserved":0,"allocated":75182080,"data_stored":59433305,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":59367,"internal_metadata":4788918297},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003290"},"pg_stats":[{"pgid":"1.18","version":"181'162","reported_seq":563,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766953+0000","last_change":"2023-09-23T15:35:45.127684+0000","last_active":"2023-09-23T15:36:18.766953+0000","last_peered":"2023-09-23T15:36:18.766953+0000","last_clean":"2023-09-23T15:36:18.766953+0000","last_became_active":"2023-09-23T15:35:31.704743+0000","last_became_peered":"2023-09-23T15:35:31.704743+0000","last_unstale":"2023-09-23T15:36:18.766953+0000","last_undegraded":"2023-09-23T15:36:18.766953+0000","last_fullsized":"2023-09-23T15:36:18.766953+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:35:45.127621+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:35:45.127621+0000","objects_scrubbed":1,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:44.864740+0000","scrub_duration":0.0017349170000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"68'1","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258188+0000","last_change":"2023-09-23T15:27:23.316505+0000","last_active":"2023-09-23T15:36:20.258188+0000","last_peered":"2023-09-23T15:36:20.258188+0000","last_clean":"2023-09-23T15:36:20.258188+0000","last_became_active":"2023-09-23T15:27:23.315892+0000","last_became_peered":"2023-09-23T15:27:23.315892+0000","last_unstale":"2023-09-23T15:36:20.258188+0000","last_undegraded":"2023-09-23T15:36:20.258188+0000","last_fullsized":"2023-09-23T15:36:20.258188+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:42.103300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":395,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920248+0000","last_change":"2023-09-23T15:35:52.691279+0000","last_active":"2023-09-23T15:36:18.920248+0000","last_peered":"2023-09-23T15:36:18.920248+0000","last_clean":"2023-09-23T15:36:18.920248+0000","last_became_active":"2023-09-23T15:35:52.690933+0000","last_became_peered":"2023-09-23T15:35:52.690933+0000","last_unstale":"2023-09-23T15:36:18.920248+0000","last_undegraded":"2023-09-23T15:36:18.920248+0000","last_fullsized":"2023-09-23T15:36:18.920248+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:42.453633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"91'161","reported_seq":541,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766977+0000","last_change":"2023-09-23T15:35:34.128191+0000","last_active":"2023-09-23T15:36:18.766977+0000","last_peered":"2023-09-23T15:36:18.766977+0000","last_clean":"2023-09-23T15:36:18.766977+0000","last_became_active":"2023-09-23T15:35:31.708660+0000","last_became_peered":"2023-09-23T15:35:31.708660+0000","last_unstale":"2023-09-23T15:36:18.766977+0000","last_undegraded":"2023-09-23T15:36:18.766977+0000","last_fullsized":"2023-09-23T15:36:18.766977+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:35:34.128097+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:35:34.128097+0000","last_clean_scrub_stamp":"2023-09-23T15:35:34.128097+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:48:25.416633+0000","scrub_duration":0.0018547609999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":446,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.778990+0000","last_change":"2023-09-23T15:27:23.313101+0000","last_active":"2023-09-23T15:36:21.778990+0000","last_peered":"2023-09-23T15:36:21.778990+0000","last_clean":"2023-09-23T15:36:21.778990+0000","last_became_active":"2023-09-23T15:27:23.312827+0000","last_became_peered":"2023-09-23T15:27:23.312827+0000","last_unstale":"2023-09-23T15:36:21.778990+0000","last_undegraded":"2023-09-23T15:36:21.778990+0000","last_fullsized":"2023-09-23T15:36:21.778990+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:30:34.474770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":517,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.117949+0000","last_change":"2023-09-23T15:27:41.421120+0000","last_active":"2023-09-23T15:36:22.117949+0000","last_peered":"2023-09-23T15:36:22.117949+0000","last_clean":"2023-09-23T15:36:22.117949+0000","last_became_active":"2023-09-23T15:27:41.420804+0000","last_became_peered":"2023-09-23T15:27:41.420804+0000","last_unstale":"2023-09-23T15:36:22.117949+0000","last_undegraded":"2023-09-23T15:36:22.117949+0000","last_fullsized":"2023-09-23T15:36:22.117949+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:49.213950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"91'161","reported_seq":653,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259771+0000","last_change":"2023-09-23T15:28:51.947545+0000","last_active":"2023-09-23T15:36:20.259771+0000","last_peered":"2023-09-23T15:36:20.259771+0000","last_clean":"2023-09-23T15:36:20.259771+0000","last_became_active":"2023-09-23T15:28:39.656539+0000","last_became_peered":"2023-09-23T15:28:39.656539+0000","last_unstale":"2023-09-23T15:36:20.259771+0000","last_undegraded":"2023-09-23T15:36:20.259771+0000","last_fullsized":"2023-09-23T15:36:20.259771+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:51.947414+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:51.947414+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:55.695683+0000","scrub_duration":0.0078583750000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":432,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919896+0000","last_change":"2023-09-23T15:35:52.689532+0000","last_active":"2023-09-23T15:36:18.919896+0000","last_peered":"2023-09-23T15:36:18.919896+0000","last_clean":"2023-09-23T15:36:18.919896+0000","last_became_active":"2023-09-23T15:35:52.689189+0000","last_became_peered":"2023-09-23T15:35:52.689189+0000","last_unstale":"2023-09-23T15:36:18.919896+0000","last_undegraded":"2023-09-23T15:36:18.919896+0000","last_fullsized":"2023-09-23T15:36:18.919896+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:07:10.452807+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":488,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771717+0000","last_change":"2023-09-23T15:35:28.686386+0000","last_active":"2023-09-23T15:36:18.771717+0000","last_peered":"2023-09-23T15:36:18.771717+0000","last_clean":"2023-09-23T15:36:18.771717+0000","last_became_active":"2023-09-23T15:35:28.685718+0000","last_became_peered":"2023-09-23T15:35:28.685718+0000","last_unstale":"2023-09-23T15:36:18.771717+0000","last_undegraded":"2023-09-23T15:36:18.771717+0000","last_fullsized":"2023-09-23T15:36:18.771717+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:19:46.720477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"91'161","reported_seq":826,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257825+0000","last_change":"2023-09-23T15:35:31.710915+0000","last_active":"2023-09-23T15:36:20.257825+0000","last_peered":"2023-09-23T15:36:20.257825+0000","last_clean":"2023-09-23T15:36:20.257825+0000","last_became_active":"2023-09-23T15:35:31.710539+0000","last_became_peered":"2023-09-23T15:35:31.710539+0000","last_unstale":"2023-09-23T15:36:20.257825+0000","last_undegraded":"2023-09-23T15:36:20.257825+0000","last_fullsized":"2023-09-23T15:36:20.257825+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:27:01.690853+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:27:01.690853+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:32:24.282583+0000","scrub_duration":0.0017049179999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":488,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765155+0000","last_change":"2023-09-23T15:35:28.678928+0000","last_active":"2023-09-23T15:36:18.765155+0000","last_peered":"2023-09-23T15:36:18.765155+0000","last_clean":"2023-09-23T15:36:18.765155+0000","last_became_active":"2023-09-23T15:35:28.678376+0000","last_became_peered":"2023-09-23T15:35:28.678376+0000","last_unstale":"2023-09-23T15:36:18.765155+0000","last_undegraded":"2023-09-23T15:36:18.765155+0000","last_fullsized":"2023-09-23T15:36:18.765155+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:13.220709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":429,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919913+0000","last_change":"2023-09-23T15:35:52.691449+0000","last_active":"2023-09-23T15:36:18.919913+0000","last_peered":"2023-09-23T15:36:18.919913+0000","last_clean":"2023-09-23T15:36:18.919913+0000","last_became_active":"2023-09-23T15:35:52.690736+0000","last_became_peered":"2023-09-23T15:35:52.690736+0000","last_unstale":"2023-09-23T15:36:18.919913+0000","last_undegraded":"2023-09-23T15:36:18.919913+0000","last_fullsized":"2023-09-23T15:36:18.919913+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:22:41.247555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"91'161","reported_seq":929,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.038991+0000","last_change":"2023-09-23T15:27:42.381672+0000","last_active":"2023-09-23T15:36:19.038991+0000","last_peered":"2023-09-23T15:36:19.038991+0000","last_clean":"2023-09-23T15:36:19.038991+0000","last_became_active":"2023-09-23T15:27:42.381283+0000","last_became_peered":"2023-09-23T15:27:42.381283+0000","last_unstale":"2023-09-23T15:36:19.038991+0000","last_undegraded":"2023-09-23T15:36:19.038991+0000","last_fullsized":"2023-09-23T15:36:19.038991+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:55.370041+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:55.370041+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:58:59.022139+0000","scrub_duration":0.0044342189999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767078+0000","last_change":"2023-09-23T15:35:28.690198+0000","last_active":"2023-09-23T15:36:18.767078+0000","last_peered":"2023-09-23T15:36:18.767078+0000","last_clean":"2023-09-23T15:36:18.767078+0000","last_became_active":"2023-09-23T15:35:28.689773+0000","last_became_peered":"2023-09-23T15:35:28.689773+0000","last_unstale":"2023-09-23T15:36:18.767078+0000","last_undegraded":"2023-09-23T15:36:18.767078+0000","last_fullsized":"2023-09-23T15:36:18.767078+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:20.156779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":389,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920023+0000","last_change":"2023-09-23T15:35:52.694201+0000","last_active":"2023-09-23T15:36:18.920023+0000","last_peered":"2023-09-23T15:36:18.920023+0000","last_clean":"2023-09-23T15:36:18.920023+0000","last_became_active":"2023-09-23T15:35:52.694015+0000","last_became_peered":"2023-09-23T15:35:52.694015+0000","last_unstale":"2023-09-23T15:36:18.920023+0000","last_undegraded":"2023-09-23T15:36:18.920023+0000","last_fullsized":"2023-09-23T15:36:18.920023+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:36:21.761252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"91'161","reported_seq":602,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767263+0000","last_change":"2023-09-23T15:35:31.709507+0000","last_active":"2023-09-23T15:36:18.767263+0000","last_peered":"2023-09-23T15:36:18.767263+0000","last_clean":"2023-09-23T15:36:18.767263+0000","last_became_active":"2023-09-23T15:35:31.708061+0000","last_became_peered":"2023-09-23T15:35:31.708061+0000","last_unstale":"2023-09-23T15:36:18.767263+0000","last_undegraded":"2023-09-23T15:36:18.767263+0000","last_fullsized":"2023-09-23T15:36:18.767263+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:06.824864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:06.824864+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:44.536644+0000","scrub_duration":0.0065159160000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"68'3","reported_seq":263,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259081+0000","last_change":"2023-09-23T15:27:42.382718+0000","last_active":"2023-09-23T15:36:20.259081+0000","last_peered":"2023-09-23T15:36:20.259081+0000","last_clean":"2023-09-23T15:36:20.259081+0000","last_became_active":"2023-09-23T15:27:42.382260+0000","last_became_peered":"2023-09-23T15:27:42.382260+0000","last_unstale":"2023-09-23T15:36:20.259081+0000","last_undegraded":"2023-09-23T15:36:20.259081+0000","last_fullsized":"2023-09-23T15:36:20.259081+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:17.908287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":414,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390209+0000","last_change":"2023-09-23T15:22:26.300491+0000","last_active":"2023-09-23T15:36:19.390209+0000","last_peered":"2023-09-23T15:36:19.390209+0000","last_clean":"2023-09-23T15:36:19.390209+0000","last_became_active":"2023-09-23T15:22:26.300200+0000","last_became_peered":"2023-09-23T15:22:26.300200+0000","last_unstale":"2023-09-23T15:36:19.390209+0000","last_undegraded":"2023-09-23T15:36:19.390209+0000","last_fullsized":"2023-09-23T15:36:19.390209+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:53.595609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"91'161","reported_seq":671,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390467+0000","last_change":"2023-09-23T15:29:00.382590+0000","last_active":"2023-09-23T15:36:19.390467+0000","last_peered":"2023-09-23T15:36:19.390467+0000","last_clean":"2023-09-23T15:36:19.390467+0000","last_became_active":"2023-09-23T15:28:39.670092+0000","last_became_peered":"2023-09-23T15:28:39.670092+0000","last_unstale":"2023-09-23T15:36:19.390467+0000","last_undegraded":"2023-09-23T15:36:19.390467+0000","last_fullsized":"2023-09-23T15:36:19.390467+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:29:00.382448+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:29:00.382448+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:18.937380+0000","scrub_duration":0.0061805410000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"91'22","reported_seq":184,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259992+0000","last_change":"2023-09-23T15:27:42.391305+0000","last_active":"2023-09-23T15:36:20.259992+0000","last_peered":"2023-09-23T15:36:20.259992+0000","last_clean":"2023-09-23T15:36:20.259992+0000","last_became_active":"2023-09-23T15:27:42.390865+0000","last_became_peered":"2023-09-23T15:27:42.390865+0000","last_unstale":"2023-09-23T15:36:20.259992+0000","last_undegraded":"2023-09-23T15:36:20.259992+0000","last_fullsized":"2023-09-23T15:36:20.259992+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:48.841828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258012+0000","last_change":"2023-09-23T15:27:23.320168+0000","last_active":"2023-09-23T15:36:20.258012+0000","last_peered":"2023-09-23T15:36:20.258012+0000","last_clean":"2023-09-23T15:36:20.258012+0000","last_became_active":"2023-09-23T15:27:23.319637+0000","last_became_peered":"2023-09-23T15:27:23.319637+0000","last_unstale":"2023-09-23T15:36:20.258012+0000","last_undegraded":"2023-09-23T15:36:20.258012+0000","last_fullsized":"2023-09-23T15:36:20.258012+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:14.781419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"91'161","reported_seq":865,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763939+0000","last_change":"2023-09-23T15:35:31.707371+0000","last_active":"2023-09-23T15:36:18.763939+0000","last_peered":"2023-09-23T15:36:18.763939+0000","last_clean":"2023-09-23T15:36:18.763939+0000","last_became_active":"2023-09-23T15:35:31.707047+0000","last_became_peered":"2023-09-23T15:35:31.707047+0000","last_unstale":"2023-09-23T15:36:18.763939+0000","last_undegraded":"2023-09-23T15:36:18.763939+0000","last_fullsized":"2023-09-23T15:36:18.763939+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:07.144662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:07.144662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:04.181094+0000","scrub_duration":0.012663096,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"68'2","reported_seq":267,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259078+0000","last_change":"2023-09-23T15:27:42.381740+0000","last_active":"2023-09-23T15:36:20.259078+0000","last_peered":"2023-09-23T15:36:20.259078+0000","last_clean":"2023-09-23T15:36:20.259078+0000","last_became_active":"2023-09-23T15:27:42.380786+0000","last_became_peered":"2023-09-23T15:27:42.380786+0000","last_unstale":"2023-09-23T15:36:20.259078+0000","last_undegraded":"2023-09-23T15:36:20.259078+0000","last_fullsized":"2023-09-23T15:36:20.259078+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:02:06.772810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":34,"num_keys_recovered":10,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":68,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039240+0000","last_change":"2023-09-23T15:33:30.765277+0000","last_active":"2023-09-23T15:36:19.039240+0000","last_peered":"2023-09-23T15:36:19.039240+0000","last_clean":"2023-09-23T15:36:19.039240+0000","last_became_active":"2023-09-23T15:33:30.764903+0000","last_became_peered":"2023-09-23T15:33:30.764903+0000","last_unstale":"2023-09-23T15:36:19.039240+0000","last_undegraded":"2023-09-23T15:36:19.039240+0000","last_fullsized":"2023-09-23T15:36:19.039240+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:24:08.782060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,7],"acting":[2,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.10","version":"91'161","reported_seq":660,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766815+0000","last_change":"2023-09-23T15:35:31.704842+0000","last_active":"2023-09-23T15:36:18.766815+0000","last_peered":"2023-09-23T15:36:18.766815+0000","last_clean":"2023-09-23T15:36:18.766815+0000","last_became_active":"2023-09-23T15:35:31.704369+0000","last_became_peered":"2023-09-23T15:35:31.704369+0000","last_unstale":"2023-09-23T15:36:18.766815+0000","last_undegraded":"2023-09-23T15:36:18.766815+0000","last_fullsized":"2023-09-23T15:36:18.766815+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:33.166785+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:33.166785+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:51:56.396764+0000","scrub_duration":0.013147114999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":514,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763984+0000","last_change":"2023-09-23T15:27:41.419908+0000","last_active":"2023-09-23T15:36:18.763984+0000","last_peered":"2023-09-23T15:36:18.763984+0000","last_clean":"2023-09-23T15:36:18.763984+0000","last_became_active":"2023-09-23T15:27:41.419523+0000","last_became_peered":"2023-09-23T15:27:41.419523+0000","last_unstale":"2023-09-23T15:36:18.763984+0000","last_undegraded":"2023-09-23T15:36:18.763984+0000","last_fullsized":"2023-09-23T15:36:18.763984+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:29.701928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767499+0000","last_change":"2023-09-23T15:35:28.690359+0000","last_active":"2023-09-23T15:36:18.767499+0000","last_peered":"2023-09-23T15:36:18.767499+0000","last_clean":"2023-09-23T15:36:18.767499+0000","last_became_active":"2023-09-23T15:35:28.689948+0000","last_became_peered":"2023-09-23T15:35:28.689948+0000","last_unstale":"2023-09-23T15:36:18.767499+0000","last_undegraded":"2023-09-23T15:36:18.767499+0000","last_fullsized":"2023-09-23T15:36:18.767499+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:25.640800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"91'161","reported_seq":847,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390237+0000","last_change":"2023-09-23T15:27:26.324674+0000","last_active":"2023-09-23T15:36:19.390237+0000","last_peered":"2023-09-23T15:36:19.390237+0000","last_clean":"2023-09-23T15:36:19.390237+0000","last_became_active":"2023-09-23T15:27:26.324282+0000","last_became_peered":"2023-09-23T15:27:26.324282+0000","last_unstale":"2023-09-23T15:36:19.390237+0000","last_undegraded":"2023-09-23T15:36:19.390237+0000","last_fullsized":"2023-09-23T15:36:19.390237+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:51.467471+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:51.467471+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:56.872051+0000","scrub_duration":0.0044341340000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":261,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258733+0000","last_change":"2023-09-23T15:27:42.381675+0000","last_active":"2023-09-23T15:36:20.258733+0000","last_peered":"2023-09-23T15:36:20.258733+0000","last_clean":"2023-09-23T15:36:20.258733+0000","last_became_active":"2023-09-23T15:27:42.381363+0000","last_became_peered":"2023-09-23T15:27:42.381363+0000","last_unstale":"2023-09-23T15:36:20.258733+0000","last_undegraded":"2023-09-23T15:36:20.258733+0000","last_fullsized":"2023-09-23T15:36:20.258733+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:21:35.074488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":503,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765004+0000","last_change":"2023-09-23T15:27:41.421182+0000","last_active":"2023-09-23T15:36:18.765004+0000","last_peered":"2023-09-23T15:36:18.765004+0000","last_clean":"2023-09-23T15:36:18.765004+0000","last_became_active":"2023-09-23T15:27:41.420834+0000","last_became_peered":"2023-09-23T15:27:41.420834+0000","last_unstale":"2023-09-23T15:36:18.765004+0000","last_undegraded":"2023-09-23T15:36:18.765004+0000","last_fullsized":"2023-09-23T15:36:18.765004+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:06.175315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"91'161","reported_seq":562,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766918+0000","last_change":"2023-09-23T15:35:31.710854+0000","last_active":"2023-09-23T15:36:18.766918+0000","last_peered":"2023-09-23T15:36:18.766918+0000","last_clean":"2023-09-23T15:36:18.766918+0000","last_became_active":"2023-09-23T15:35:31.710465+0000","last_became_peered":"2023-09-23T15:35:31.710465+0000","last_unstale":"2023-09-23T15:36:18.766918+0000","last_undegraded":"2023-09-23T15:36:18.766918+0000","last_fullsized":"2023-09-23T15:36:18.766918+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:37.289291+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:12:37.289291+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:09:24.612228+0000","scrub_duration":0.0022912570000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"68'8","reported_seq":494,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.770883+0000","last_change":"2023-09-23T15:35:28.699648+0000","last_active":"2023-09-23T15:36:18.770883+0000","last_peered":"2023-09-23T15:36:18.770883+0000","last_clean":"2023-09-23T15:36:18.770883+0000","last_became_active":"2023-09-23T15:35:28.686841+0000","last_became_peered":"2023-09-23T15:35:28.686841+0000","last_unstale":"2023-09-23T15:36:18.770883+0000","last_undegraded":"2023-09-23T15:36:18.770883+0000","last_fullsized":"2023-09-23T15:36:18.770883+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:20.963381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":440,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920558+0000","last_change":"2023-09-23T15:35:52.697013+0000","last_active":"2023-09-23T15:36:18.920558+0000","last_peered":"2023-09-23T15:36:18.920558+0000","last_clean":"2023-09-23T15:36:18.920558+0000","last_became_active":"2023-09-23T15:35:52.696731+0000","last_became_peered":"2023-09-23T15:35:52.696731+0000","last_unstale":"2023-09-23T15:36:18.920558+0000","last_undegraded":"2023-09-23T15:36:18.920558+0000","last_fullsized":"2023-09-23T15:36:18.920558+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:32:19.949603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"91'161","reported_seq":814,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259102+0000","last_change":"2023-09-23T15:28:56.386871+0000","last_active":"2023-09-23T15:36:20.259102+0000","last_peered":"2023-09-23T15:36:20.259102+0000","last_clean":"2023-09-23T15:36:20.259102+0000","last_became_active":"2023-09-23T15:27:42.381231+0000","last_became_peered":"2023-09-23T15:27:42.381231+0000","last_unstale":"2023-09-23T15:36:20.259102+0000","last_undegraded":"2023-09-23T15:36:20.259102+0000","last_fullsized":"2023-09-23T15:36:20.259102+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:56.386796+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:56.386796+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:31.798374+0000","scrub_duration":0.0086882850000000005,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"68'1","reported_seq":257,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259471+0000","last_change":"2023-09-23T15:27:42.381632+0000","last_active":"2023-09-23T15:36:20.259471+0000","last_peered":"2023-09-23T15:36:20.259471+0000","last_clean":"2023-09-23T15:36:20.259471+0000","last_became_active":"2023-09-23T15:27:42.381272+0000","last_became_peered":"2023-09-23T15:27:42.381272+0000","last_unstale":"2023-09-23T15:36:20.259471+0000","last_undegraded":"2023-09-23T15:36:20.259471+0000","last_fullsized":"2023-09-23T15:36:20.259471+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:54.951796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766577+0000","last_change":"2023-09-23T15:35:28.691240+0000","last_active":"2023-09-23T15:36:18.766577+0000","last_peered":"2023-09-23T15:36:18.766577+0000","last_clean":"2023-09-23T15:36:18.766577+0000","last_became_active":"2023-09-23T15:35:28.690862+0000","last_became_peered":"2023-09-23T15:35:28.690862+0000","last_unstale":"2023-09-23T15:36:18.766577+0000","last_undegraded":"2023-09-23T15:36:18.766577+0000","last_fullsized":"2023-09-23T15:36:18.766577+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:45:41.052218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"91'161","reported_seq":761,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920257+0000","last_change":"2023-09-23T15:35:52.691576+0000","last_active":"2023-09-23T15:36:18.920257+0000","last_peered":"2023-09-23T15:36:18.920257+0000","last_clean":"2023-09-23T15:36:18.920257+0000","last_became_active":"2023-09-23T15:35:52.691182+0000","last_became_peered":"2023-09-23T15:35:52.691182+0000","last_unstale":"2023-09-23T15:36:18.920257+0000","last_undegraded":"2023-09-23T15:36:18.920257+0000","last_fullsized":"2023-09-23T15:36:18.920257+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:48.890181+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:48.890181+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:32.354840+0000","scrub_duration":0.08250528,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"68'2","reported_seq":257,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258802+0000","last_change":"2023-09-23T15:27:42.387885+0000","last_active":"2023-09-23T15:36:20.258802+0000","last_peered":"2023-09-23T15:36:20.258802+0000","last_clean":"2023-09-23T15:36:20.258802+0000","last_became_active":"2023-09-23T15:27:42.387510+0000","last_became_peered":"2023-09-23T15:27:42.387510+0000","last_unstale":"2023-09-23T15:36:20.258802+0000","last_undegraded":"2023-09-23T15:36:20.258802+0000","last_fullsized":"2023-09-23T15:36:20.258802+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:11:55.326416+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":170,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258788+0000","last_change":"2023-09-23T15:33:30.765181+0000","last_active":"2023-09-23T15:36:20.258788+0000","last_peered":"2023-09-23T15:36:20.258788+0000","last_clean":"2023-09-23T15:36:20.258788+0000","last_became_active":"2023-09-23T15:33:30.764802+0000","last_became_peered":"2023-09-23T15:33:30.764802+0000","last_unstale":"2023-09-23T15:36:20.258788+0000","last_undegraded":"2023-09-23T15:36:20.258788+0000","last_fullsized":"2023-09-23T15:36:20.258788+0000","mapping_epoch":460,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":461,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:36:15.536014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"91'161","reported_seq":805,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771619+0000","last_change":"2023-09-23T15:35:31.703608+0000","last_active":"2023-09-23T15:36:18.771619+0000","last_peered":"2023-09-23T15:36:18.771619+0000","last_clean":"2023-09-23T15:36:18.771619+0000","last_became_active":"2023-09-23T15:35:31.703159+0000","last_became_peered":"2023-09-23T15:35:31.703159+0000","last_unstale":"2023-09-23T15:36:18.771619+0000","last_undegraded":"2023-09-23T15:36:18.771619+0000","last_fullsized":"2023-09-23T15:36:18.771619+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:49.360973+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:49.360973+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:46.705865+0000","scrub_duration":0.0015744279999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"68'1","reported_seq":273,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258897+0000","last_change":"2023-09-23T15:35:28.702405+0000","last_active":"2023-09-23T15:36:20.258897+0000","last_peered":"2023-09-23T15:36:20.258897+0000","last_clean":"2023-09-23T15:36:20.258897+0000","last_became_active":"2023-09-23T15:35:28.689620+0000","last_became_peered":"2023-09-23T15:35:28.689620+0000","last_unstale":"2023-09-23T15:36:20.258897+0000","last_undegraded":"2023-09-23T15:36:20.258897+0000","last_fullsized":"2023-09-23T15:36:20.258897+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:25:04.339529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":498,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771072+0000","last_change":"2023-09-23T15:27:41.426402+0000","last_active":"2023-09-23T15:36:18.771072+0000","last_peered":"2023-09-23T15:36:18.771072+0000","last_clean":"2023-09-23T15:36:18.771072+0000","last_became_active":"2023-09-23T15:27:41.426081+0000","last_became_peered":"2023-09-23T15:27:41.426081+0000","last_unstale":"2023-09-23T15:36:18.771072+0000","last_undegraded":"2023-09-23T15:36:18.771072+0000","last_fullsized":"2023-09-23T15:36:18.771072+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:36:36.351589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"91'161","reported_seq":735,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260255+0000","last_change":"2023-09-23T15:28:49.949013+0000","last_active":"2023-09-23T15:36:20.260255+0000","last_peered":"2023-09-23T15:36:20.260255+0000","last_clean":"2023-09-23T15:36:20.260255+0000","last_became_active":"2023-09-23T15:27:42.400810+0000","last_became_peered":"2023-09-23T15:27:42.400810+0000","last_unstale":"2023-09-23T15:36:20.260255+0000","last_undegraded":"2023-09-23T15:36:20.260255+0000","last_fullsized":"2023-09-23T15:36:20.260255+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.948909+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.948909+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:39.930266+0000","scrub_duration":0.0021046950000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767307+0000","last_change":"2023-09-23T15:35:28.685014+0000","last_active":"2023-09-23T15:36:18.767307+0000","last_peered":"2023-09-23T15:36:18.767307+0000","last_clean":"2023-09-23T15:36:18.767307+0000","last_became_active":"2023-09-23T15:35:28.684585+0000","last_became_peered":"2023-09-23T15:35:28.684585+0000","last_unstale":"2023-09-23T15:36:18.767307+0000","last_undegraded":"2023-09-23T15:36:18.767307+0000","last_fullsized":"2023-09-23T15:36:18.767307+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:52.512630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":522,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764395+0000","last_change":"2023-09-23T15:27:23.321508+0000","last_active":"2023-09-23T15:36:18.764395+0000","last_peered":"2023-09-23T15:36:18.764395+0000","last_clean":"2023-09-23T15:36:18.764395+0000","last_became_active":"2023-09-23T15:27:23.320681+0000","last_became_peered":"2023-09-23T15:27:23.320681+0000","last_unstale":"2023-09-23T15:36:18.764395+0000","last_undegraded":"2023-09-23T15:36:18.764395+0000","last_fullsized":"2023-09-23T15:36:18.764395+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:29.301881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"91'161","reported_seq":770,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258549+0000","last_change":"2023-09-23T15:28:52.028751+0000","last_active":"2023-09-23T15:36:20.258549+0000","last_peered":"2023-09-23T15:36:20.258549+0000","last_clean":"2023-09-23T15:36:20.258549+0000","last_became_active":"2023-09-23T15:27:41.438996+0000","last_became_peered":"2023-09-23T15:27:41.438996+0000","last_unstale":"2023-09-23T15:36:20.258549+0000","last_undegraded":"2023-09-23T15:36:20.258549+0000","last_fullsized":"2023-09-23T15:36:20.258549+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:52.028655+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:52.028655+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:14.664152+0000","scrub_duration":0.014149948000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"68'2","reported_seq":436,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919963+0000","last_change":"2023-09-23T15:35:52.692066+0000","last_active":"2023-09-23T15:36:18.919963+0000","last_peered":"2023-09-23T15:36:18.919963+0000","last_clean":"2023-09-23T15:36:18.919963+0000","last_became_active":"2023-09-23T15:35:52.691748+0000","last_became_peered":"2023-09-23T15:35:52.691748+0000","last_unstale":"2023-09-23T15:36:18.919963+0000","last_undegraded":"2023-09-23T15:36:18.919963+0000","last_fullsized":"2023-09-23T15:36:18.919963+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:42.388322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257975+0000","last_change":"2023-09-23T15:27:23.315912+0000","last_active":"2023-09-23T15:36:20.257975+0000","last_peered":"2023-09-23T15:36:20.257975+0000","last_clean":"2023-09-23T15:36:20.257975+0000","last_became_active":"2023-09-23T15:27:23.313953+0000","last_became_peered":"2023-09-23T15:27:23.313953+0000","last_unstale":"2023-09-23T15:36:20.257975+0000","last_undegraded":"2023-09-23T15:36:20.257975+0000","last_fullsized":"2023-09-23T15:36:20.257975+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:00.502950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"181'162","reported_seq":633,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766903+0000","last_change":"2023-09-23T15:35:31.701840+0000","last_active":"2023-09-23T15:36:18.766903+0000","last_peered":"2023-09-23T15:36:18.766903+0000","last_clean":"2023-09-23T15:36:18.766903+0000","last_became_active":"2023-09-23T15:35:31.701439+0000","last_became_peered":"2023-09-23T15:35:31.701439+0000","last_unstale":"2023-09-23T15:36:18.766903+0000","last_undegraded":"2023-09-23T15:36:18.766903+0000","last_fullsized":"2023-09-23T15:36:18.766903+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"181'162","last_scrub_stamp":"2023-09-23T15:33:05.479226+0000","last_deep_scrub":"181'162","last_deep_scrub_stamp":"2023-09-23T15:12:40.331327+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.479226+0000","objects_scrubbed":0,"log_size":162,"log_dups_size":0,"ondisk_log_size":162,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:16.177908+0000","scrub_duration":0.0055363749999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":485,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771115+0000","last_change":"2023-09-23T15:27:23.310440+0000","last_active":"2023-09-23T15:36:18.771115+0000","last_peered":"2023-09-23T15:36:18.771115+0000","last_clean":"2023-09-23T15:36:18.771115+0000","last_became_active":"2023-09-23T15:27:23.309671+0000","last_became_peered":"2023-09-23T15:27:23.309671+0000","last_unstale":"2023-09-23T15:36:18.771115+0000","last_undegraded":"2023-09-23T15:36:18.771115+0000","last_fullsized":"2023-09-23T15:36:18.771115+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:10:02.414591+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258006+0000","last_change":"2023-09-23T15:27:23.317562+0000","last_active":"2023-09-23T15:36:20.258006+0000","last_peered":"2023-09-23T15:36:20.258006+0000","last_clean":"2023-09-23T15:36:20.258006+0000","last_became_active":"2023-09-23T15:27:23.316501+0000","last_became_peered":"2023-09-23T15:27:23.316501+0000","last_unstale":"2023-09-23T15:36:20.258006+0000","last_undegraded":"2023-09-23T15:36:20.258006+0000","last_fullsized":"2023-09-23T15:36:20.258006+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:02.121786+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"91'161","reported_seq":835,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779066+0000","last_change":"2023-09-23T15:35:31.709834+0000","last_active":"2023-09-23T15:36:21.779066+0000","last_peered":"2023-09-23T15:36:21.779066+0000","last_clean":"2023-09-23T15:36:21.779066+0000","last_became_active":"2023-09-23T15:35:31.709429+0000","last_became_peered":"2023-09-23T15:35:31.709429+0000","last_unstale":"2023-09-23T15:36:21.779066+0000","last_undegraded":"2023-09-23T15:36:21.779066+0000","last_fullsized":"2023-09-23T15:36:21.779066+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":152,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:50.442222+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:50.442222+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:06.567918+0000","scrub_duration":0.037587207999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":174,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258881+0000","last_change":"2023-09-23T15:27:42.382531+0000","last_active":"2023-09-23T15:36:20.258881+0000","last_peered":"2023-09-23T15:36:20.258881+0000","last_clean":"2023-09-23T15:36:20.258881+0000","last_became_active":"2023-09-23T15:27:42.381174+0000","last_became_peered":"2023-09-23T15:27:42.381174+0000","last_unstale":"2023-09-23T15:36:20.258881+0000","last_undegraded":"2023-09-23T15:36:20.258881+0000","last_fullsized":"2023-09-23T15:36:20.258881+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:07:31.908182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260275+0000","last_change":"2023-09-23T15:32:16.253887+0000","last_active":"2023-09-23T15:36:20.260275+0000","last_peered":"2023-09-23T15:36:20.260275+0000","last_clean":"2023-09-23T15:36:20.260275+0000","last_became_active":"2023-09-23T15:32:16.253556+0000","last_became_peered":"2023-09-23T15:32:16.253556+0000","last_unstale":"2023-09-23T15:36:20.260275+0000","last_undegraded":"2023-09-23T15:36:20.260275+0000","last_fullsized":"2023-09-23T15:36:20.260275+0000","mapping_epoch":442,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:16.930439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"91'161","reported_seq":628,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766808+0000","last_change":"2023-09-23T15:35:31.705105+0000","last_active":"2023-09-23T15:36:18.766808+0000","last_peered":"2023-09-23T15:36:18.766808+0000","last_clean":"2023-09-23T15:36:18.766808+0000","last_became_active":"2023-09-23T15:35:31.704697+0000","last_became_peered":"2023-09-23T15:35:31.704697+0000","last_unstale":"2023-09-23T15:36:18.766808+0000","last_undegraded":"2023-09-23T15:36:18.766808+0000","last_fullsized":"2023-09-23T15:36:18.766808+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:30.357860+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:30.357860+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:20.163569+0000","scrub_duration":0.0015641730000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":450,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919814+0000","last_change":"2023-09-23T15:35:52.695833+0000","last_active":"2023-09-23T15:36:18.919814+0000","last_peered":"2023-09-23T15:36:18.919814+0000","last_clean":"2023-09-23T15:36:18.919814+0000","last_became_active":"2023-09-23T15:35:52.695109+0000","last_became_peered":"2023-09-23T15:35:52.695109+0000","last_unstale":"2023-09-23T15:36:18.919814+0000","last_undegraded":"2023-09-23T15:36:18.919814+0000","last_fullsized":"2023-09-23T15:36:18.919814+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:39.595373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":455,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390622+0000","last_change":"2023-09-23T15:27:23.314695+0000","last_active":"2023-09-23T15:36:19.390622+0000","last_peered":"2023-09-23T15:36:19.390622+0000","last_clean":"2023-09-23T15:36:19.390622+0000","last_became_active":"2023-09-23T15:27:23.313927+0000","last_became_peered":"2023-09-23T15:27:23.313927+0000","last_unstale":"2023-09-23T15:36:19.390622+0000","last_undegraded":"2023-09-23T15:36:19.390622+0000","last_fullsized":"2023-09-23T15:36:19.390622+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:18.346825+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"91'161","reported_seq":758,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771636+0000","last_change":"2023-09-23T15:28:49.592908+0000","last_active":"2023-09-23T15:36:18.771636+0000","last_peered":"2023-09-23T15:36:18.771636+0000","last_clean":"2023-09-23T15:36:18.771636+0000","last_became_active":"2023-09-23T15:27:41.419745+0000","last_became_peered":"2023-09-23T15:27:41.419745+0000","last_unstale":"2023-09-23T15:36:18.771636+0000","last_undegraded":"2023-09-23T15:36:18.771636+0000","last_fullsized":"2023-09-23T15:36:18.771636+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:49.592826+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:49.592826+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:02.902764+0000","scrub_duration":0.11607071400000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"488'163","reported_seq":1039,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390584+0000","last_change":"2023-09-23T15:34:54.709256+0000","last_active":"2023-09-23T15:36:19.390584+0000","last_peered":"2023-09-23T15:36:19.390584+0000","last_clean":"2023-09-23T15:36:19.390584+0000","last_became_active":"2023-09-23T15:27:41.437781+0000","last_became_peered":"2023-09-23T15:27:41.437781+0000","last_unstale":"2023-09-23T15:36:19.390584+0000","last_undegraded":"2023-09-23T15:36:19.390584+0000","last_fullsized":"2023-09-23T15:36:19.390584+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:46:49.225759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021979399999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":726,"num_read_kb":1976,"num_write":174,"num_write_kb":1249,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":30,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767382+0000","last_change":"2023-09-23T15:35:28.684619+0000","last_active":"2023-09-23T15:36:18.767382+0000","last_peered":"2023-09-23T15:36:18.767382+0000","last_clean":"2023-09-23T15:36:18.767382+0000","last_became_active":"2023-09-23T15:35:28.684056+0000","last_became_peered":"2023-09-23T15:35:28.684056+0000","last_unstale":"2023-09-23T15:36:18.767382+0000","last_undegraded":"2023-09-23T15:36:18.767382+0000","last_fullsized":"2023-09-23T15:36:18.767382+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:46.348436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":502,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.117336+0000","last_change":"2023-09-23T15:35:28.684478+0000","last_active":"2023-09-23T15:36:22.117336+0000","last_peered":"2023-09-23T15:36:22.117336+0000","last_clean":"2023-09-23T15:36:22.117336+0000","last_became_active":"2023-09-23T15:35:28.684083+0000","last_became_peered":"2023-09-23T15:35:28.684083+0000","last_unstale":"2023-09-23T15:36:22.117336+0000","last_undegraded":"2023-09-23T15:36:22.117336+0000","last_fullsized":"2023-09-23T15:36:22.117336+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.700819+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1","version":"91'161","reported_seq":626,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766778+0000","last_change":"2023-09-23T15:35:31.708912+0000","last_active":"2023-09-23T15:36:18.766778+0000","last_peered":"2023-09-23T15:36:18.766778+0000","last_clean":"2023-09-23T15:36:18.766778+0000","last_became_active":"2023-09-23T15:35:31.708400+0000","last_became_peered":"2023-09-23T15:35:31.708400+0000","last_unstale":"2023-09-23T15:36:18.766778+0000","last_undegraded":"2023-09-23T15:36:18.766778+0000","last_fullsized":"2023-09-23T15:36:18.766778+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:33:05.797446+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:33:05.797446+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:22.876441+0000","scrub_duration":0.0090775909999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"488'157","reported_seq":1115,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766829+0000","last_change":"2023-09-23T15:35:31.711290+0000","last_active":"2023-09-23T15:36:18.766829+0000","last_peered":"2023-09-23T15:36:18.766829+0000","last_clean":"2023-09-23T15:36:18.766829+0000","last_became_active":"2023-09-23T15:35:31.710579+0000","last_became_peered":"2023-09-23T15:35:31.710579+0000","last_unstale":"2023-09-23T15:36:18.766829+0000","last_undegraded":"2023-09-23T15:36:18.766829+0000","last_fullsized":"2023-09-23T15:36:18.766829+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:22.204523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033387800000000002,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1377,"num_read_kb":7709,"num_write":130,"num_write_kb":5863,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":29,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258349+0000","last_change":"2023-09-23T15:27:23.319579+0000","last_active":"2023-09-23T15:36:20.258349+0000","last_peered":"2023-09-23T15:36:20.258349+0000","last_clean":"2023-09-23T15:36:20.258349+0000","last_became_active":"2023-09-23T15:27:23.319169+0000","last_became_peered":"2023-09-23T15:27:23.319169+0000","last_unstale":"2023-09-23T15:36:20.258349+0000","last_undegraded":"2023-09-23T15:36:20.258349+0000","last_fullsized":"2023-09-23T15:36:20.258349+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:40:39.760113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":420,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390395+0000","last_change":"2023-09-23T15:12:41.859735+0000","last_active":"2023-09-23T15:36:19.390395+0000","last_peered":"2023-09-23T15:36:19.390395+0000","last_clean":"2023-09-23T15:36:19.390395+0000","last_became_active":"2023-09-23T15:12:41.859431+0000","last_became_peered":"2023-09-23T15:12:41.859431+0000","last_unstale":"2023-09-23T15:36:19.390395+0000","last_undegraded":"2023-09-23T15:36:19.390395+0000","last_fullsized":"2023-09-23T15:36:19.390395+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:58.082184+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"91'161","reported_seq":881,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258518+0000","last_change":"2023-09-23T15:27:42.382085+0000","last_active":"2023-09-23T15:36:20.258518+0000","last_peered":"2023-09-23T15:36:20.258518+0000","last_clean":"2023-09-23T15:36:20.258518+0000","last_became_active":"2023-09-23T15:27:42.381077+0000","last_became_peered":"2023-09-23T15:27:42.381077+0000","last_unstale":"2023-09-23T15:36:20.258518+0000","last_undegraded":"2023-09-23T15:36:20.258518+0000","last_fullsized":"2023-09-23T15:36:20.258518+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":4,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:13.069567+0000","last_clean_scrub_stamp":"2023-09-23T15:12:13.069567+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:37.092844+0000","scrub_duration":0.0097494209999999994,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"488'400","reported_seq":1945,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.116932+0000","last_change":"2023-09-23T15:34:54.705587+0000","last_active":"2023-09-23T15:36:22.116932+0000","last_peered":"2023-09-23T15:36:22.116932+0000","last_clean":"2023-09-23T15:36:22.116932+0000","last_became_active":"2023-09-23T15:06:16.128994+0000","last_became_peered":"2023-09-23T15:06:16.128994+0000","last_unstale":"2023-09-23T15:36:22.116932+0000","last_undegraded":"2023-09-23T15:36:22.116932+0000","last_fullsized":"2023-09-23T15:36:22.116932+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":400,"log_dups_size":0,"ondisk_log_size":400,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:36:20.482728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047212799999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2483,"num_read_kb":3714,"num_write":351,"num_write_kb":5627,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":416,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779043+0000","last_change":"2023-09-23T15:22:26.299869+0000","last_active":"2023-09-23T15:36:21.779043+0000","last_peered":"2023-09-23T15:36:21.779043+0000","last_clean":"2023-09-23T15:36:21.779043+0000","last_became_active":"2023-09-23T15:22:26.299537+0000","last_became_peered":"2023-09-23T15:22:26.299537+0000","last_unstale":"2023-09-23T15:36:21.779043+0000","last_undegraded":"2023-09-23T15:36:21.779043+0000","last_fullsized":"2023-09-23T15:36:21.779043+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":305,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:01.048777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":442,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779190+0000","last_change":"2023-09-23T15:35:28.685371+0000","last_active":"2023-09-23T15:36:21.779190+0000","last_peered":"2023-09-23T15:36:21.779190+0000","last_clean":"2023-09-23T15:36:21.779190+0000","last_became_active":"2023-09-23T15:35:28.685039+0000","last_became_peered":"2023-09-23T15:35:28.685039+0000","last_unstale":"2023-09-23T15:36:21.779190+0000","last_undegraded":"2023-09-23T15:36:21.779190+0000","last_fullsized":"2023-09-23T15:36:21.779190+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:35.106519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"488'306","reported_seq":1008,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.766858+0000","last_change":"2023-09-23T15:35:31.706131+0000","last_active":"2023-09-23T15:36:18.766858+0000","last_peered":"2023-09-23T15:36:18.766858+0000","last_clean":"2023-09-23T15:36:18.766858+0000","last_became_active":"2023-09-23T15:35:31.705360+0000","last_became_peered":"2023-09-23T15:35:31.705360+0000","last_unstale":"2023-09-23T15:36:18.766858+0000","last_undegraded":"2023-09-23T15:36:18.766858+0000","last_fullsized":"2023-09-23T15:36:18.766858+0000","mapping_epoch":500,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":501,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":306,"log_dups_size":0,"ondisk_log_size":306,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:37.831256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00022811199999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1196,"num_read_kb":2267,"num_write":287,"num_write_kb":1314,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"91'161","reported_seq":824,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258479+0000","last_change":"2023-09-23T15:27:41.449939+0000","last_active":"2023-09-23T15:36:20.258479+0000","last_peered":"2023-09-23T15:36:20.258479+0000","last_clean":"2023-09-23T15:36:20.258479+0000","last_became_active":"2023-09-23T15:27:41.444590+0000","last_became_peered":"2023-09-23T15:27:41.444590+0000","last_unstale":"2023-09-23T15:36:20.258479+0000","last_undegraded":"2023-09-23T15:36:20.258479+0000","last_fullsized":"2023-09-23T15:36:20.258479+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:12:41.361856+0000","last_clean_scrub_stamp":"2023-09-23T15:12:41.361856+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:25.036712+0000","scrub_duration":0.011980884000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":364,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919952+0000","last_change":"2023-09-23T15:35:52.695806+0000","last_active":"2023-09-23T15:36:18.919952+0000","last_peered":"2023-09-23T15:36:18.919952+0000","last_clean":"2023-09-23T15:36:18.919952+0000","last_became_active":"2023-09-23T15:35:52.695503+0000","last_became_peered":"2023-09-23T15:35:52.695503+0000","last_unstale":"2023-09-23T15:36:18.919952+0000","last_undegraded":"2023-09-23T15:36:18.919952+0000","last_fullsized":"2023-09-23T15:36:18.919952+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:22.363061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":510,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771475+0000","last_change":"2023-09-23T15:27:42.388413+0000","last_active":"2023-09-23T15:36:18.771475+0000","last_peered":"2023-09-23T15:36:18.771475+0000","last_clean":"2023-09-23T15:36:18.771475+0000","last_became_active":"2023-09-23T15:27:42.388111+0000","last_became_peered":"2023-09-23T15:27:42.388111+0000","last_unstale":"2023-09-23T15:36:18.771475+0000","last_undegraded":"2023-09-23T15:36:18.771475+0000","last_fullsized":"2023-09-23T15:36:18.771475+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:03.286346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"91'161","reported_seq":864,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390272+0000","last_change":"2023-09-23T15:27:26.323205+0000","last_active":"2023-09-23T15:36:19.390272+0000","last_peered":"2023-09-23T15:36:19.390272+0000","last_clean":"2023-09-23T15:36:19.390272+0000","last_became_active":"2023-09-23T15:27:26.322895+0000","last_became_peered":"2023-09-23T15:27:26.322895+0000","last_unstale":"2023-09-23T15:36:19.390272+0000","last_undegraded":"2023-09-23T15:36:19.390272+0000","last_fullsized":"2023-09-23T15:36:19.390272+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:48.525965+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:48.525965+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:30:41.664107+0000","scrub_duration":0.034986113999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"488'240","reported_seq":648,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258128+0000","last_change":"2023-09-23T15:34:54.708597+0000","last_active":"2023-09-23T15:36:20.258128+0000","last_peered":"2023-09-23T15:36:20.258128+0000","last_clean":"2023-09-23T15:36:20.258128+0000","last_became_active":"2023-09-23T15:27:41.444288+0000","last_became_peered":"2023-09-23T15:27:41.444288+0000","last_unstale":"2023-09-23T15:36:20.258128+0000","last_undegraded":"2023-09-23T15:36:20.258128+0000","last_fullsized":"2023-09-23T15:36:20.258128+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":240,"log_dups_size":0,"ondisk_log_size":240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:10.440732+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0013449600000000001,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1148,"num_read_kb":1983,"num_write":238,"num_write_kb":6264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":526,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.763939+0000","last_change":"2023-09-23T15:35:10.809299+0000","last_active":"2023-09-23T15:36:18.763939+0000","last_peered":"2023-09-23T15:36:18.763939+0000","last_clean":"2023-09-23T15:36:18.763939+0000","last_became_active":"2023-09-23T15:35:10.808955+0000","last_became_peered":"2023-09-23T15:35:10.808955+0000","last_unstale":"2023-09-23T15:36:18.763939+0000","last_undegraded":"2023-09-23T15:36:18.763939+0000","last_fullsized":"2023-09-23T15:36:18.763939+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:53.650771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":501,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764845+0000","last_change":"2023-09-23T15:12:41.927438+0000","last_active":"2023-09-23T15:36:18.764845+0000","last_peered":"2023-09-23T15:36:18.764845+0000","last_clean":"2023-09-23T15:36:18.764845+0000","last_became_active":"2023-09-23T15:12:41.927081+0000","last_became_peered":"2023-09-23T15:12:41.927081+0000","last_unstale":"2023-09-23T15:36:18.764845+0000","last_undegraded":"2023-09-23T15:36:18.764845+0000","last_fullsized":"2023-09-23T15:36:18.764845+0000","mapping_epoch":183,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":184,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:06.420455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"91'161","reported_seq":844,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771484+0000","last_change":"2023-09-23T15:28:48.566329+0000","last_active":"2023-09-23T15:36:18.771484+0000","last_peered":"2023-09-23T15:36:18.771484+0000","last_clean":"2023-09-23T15:36:18.771484+0000","last_became_active":"2023-09-23T15:26:50.305038+0000","last_became_peered":"2023-09-23T15:26:50.305038+0000","last_unstale":"2023-09-23T15:36:18.771484+0000","last_undegraded":"2023-09-23T15:36:18.771484+0000","last_fullsized":"2023-09-23T15:36:18.771484+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":358,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:48.566207+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:48.566207+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:39.374385+0000","scrub_duration":0.044321790999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"488'207","reported_seq":1472,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258952+0000","last_change":"2023-09-23T15:34:54.709815+0000","last_active":"2023-09-23T15:36:20.258952+0000","last_peered":"2023-09-23T15:36:20.258952+0000","last_clean":"2023-09-23T15:36:20.258952+0000","last_became_active":"2023-09-23T15:27:42.382617+0000","last_became_peered":"2023-09-23T15:27:42.382617+0000","last_unstale":"2023-09-23T15:36:20.258952+0000","last_undegraded":"2023-09-23T15:36:20.258952+0000","last_fullsized":"2023-09-23T15:36:20.258952+0000","mapping_epoch":375,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":376,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":207,"log_dups_size":0,"ondisk_log_size":207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:29.358447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00038879700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1613,"num_read_kb":2033,"num_write":161,"num_write_kb":409,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":418,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.390516+0000","last_change":"2023-09-23T15:35:28.685172+0000","last_active":"2023-09-23T15:36:19.390516+0000","last_peered":"2023-09-23T15:36:19.390516+0000","last_clean":"2023-09-23T15:36:19.390516+0000","last_became_active":"2023-09-23T15:35:28.684751+0000","last_became_peered":"2023-09-23T15:35:28.684751+0000","last_unstale":"2023-09-23T15:36:19.390516+0000","last_undegraded":"2023-09-23T15:36:19.390516+0000","last_fullsized":"2023-09-23T15:36:19.390516+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:54.436320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":519,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764621+0000","last_change":"2023-09-23T15:35:28.690179+0000","last_active":"2023-09-23T15:36:18.764621+0000","last_peered":"2023-09-23T15:36:18.764621+0000","last_clean":"2023-09-23T15:36:18.764621+0000","last_became_active":"2023-09-23T15:35:28.689871+0000","last_became_peered":"2023-09-23T15:35:28.689871+0000","last_unstale":"2023-09-23T15:36:18.764621+0000","last_undegraded":"2023-09-23T15:36:18.764621+0000","last_fullsized":"2023-09-23T15:36:18.764621+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:27.075241+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"91'161","reported_seq":809,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.260233+0000","last_change":"2023-09-23T15:27:26.319148+0000","last_active":"2023-09-23T15:36:20.260233+0000","last_peered":"2023-09-23T15:36:20.260233+0000","last_clean":"2023-09-23T15:36:20.260233+0000","last_became_active":"2023-09-23T15:27:26.318738+0000","last_became_peered":"2023-09-23T15:27:26.318738+0000","last_unstale":"2023-09-23T15:36:20.260233+0000","last_undegraded":"2023-09-23T15:36:20.260233+0000","last_fullsized":"2023-09-23T15:36:20.260233+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:26:58.686717+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:26:58.686717+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.815924+0000","scrub_duration":0.0014065569999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"68'2","reported_seq":292,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039201+0000","last_change":"2023-09-23T15:27:41.422640+0000","last_active":"2023-09-23T15:36:19.039201+0000","last_peered":"2023-09-23T15:36:19.039201+0000","last_clean":"2023-09-23T15:36:19.039201+0000","last_became_active":"2023-09-23T15:27:41.422146+0000","last_became_peered":"2023-09-23T15:27:41.422146+0000","last_unstale":"2023-09-23T15:36:19.039201+0000","last_undegraded":"2023-09-23T15:36:19.039201+0000","last_fullsized":"2023-09-23T15:36:19.039201+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:08.199284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"488'320","reported_seq":1857,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779452+0000","last_change":"2023-09-23T15:34:54.710828+0000","last_active":"2023-09-23T15:36:21.779452+0000","last_peered":"2023-09-23T15:36:21.779452+0000","last_clean":"2023-09-23T15:36:21.779452+0000","last_became_active":"2023-09-23T15:27:26.322164+0000","last_became_peered":"2023-09-23T15:27:26.322164+0000","last_unstale":"2023-09-23T15:36:21.779452+0000","last_undegraded":"2023-09-23T15:36:21.779452+0000","last_fullsized":"2023-09-23T15:36:21.779452+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":320,"log_dups_size":0,"ondisk_log_size":320,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:36.779112+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0011064320000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":2324,"num_read_kb":4368,"num_write":216,"num_write_kb":2345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":430,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920063+0000","last_change":"2023-09-23T15:35:52.696068+0000","last_active":"2023-09-23T15:36:18.920063+0000","last_peered":"2023-09-23T15:36:18.920063+0000","last_clean":"2023-09-23T15:36:18.920063+0000","last_became_active":"2023-09-23T15:35:52.695534+0000","last_became_peered":"2023-09-23T15:35:52.695534+0000","last_unstale":"2023-09-23T15:36:18.920063+0000","last_undegraded":"2023-09-23T15:36:18.920063+0000","last_fullsized":"2023-09-23T15:36:18.920063+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:08.843615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"91'161","reported_seq":768,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920163+0000","last_change":"2023-09-23T15:35:52.695854+0000","last_active":"2023-09-23T15:36:18.920163+0000","last_peered":"2023-09-23T15:36:18.920163+0000","last_clean":"2023-09-23T15:36:18.920163+0000","last_became_active":"2023-09-23T15:35:52.695506+0000","last_became_peered":"2023-09-23T15:35:52.695506+0000","last_unstale":"2023-09-23T15:36:18.920163+0000","last_undegraded":"2023-09-23T15:36:18.920163+0000","last_fullsized":"2023-09-23T15:36:18.920163+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":143,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":3,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:16:41.645647+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:16:41.645647+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:01:33.677784+0000","scrub_duration":0.0014733669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"488'157","reported_seq":1130,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.771724+0000","last_change":"2023-09-23T15:34:54.714657+0000","last_active":"2023-09-23T15:36:18.771724+0000","last_peered":"2023-09-23T15:36:18.771724+0000","last_clean":"2023-09-23T15:36:18.771724+0000","last_became_active":"2023-09-23T15:27:41.428427+0000","last_became_peered":"2023-09-23T15:27:41.428427+0000","last_unstale":"2023-09-23T15:36:18.771724+0000","last_undegraded":"2023-09-23T15:36:18.771724+0000","last_fullsized":"2023-09-23T15:36:18.771724+0000","mapping_epoch":374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:55:41.647345+0000","last_clean_scrub_stamp":"2023-09-23T14:55:41.647345+0000","objects_scrubbed":0,"log_size":157,"log_dups_size":0,"ondisk_log_size":157,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:26:56.767434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031768699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1077,"num_read_kb":2328,"num_write":133,"num_write_kb":2319,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"68'1","reported_seq":560,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.764916+0000","last_change":"2023-09-23T15:27:23.325947+0000","last_active":"2023-09-23T15:36:18.764916+0000","last_peered":"2023-09-23T15:36:18.764916+0000","last_clean":"2023-09-23T15:36:18.764916+0000","last_became_active":"2023-09-23T15:27:23.320288+0000","last_became_peered":"2023-09-23T15:27:23.320288+0000","last_unstale":"2023-09-23T15:36:18.764916+0000","last_undegraded":"2023-09-23T15:36:18.764916+0000","last_fullsized":"2023-09-23T15:36:18.764916+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:52.042901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":180,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258967+0000","last_change":"2023-09-23T15:35:28.678721+0000","last_active":"2023-09-23T15:36:20.258967+0000","last_peered":"2023-09-23T15:36:20.258967+0000","last_clean":"2023-09-23T15:36:20.258967+0000","last_became_active":"2023-09-23T15:35:28.678381+0000","last_became_peered":"2023-09-23T15:35:28.678381+0000","last_unstale":"2023-09-23T15:36:20.258967+0000","last_undegraded":"2023-09-23T15:36:20.258967+0000","last_fullsized":"2023-09-23T15:36:20.258967+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:45.972251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"91'161","reported_seq":675,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.919838+0000","last_change":"2023-09-23T15:35:52.691580+0000","last_active":"2023-09-23T15:36:18.919838+0000","last_peered":"2023-09-23T15:36:18.919838+0000","last_clean":"2023-09-23T15:36:18.919838+0000","last_became_active":"2023-09-23T15:35:52.691171+0000","last_became_peered":"2023-09-23T15:35:52.691171+0000","last_unstale":"2023-09-23T15:36:18.919838+0000","last_undegraded":"2023-09-23T15:36:18.919838+0000","last_fullsized":"2023-09-23T15:36:18.919838+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:26.740087+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:26.740087+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:32:33.268462+0000","scrub_duration":0.017140743999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"68'1","reported_seq":58,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:22.117788+0000","last_change":"2023-09-23T15:34:12.777336+0000","last_active":"2023-09-23T15:36:22.117788+0000","last_peered":"2023-09-23T15:36:22.117788+0000","last_clean":"2023-09-23T15:36:22.117788+0000","last_became_active":"2023-09-23T15:34:12.777101+0000","last_became_peered":"2023-09-23T15:34:12.777101+0000","last_unstale":"2023-09-23T15:36:22.117788+0000","last_undegraded":"2023-09-23T15:36:22.117788+0000","last_fullsized":"2023-09-23T15:36:22.117788+0000","mapping_epoch":471,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":472,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:13:25.546836+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":49,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:19.039077+0000","last_change":"2023-09-23T15:34:44.672844+0000","last_active":"2023-09-23T15:36:19.039077+0000","last_peered":"2023-09-23T15:36:19.039077+0000","last_clean":"2023-09-23T15:36:19.039077+0000","last_became_active":"2023-09-23T15:34:44.672523+0000","last_became_peered":"2023-09-23T15:34:44.672523+0000","last_unstale":"2023-09-23T15:36:19.039077+0000","last_undegraded":"2023-09-23T15:36:19.039077+0000","last_fullsized":"2023-09-23T15:36:19.039077+0000","mapping_epoch":478,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":479,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:26.341913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"91'161","reported_seq":644,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920282+0000","last_change":"2023-09-23T15:35:52.694927+0000","last_active":"2023-09-23T15:36:18.920282+0000","last_peered":"2023-09-23T15:36:18.920282+0000","last_clean":"2023-09-23T15:36:18.920282+0000","last_became_active":"2023-09-23T15:35:52.694716+0000","last_became_peered":"2023-09-23T15:35:52.694716+0000","last_unstale":"2023-09-23T15:36:18.920282+0000","last_undegraded":"2023-09-23T15:36:18.920282+0000","last_fullsized":"2023-09-23T15:36:18.920282+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:30:28.373089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:30:28.373089+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:12.464049+0000","scrub_duration":0.011035615,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"68'2","reported_seq":34,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767205+0000","last_change":"2023-09-23T15:35:28.693370+0000","last_active":"2023-09-23T15:36:18.767205+0000","last_peered":"2023-09-23T15:36:18.767205+0000","last_clean":"2023-09-23T15:36:18.767205+0000","last_became_active":"2023-09-23T15:35:28.686120+0000","last_became_peered":"2023-09-23T15:35:28.686120+0000","last_unstale":"2023-09-23T15:36:18.767205+0000","last_undegraded":"2023-09-23T15:36:18.767205+0000","last_fullsized":"2023-09-23T15:36:18.767205+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:19:19.106941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258223+0000","last_change":"2023-09-23T15:27:23.317391+0000","last_active":"2023-09-23T15:36:20.258223+0000","last_peered":"2023-09-23T15:36:20.258223+0000","last_clean":"2023-09-23T15:36:20.258223+0000","last_became_active":"2023-09-23T15:27:23.315998+0000","last_became_peered":"2023-09-23T15:27:23.315998+0000","last_unstale":"2023-09-23T15:36:20.258223+0000","last_undegraded":"2023-09-23T15:36:20.258223+0000","last_fullsized":"2023-09-23T15:36:20.258223+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T15:08:39.558534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"68'2","reported_seq":34,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.767625+0000","last_change":"2023-09-23T15:35:28.696024+0000","last_active":"2023-09-23T15:36:18.767625+0000","last_peered":"2023-09-23T15:36:18.767625+0000","last_clean":"2023-09-23T15:36:18.767625+0000","last_became_active":"2023-09-23T15:35:28.690465+0000","last_became_peered":"2023-09-23T15:35:28.690465+0000","last_unstale":"2023-09-23T15:36:18.767625+0000","last_undegraded":"2023-09-23T15:36:18.767625+0000","last_fullsized":"2023-09-23T15:36:18.767625+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:57:20.906445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":33,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.259089+0000","last_change":"2023-09-23T15:35:10.809180+0000","last_active":"2023-09-23T15:36:20.259089+0000","last_peered":"2023-09-23T15:36:20.259089+0000","last_clean":"2023-09-23T15:36:20.259089+0000","last_became_active":"2023-09-23T15:35:10.808819+0000","last_became_peered":"2023-09-23T15:35:10.808819+0000","last_unstale":"2023-09-23T15:36:20.259089+0000","last_undegraded":"2023-09-23T15:36:20.259089+0000","last_fullsized":"2023-09-23T15:36:20.259089+0000","mapping_epoch":494,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":495,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:22.677054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"91'161","reported_seq":705,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.258804+0000","last_change":"2023-09-23T15:28:58.405483+0000","last_active":"2023-09-23T15:36:20.258804+0000","last_peered":"2023-09-23T15:36:20.258804+0000","last_clean":"2023-09-23T15:36:20.258804+0000","last_became_active":"2023-09-23T15:28:39.659153+0000","last_became_peered":"2023-09-23T15:28:39.659153+0000","last_unstale":"2023-09-23T15:36:20.258804+0000","last_undegraded":"2023-09-23T15:36:20.258804+0000","last_fullsized":"2023-09-23T15:36:20.258804+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_deep_scrub":"91'161","last_deep_scrub_stamp":"2023-09-23T15:28:58.405383+0000","last_clean_scrub_stamp":"2023-09-23T15:28:58.405383+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:38.704297+0000","scrub_duration":0.006184742,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"68'1","reported_seq":421,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920492+0000","last_change":"2023-09-23T15:35:52.693192+0000","last_active":"2023-09-23T15:36:18.920492+0000","last_peered":"2023-09-23T15:36:18.920492+0000","last_clean":"2023-09-23T15:36:18.920492+0000","last_became_active":"2023-09-23T15:35:52.692840+0000","last_became_peered":"2023-09-23T15:35:52.692840+0000","last_unstale":"2023-09-23T15:36:18.920492+0000","last_undegraded":"2023-09-23T15:36:18.920492+0000","last_fullsized":"2023-09-23T15:36:18.920492+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:19.258799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":419,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.920011+0000","last_change":"2023-09-23T15:35:52.692698+0000","last_active":"2023-09-23T15:36:18.920011+0000","last_peered":"2023-09-23T15:36:18.920011+0000","last_clean":"2023-09-23T15:36:18.920011+0000","last_became_active":"2023-09-23T15:35:52.692407+0000","last_became_peered":"2023-09-23T15:35:52.692407+0000","last_unstale":"2023-09-23T15:36:18.920011+0000","last_undegraded":"2023-09-23T15:36:18.920011+0000","last_fullsized":"2023-09-23T15:36:18.920011+0000","mapping_epoch":507,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":508,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T16:10:44.027672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"91'161","reported_seq":701,"reported_epoch":518,"state":"active+clean","last_fresh":"2023-09-23T15:36:21.779186+0000","last_change":"2023-09-23T15:28:57.434264+0000","last_active":"2023-09-23T15:36:21.779186+0000","last_peered":"2023-09-23T15:36:21.779186+0000","last_clean":"2023-09-23T15:36:21.779186+0000","last_became_active":"2023-09-23T15:28:39.655479+0000","last_became_peered":"2023-09-23T15:28:39.655479+0000","last_unstale":"2023-09-23T15:36:21.779186+0000","last_undegraded":"2023-09-23T15:36:21.779186+0000","last_fullsized":"2023-09-23T15:36:21.779186+0000","mapping_epoch":401,"log_start":"0'0","ondisk_log_start":"0'0","created":357,"last_epoch_clean":402,"parent":"0.0","parent_split_bits":5,"last_scrub":"91'161","last_scrub_stamp":"2023-09-23T15:28:57.434116+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:53:07.968655+0000","last_clean_scrub_stamp":"2023-09-23T15:28:57.434116+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:52:39.298667+0000","scrub_duration":0.022556093999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"68'4","reported_seq":505,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:18.765199+0000","last_change":"2023-09-23T15:35:28.693557+0000","last_active":"2023-09-23T15:36:18.765199+0000","last_peered":"2023-09-23T15:36:18.765199+0000","last_clean":"2023-09-23T15:36:18.765199+0000","last_became_active":"2023-09-23T15:35:28.685966+0000","last_became_peered":"2023-09-23T15:35:28.685966+0000","last_unstale":"2023-09-23T15:36:18.765199+0000","last_undegraded":"2023-09-23T15:36:18.765199+0000","last_fullsized":"2023-09-23T15:36:18.765199+0000","mapping_epoch":497,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":498,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:30.315452+0000","last_clean_scrub_stamp":"2023-09-23T14:56:30.315452+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:45.228139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":160,"reported_epoch":517,"state":"active+clean","last_fresh":"2023-09-23T15:36:20.257819+0000","last_change":"2023-09-23T15:27:23.315639+0000","last_active":"2023-09-23T15:36:20.257819+0000","last_peered":"2023-09-23T15:36:20.257819+0000","last_clean":"2023-09-23T15:36:20.257819+0000","last_became_active":"2023-09-23T15:27:23.313680+0000","last_became_peered":"2023-09-23T15:27:23.313680+0000","last_unstale":"2023-09-23T15:36:20.257819+0000","last_undegraded":"2023-09-23T15:36:20.257819+0000","last_fullsized":"2023-09-23T15:36:20.257819+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":368,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T14:56:31.316859+0000","last_clean_scrub_stamp":"2023-09-23T14:56:31.316859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:34:31.724986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":11022,"num_keys_recovered":10,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":53057,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4993,"ondisk_log_size":4993,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11944,"num_read_kb":26378,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":161,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4992,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":374,"seq":1606317768818,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":150444,"kb_used_data":9764,"kb_used_omap":13,"kb_used_meta":140658,"kb_avail":93594708,"statfs":{"total":95995035648,"available":95840980992,"internally_reserved":0,"allocated":9998336,"data_stored":8451230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14242,"internal_metadata":144033886},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":3.2440000000000002,"15min":4.992},"min":{"1min":0.34799999999999998,"5min":0.23599999999999999,"15min":0.21099999999999999},"max":{"1min":0.83299999999999996,"5min":121.012,"15min":121.012},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.626,"5min":3.032,"15min":4.9420000000000002},"min":{"1min":0.42699999999999999,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.84499999999999997,"5min":120.97,"15min":120.97},"last":0.63900000000000001}]},{"osd":1,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":1.2769999999999999,"15min":11.167},"min":{"1min":0.39700000000000002,"5min":0.26900000000000002,"15min":0.24399999999999999},"max":{"1min":0.65300000000000002,"5min":10.093999999999999,"15min":157.096},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":1.1419999999999999,"15min":11.09},"min":{"1min":0.377,"5min":0.27700000000000002,"15min":0.26200000000000001},"max":{"1min":0.89000000000000001,"5min":9.9740000000000002,"15min":157.011},"last":0.67400000000000004}]},{"osd":2,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":7.3710000000000004,"15min":3.343},"min":{"1min":0.34999999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.76000000000000001,"5min":220.875,"15min":220.875},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":7.226,"15min":3.2909999999999999},"min":{"1min":0.38300000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.79100000000000004,"5min":220.84399999999999,"15min":220.84399999999999},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.3460000000000001,"15min":2.9260000000000002},"min":{"1min":0.36799999999999999,"5min":0.32700000000000001,"15min":0.25900000000000001},"max":{"1min":0.90100000000000002,"5min":10.061,"15min":76.019000000000005},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.536,"15min":3.246},"min":{"1min":0.437,"5min":0.314,"15min":0.29799999999999999},"max":{"1min":0.81399999999999995,"5min":12.606999999999999,"15min":76.317999999999998},"last":0.60999999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":1.4470000000000001,"15min":1.6220000000000001},"min":{"1min":0.42899999999999999,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.82999999999999996,"5min":43.737000000000002,"15min":170.15799999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.65600000000000003,"5min":1.633,"15min":1.659},"min":{"1min":0.39600000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.86599999999999999,"5min":43.844000000000001,"15min":170.017},"last":0.69199999999999995}]},{"osd":5,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.91700000000000004,"15min":0.83499999999999996},"min":{"1min":0.36799999999999999,"5min":0.28899999999999998,"15min":0.28299999999999997},"max":{"1min":0.86199999999999999,"5min":7.2910000000000004,"15min":7.2910000000000004},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.71999999999999997,"5min":1.242,"15min":0.99299999999999999},"min":{"1min":0.57299999999999995,"5min":0.35399999999999998,"15min":0.23300000000000001},"max":{"1min":0.95599999999999996,"5min":8.3079999999999998,"15min":8.3079999999999998},"last":0.78400000000000003}]},{"osd":6,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":1.4930000000000001,"15min":2.0579999999999998},"min":{"1min":0.45500000000000002,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.89400000000000002,"5min":45.136000000000003,"15min":262.38099999999997},"last":0.73799999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":1.5589999999999999,"15min":2.0950000000000002},"min":{"1min":0.59399999999999997,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.90600000000000003,"5min":44.945,"15min":261.488},"last":0.75800000000000001}]}]},{"osd":6,"up_from":304,"seq":1305670058161,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":213728,"kb_used_data":5720,"kb_used_omap":13,"kb_used_meta":207986,"kb_avail":93531424,"statfs":{"total":95995035648,"available":95776178176,"internally_reserved":0,"allocated":5857280,"data_stored":4321791,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":212978019},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":2.8839999999999999,"15min":3.2559999999999998},"min":{"1min":0.22500000000000001,"5min":0.217,"15min":0.217},"max":{"1min":0.88200000000000001,"5min":44.853999999999999,"15min":184.53200000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.53100000000000003,"5min":3.4940000000000002,"15min":3.2480000000000002},"min":{"1min":0.29099999999999998,"5min":0.245,"15min":0.23499999999999999},"max":{"1min":0.97299999999999998,"5min":40.959000000000003,"15min":86.227999999999994},"last":0.40799999999999997}]},{"osd":1,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":1.1970000000000001,"15min":2.2269999999999999},"min":{"1min":0.32000000000000001,"5min":0.248,"15min":0.221},"max":{"1min":0.95099999999999996,"5min":7.907,"15min":86.534000000000006},"last":0.438},{"interface":"front","average":{"1min":0.58999999999999997,"5min":1.1200000000000001,"15min":2.0510000000000002},"min":{"1min":0.318,"5min":0.25900000000000001,"15min":0.25900000000000001},"max":{"1min":0.95499999999999996,"5min":6.4909999999999997,"15min":86.144000000000005},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6779999999999999,"15min":2.0699999999999998},"min":{"1min":0.33200000000000002,"5min":0.28000000000000003,"15min":0.26300000000000001},"max":{"1min":1.03,"5min":63.512,"15min":63.512},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.60599999999999998,"5min":2.972,"15min":2.181},"min":{"1min":0.41399999999999998,"5min":0.34200000000000003,"15min":0.307},"max":{"1min":0.98799999999999999,"5min":63.526000000000003,"15min":63.526000000000003},"last":0.51200000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":1.3160000000000001,"15min":2.343},"min":{"1min":0.36099999999999999,"5min":0.30299999999999999,"15min":0.26800000000000002},"max":{"1min":1.006,"5min":11.176,"15min":142.67599999999999},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.57299999999999995,"5min":1.8280000000000001,"15min":2.7429999999999999},"min":{"1min":0.35099999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.83899999999999997,"5min":11.795,"15min":142.541},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.68700000000000006,"15min":1.421},"min":{"1min":0.34100000000000003,"5min":0.26400000000000001,"15min":0.23999999999999999},"max":{"1min":0.69499999999999995,"5min":4.3710000000000004,"15min":128.11600000000001},"last":0.625},{"interface":"front","average":{"1min":0.56699999999999995,"5min":1.254,"15min":1.9319999999999999},"min":{"1min":0.36799999999999999,"5min":0.29299999999999998,"15min":0.26400000000000001},"max":{"1min":0.91800000000000004,"5min":8.2449999999999992,"15min":128.095},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":1.214,"15min":2.6190000000000002},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.248},"max":{"1min":0.74399999999999999,"5min":9.2560000000000002,"15min":130.91300000000001},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.73299999999999998,"15min":2.2149999999999999},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29099999999999998},"max":{"1min":1.0580000000000001,"5min":4.4400000000000004,"15min":130.92599999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:36:00 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.339,"15min":1.2809999999999999},"min":{"1min":0.373,"5min":0.34699999999999998,"15min":0.28000000000000003},"max":{"1min":0.93899999999999995,"5min":10.087,"15min":10.087},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.88300000000000001,"15min":1.2549999999999999},"min":{"1min":0.40300000000000002,"5min":0.374,"15min":0.308},"max":{"1min":1.091,"5min":6.9009999999999998,"15min":6.9009999999999998},"last":0.66600000000000004}]}]},{"osd":1,"up_from":183,"seq":785979015462,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":700544,"kb_used_data":10616,"kb_used_omap":6,"kb_used_meta":689913,"kb_avail":93044608,"statfs":{"total":95995035648,"available":95277678592,"internally_reserved":0,"allocated":10870784,"data_stored":8342958,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":706470987},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34399999999999997,"5min":2.8759999999999999,"15min":3.3079999999999998},"min":{"1min":0.26300000000000001,"5min":0.216,"15min":0.19400000000000001},"max":{"1min":0.44,"5min":67.236000000000004,"15min":101.27800000000001},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.33100000000000002,"5min":2.9449999999999998,"15min":3.6059999999999999},"min":{"1min":0.27400000000000002,"5min":0.23799999999999999,"15min":0.221},"max":{"1min":0.45600000000000002,"5min":64.620999999999995,"15min":101.051},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":3.5099999999999998,"15min":3.5499999999999998},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":0.502,"5min":125.94,"15min":186.304},"last":0.27700000000000002},{"interface":"front","average":{"1min":0.36699999999999999,"5min":3.1389999999999998,"15min":3.1840000000000002},"min":{"1min":0.28999999999999998,"5min":0.23499999999999999,"15min":0.216},"max":{"1min":0.46500000000000002,"5min":126.54600000000001,"15min":186.541},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.268,"15min":2.1160000000000001},"min":{"1min":0.29299999999999998,"5min":0.27300000000000002,"15min":0.22700000000000001},"max":{"1min":0.54300000000000004,"5min":11.896000000000001,"15min":64.540999999999997},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.41499999999999998,"5min":1.2090000000000001,"15min":2.1360000000000001},"min":{"1min":0.35999999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.52600000000000002,"5min":13.98,"15min":67.465000000000003},"last":0.29599999999999999}]},{"osd":4,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":1.367,"15min":2.2850000000000001},"min":{"1min":0.40600000000000003,"5min":0.374,"15min":0.32100000000000001},"max":{"1min":0.88100000000000001,"5min":8.5289999999999999,"15min":72.602000000000004},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":1.165,"15min":1.9139999999999999},"min":{"1min":0.41099999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.77700000000000002,"5min":8.7729999999999997,"15min":73.454999999999998},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 15:36:33 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.3979999999999999,"15min":2.984},"min":{"1min":0.41999999999999998,"5min":0.35299999999999998,"15min":0.312},"max":{"1min":0.94799999999999995,"5min":30.009,"15min":173.18799999999999},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.764,"15min":3.3130000000000002},"min":{"1min":0.42299999999999999,"5min":0.34100000000000003,"15min":0.29099999999999998},"max":{"1min":0.79400000000000004,"5min":29.983000000000001,"15min":172.822},"last":0.55300000000000005}]},{"osd":6,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":1.3759999999999999,"15min":2.4340000000000002},"min":{"1min":0.497,"5min":0.38100000000000001,"15min":0.34899999999999998},"max":{"1min":1.052,"5min":8.7170000000000005,"15min":54.198},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.2010000000000001,"15min":1.657},"min":{"1min":0.61199999999999999,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.82899999999999996,"5min":10.996,"15min":15.605},"last":0.64800000000000002}]},{"osd":7,"last update":"Sat Sep 23 15:37:00 2023","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":1.167,"15min":1.7070000000000001},"min":{"1min":0.54300000000000004,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.99399999999999999,"5min":11.032999999999999,"15min":11.032999999999999},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":1.3839999999999999,"15min":3.323},"min":{"1min":0.53600000000000003,"5min":0.41899999999999998,"15min":0.40899999999999997},"max":{"1min":1.0269999999999999,"5min":8.5730000000000004,"15min":54.223999999999997},"last":0.63100000000000001}]}]},{"osd":0,"up_from":95,"seq":408021893508,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898684,"kb_used_data":14836,"kb_used_omap":8,"kb_used_meta":883831,"kb_avail":92846468,"statfs":{"total":95995035648,"available":95074783232,"internally_reserved":0,"allocated":15192064,"data_stored":12841351,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":905043540},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 15:36:20 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":2.056,"15min":3.9609999999999999},"min":{"1min":0.23599999999999999,"5min":0.20699999999999999,"15min":0.20599999999999999},"max":{"1min":0.61799999999999999,"5min":63.393000000000001,"15min":141.52199999999999},"last":0.434},{"interface":"front","average":{"1min":0.376,"5min":1.9750000000000001,"15min":3.8879999999999999},"min":{"1min":0.23699999999999999,"5min":0.217,"15min":0.21299999999999999},"max":{"1min":0.66400000000000003,"5min":63.417999999999999,"15min":141.62799999999999},"last":0.32300000000000001}]},{"osd":2,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":2.907,"15min":2.3639999999999999},"min":{"1min":0.28899999999999998,"5min":0.23699999999999999,"15min":0.20200000000000001},"max":{"1min":0.48799999999999999,"5min":66.132999999999996,"15min":66.132999999999996},"last":0.30299999999999999},{"interface":"front","average":{"1min":0.38700000000000001,"5min":3.258,"15min":2.3650000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.20300000000000001},"max":{"1min":0.68600000000000005,"5min":68.296999999999997,"15min":68.296999999999997},"last":0.34799999999999998}]},{"osd":3,"last update":"Sat Sep 23 15:36:09 2023","interfaces":[{"interface":"back","average":{"1min":0.40799999999999997,"5min":2.6269999999999998,"15min":4.2489999999999997},"min":{"1min":0.29399999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.59099999999999997,"5min":65.251999999999995,"15min":197.78899999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.46200000000000002,"5min":2.847,"15min":4.1779999999999999},"min":{"1min":0.36399999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68999999999999995,"5min":65.269999999999996,"15min":200.58799999999999},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":2.6080000000000001,"15min":2.2200000000000002},"min":{"1min":0.46000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.84899999999999998,"5min":60.771999999999998,"15min":60.771999999999998},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.512,"15min":2.1880000000000002},"min":{"1min":0.40899999999999997,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.746,"5min":60.725999999999999,"15min":60.725999999999999},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":1.915,"15min":2.056},"min":{"1min":0.44900000000000001,"5min":0.312,"15min":0.28100000000000003},"max":{"1min":0.83699999999999997,"5min":33.158999999999999,"15min":44.57},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.64600000000000002,"5min":2.016,"15min":2.3250000000000002},"min":{"1min":0.43099999999999999,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.78200000000000003,"5min":33.140000000000001,"15min":44.588000000000001},"last":0.68899999999999995}]},{"osd":6,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":2.6640000000000001,"15min":2.536},"min":{"1min":0.42999999999999999,"5min":0.379,"15min":0.379},"max":{"1min":0.91200000000000003,"5min":62.168999999999997,"15min":62.168999999999997},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.624,"5min":2.6880000000000002,"15min":2.7719999999999998},"min":{"1min":0.45100000000000001,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.81899999999999995,"5min":61.972000000000001,"15min":61.972000000000001},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.65600000000000003,"5min":1.9339999999999999,"15min":2.1829999999999998},"min":{"1min":0.51300000000000001,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.88800000000000001,"5min":12.954000000000001,"15min":36.848999999999997},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.64900000000000002,"5min":1.952,"15min":3.2090000000000001},"min":{"1min":0.46899999999999997,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.83299999999999996,"5min":11.445,"15min":36.947000000000003},"last":0.59399999999999997}]}]},{"osd":2,"up_from":105,"seq":450971566458,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":854840,"kb_used_data":10672,"kb_used_omap":11,"kb_used_meta":844148,"kb_avail":92890312,"statfs":{"total":95995035648,"available":95119679488,"internally_reserved":0,"allocated":10928128,"data_stored":8581201,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11504,"internal_metadata":864408336},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":2.883,"15min":4.5609999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.19800000000000001},"max":{"1min":0.56999999999999995,"5min":82.878,"15min":235.517},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":2.855,"15min":4.484},"min":{"1min":0.24099999999999999,"5min":0.17999999999999999,"15min":0.17999999999999999},"max":{"1min":0.64900000000000002,"5min":82.762,"15min":235.33799999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 15:36:35 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.5589999999999999,"15min":2.7919999999999998},"min":{"1min":0.26300000000000001,"5min":0.22700000000000001,"15min":0.21099999999999999},"max":{"1min":0.47999999999999998,"5min":11.468,"15min":71.819000000000003},"last":0.378},{"interface":"front","average":{"1min":0.38200000000000001,"5min":1.4810000000000001,"15min":2.77},"min":{"1min":0.28299999999999997,"5min":0.22900000000000001,"15min":0.20899999999999999},"max":{"1min":0.54600000000000004,"5min":20.048999999999999,"15min":71.805000000000007},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 15:36:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.355,"15min":3.0659999999999998},"min":{"1min":0.26400000000000001,"5min":0.22,"15min":0.216},"max":{"1min":0.93700000000000006,"5min":11.324999999999999,"15min":77.031999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.41199999999999998,"5min":1.5269999999999999,"15min":3.1200000000000001},"min":{"1min":0.316,"5min":0.247,"15min":0.247},"max":{"1min":0.66300000000000003,"5min":11.420999999999999,"15min":76.742000000000004},"last":0.439}]},{"osd":4,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.242,"15min":1.6719999999999999},"min":{"1min":0.39900000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.79700000000000004,"5min":9.6210000000000004,"15min":63.747999999999998},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.58499999999999996,"5min":1.355,"15min":1.831},"min":{"1min":0.40600000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.79100000000000004,"5min":9.3480000000000008,"15min":63.954000000000001},"last":0.53300000000000003}]},{"osd":5,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.175,"15min":2.0739999999999998},"min":{"1min":0.45300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.92600000000000005,"5min":9.9559999999999995,"15min":107.021},"last":0.58999999999999997},{"interface":"front","average":{"1min":0.59899999999999998,"5min":1.204,"15min":2.0699999999999998},"min":{"1min":0.46600000000000003,"5min":0.31,"15min":0.31},"max":{"1min":0.77600000000000002,"5min":9.577,"15min":107.136},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.266,"15min":2.2610000000000001},"min":{"1min":0.41699999999999998,"5min":0.318,"15min":0.318},"max":{"1min":1.006,"5min":10.51,"15min":156.16},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.1459999999999999,"15min":2.0710000000000002},"min":{"1min":0.48599999999999999,"5min":0.33500000000000002,"15min":0.308},"max":{"1min":0.84299999999999997,"5min":9.2680000000000007,"15min":155.226},"last":0.65600000000000003}]},{"osd":7,"last update":"Sat Sep 23 15:36:56 2023","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":1.26,"15min":2.4329999999999998},"min":{"1min":0.5,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.78200000000000003,"5min":9.3200000000000003,"15min":112.70699999999999},"last":0.60999999999999999},{"interface":"front","average":{"1min":0.627,"5min":1.345,"15min":2.77},"min":{"1min":0.44700000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.96999999999999997,"5min":9.9280000000000008,"15min":109.402},"last":0.54800000000000004}]}]},{"osd":3,"up_from":110,"seq":472446402930,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":866316,"kb_used_data":10756,"kb_used_omap":3,"kb_used_meta":855548,"kb_avail":92878836,"statfs":{"total":95995035648,"available":95107928064,"internally_reserved":0,"allocated":11014144,"data_stored":8647987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":876081321},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.35899999999999999,"5min":2.6760000000000002,"15min":3.8170000000000002},"min":{"1min":0.26200000000000001,"5min":0.187,"15min":0.183},"max":{"1min":0.51300000000000001,"5min":51.253,"15min":141.934},"last":0.251},{"interface":"front","average":{"1min":0.375,"5min":3.2280000000000002,"15min":4.3179999999999996},"min":{"1min":0.29099999999999998,"5min":0.19600000000000001,"15min":0.189},"max":{"1min":0.56499999999999995,"5min":53.731000000000002,"15min":141.72800000000001},"last":0.308}]},{"osd":1,"last update":"Sat Sep 23 15:36:34 2023","interfaces":[{"interface":"back","average":{"1min":0.38500000000000001,"5min":1.7290000000000001,"15min":3.1949999999999998},"min":{"1min":0.24399999999999999,"5min":0.215,"15min":0.21299999999999999},"max":{"1min":0.76300000000000001,"5min":16.623000000000001,"15min":126.875},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.381,"15min":3.4289999999999998},"min":{"1min":0.28299999999999997,"5min":0.23100000000000001,"15min":0.20000000000000001},"max":{"1min":2.641,"5min":12.243,"15min":128.249},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.44,"5min":3.0089999999999999,"15min":2.8919999999999999},"min":{"1min":0.32400000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.002,"5min":76.468999999999994,"15min":144.35599999999999},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.47299999999999998,"5min":3.5150000000000001,"15min":3.367},"min":{"1min":0.32000000000000001,"5min":0.25900000000000001,"15min":0.246},"max":{"1min":1.081,"5min":85.108000000000004,"15min":144.494},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":2.2160000000000002,"15min":1.8819999999999999},"min":{"1min":0.41099999999999998,"5min":0.29699999999999999,"15min":0.28999999999999998},"max":{"1min":0.69999999999999996,"5min":44.289000000000001,"15min":44.289000000000001},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":2.097,"15min":1.8049999999999999},"min":{"1min":0.38700000000000001,"5min":0.30299999999999999,"15min":0.27400000000000002},"max":{"1min":0.85599999999999998,"5min":44.055,"15min":44.055},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 15:36:08 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.9419999999999999,"15min":3.1320000000000001},"min":{"1min":0.436,"5min":0.33100000000000002,"15min":0.31},"max":{"1min":0.82999999999999996,"5min":14.083,"15min":148.619},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.627,"5min":1.8620000000000001,"15min":2.9329999999999998},"min":{"1min":0.46899999999999997,"5min":0.32900000000000001,"15min":0.28100000000000003},"max":{"1min":0.81100000000000005,"5min":16.506,"15min":148.56999999999999},"last":0.65000000000000002}]},{"osd":6,"last update":"Sat Sep 23 15:36:49 2023","interfaces":[{"interface":"back","average":{"1min":0.63500000000000001,"5min":2.4550000000000001,"15min":2.714},"min":{"1min":0.436,"5min":0.373,"15min":0.32700000000000001},"max":{"1min":0.82499999999999996,"5min":44.356000000000002,"15min":84.665000000000006},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.64200000000000002,"5min":2.5369999999999999,"15min":2.988},"min":{"1min":0.5,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.77000000000000002,"5min":44.453000000000003,"15min":84.897999999999996},"last":0.69399999999999995}]},{"osd":7,"last update":"Sat Sep 23 15:35:57 2023","interfaces":[{"interface":"back","average":{"1min":0.65500000000000003,"5min":1.514,"15min":7.5990000000000002},"min":{"1min":0.45100000000000001,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.1160000000000001,"5min":9.3729999999999993,"15min":216.56899999999999},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":2.0670000000000002,"15min":8.5},"min":{"1min":0.51000000000000001,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81599999999999995,"5min":14.222,"15min":216.548},"last":0.57299999999999995}]}]},{"osd":4,"up_from":31,"seq":133143986697,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":525284,"kb_used_data":5404,"kb_used_omap":0,"kb_used_meta":519872,"kb_avail":93219868,"statfs":{"total":95995035648,"available":95457144832,"internally_reserved":0,"allocated":5533696,"data_stored":3994168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":532348928},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:40 2023","interfaces":[{"interface":"back","average":{"1min":0.502,"5min":1.3640000000000001,"15min":3.2280000000000002},"min":{"1min":0.32400000000000001,"5min":0.22500000000000001,"15min":0.222},"max":{"1min":0.76900000000000002,"5min":28.805,"15min":162.75999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":0.59299999999999997,"5min":1.6120000000000001,"15min":3.2309999999999999},"min":{"1min":0.40000000000000002,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":1.0009999999999999,"5min":29.167000000000002,"15min":162.34800000000001},"last":0.52700000000000002}]},{"osd":1,"last update":"Sat Sep 23 15:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":1.6160000000000001,"15min":3.5449999999999999},"min":{"1min":0.34100000000000003,"5min":0.26800000000000002,"15min":0.249},"max":{"1min":0.752,"5min":41.354999999999997,"15min":144.76900000000001},"last":0.377},{"interface":"front","average":{"1min":0.54500000000000004,"5min":1.478,"15min":3.609},"min":{"1min":0.33300000000000002,"5min":0.26900000000000002,"15min":0.25},"max":{"1min":0.86399999999999999,"5min":41.343000000000004,"15min":144.58799999999999},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 15:36:40 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":3.2909999999999999,"15min":2.419},"min":{"1min":0.35199999999999998,"5min":0.31900000000000001,"15min":0.27100000000000002},"max":{"1min":0.94799999999999995,"5min":89.980000000000004,"15min":89.980000000000004},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65500000000000003,"5min":3.278,"15min":2.492},"min":{"1min":0.48399999999999999,"5min":0.36499999999999999,"15min":0.30599999999999999},"max":{"1min":0.874,"5min":90.016999999999996,"15min":90.016999999999996},"last":0.58799999999999997}]},{"osd":3,"last update":"Sat Sep 23 15:36:16 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.3109999999999999,"15min":2.3069999999999999},"min":{"1min":0.374,"5min":0.308,"15min":0.308},"max":{"1min":0.82299999999999995,"5min":9.7550000000000008,"15min":112.681},"last":0.497},{"interface":"front","average":{"1min":0.57599999999999996,"5min":1.343,"15min":2.3069999999999999},"min":{"1min":0.39100000000000001,"5min":0.34100000000000003,"15min":0.28499999999999998},"max":{"1min":0.83599999999999997,"5min":7.6500000000000004,"15min":112.621},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 23 15:36:26 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":1.0289999999999999,"15min":1.222},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.221},"max":{"1min":0.82799999999999996,"5min":11.904999999999999,"15min":40.685000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.61799999999999999,"5min":1.0009999999999999,"15min":1.3320000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.246},"max":{"1min":0.81599999999999995,"5min":6.4089999999999998,"15min":38.439},"last":0.61699999999999999}]},{"osd":6,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.65200000000000002,"5min":1.1439999999999999,"15min":2.0179999999999998},"min":{"1min":0.33700000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.84499999999999997,"5min":25.437999999999999,"15min":228.93199999999999},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.67600000000000005,"5min":1.345,"15min":2.2770000000000001},"min":{"1min":0.378,"5min":0.36499999999999999,"15min":0.29499999999999998},"max":{"1min":0.875,"5min":25.768999999999998,"15min":232.54900000000001},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:36:50 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.93200000000000005,"15min":1.8440000000000001},"min":{"1min":0.52000000000000002,"5min":0.36199999999999999,"15min":0.36099999999999999},"max":{"1min":0.90200000000000002,"5min":5.2539999999999996,"15min":183.94800000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.67800000000000005,"5min":1.1100000000000001,"15min":1.742},"min":{"1min":0.42799999999999999,"5min":0.33400000000000002,"15min":0.314},"max":{"1min":0.871,"5min":7.5060000000000002,"15min":183.72399999999999},"last":0.60599999999999998}]}]},{"osd":5,"up_from":39,"seq":167503725060,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":540380,"kb_used_data":5652,"kb_used_omap":0,"kb_used_meta":534720,"kb_avail":93204772,"statfs":{"total":95995035648,"available":95441686528,"internally_reserved":0,"allocated":5787648,"data_stored":4252619,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":547553280},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":2.8759999999999999,"15min":5.7649999999999997},"min":{"1min":0.35499999999999998,"5min":0.23799999999999999,"15min":0.20799999999999999},"max":{"1min":0.91100000000000003,"5min":116.307,"15min":397.84199999999998},"last":0.95799999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":2.669,"15min":5.7279999999999998},"min":{"1min":0.33500000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":1.0469999999999999,"5min":113.917,"15min":397.28100000000001},"last":0.81599999999999995}]},{"osd":1,"last update":"Sat Sep 23 15:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.613,"15min":3.2639999999999998},"min":{"1min":0.33600000000000002,"5min":0.25,"15min":0.23499999999999999},"max":{"1min":0.88400000000000001,"5min":30.427,"15min":193.66499999999999},"last":0.82999999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.54,"15min":2.879},"min":{"1min":0.33100000000000002,"5min":0.27300000000000002,"15min":0.25600000000000001},"max":{"1min":1.075,"5min":30.382999999999999,"15min":193.52099999999999},"last":0.94299999999999995}]},{"osd":2,"last update":"Sat Sep 23 15:36:43 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":3.3820000000000001,"15min":2.8660000000000001},"min":{"1min":0.36899999999999999,"5min":0.35399999999999998,"15min":0.25},"max":{"1min":2.3199999999999998,"5min":107.907,"15min":107.907},"last":0.84299999999999997},{"interface":"front","average":{"1min":0.65800000000000003,"5min":2.9249999999999998,"15min":2.843},"min":{"1min":0.376,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.0940000000000001,"5min":107.938,"15min":107.938},"last":0.91300000000000003}]},{"osd":3,"last update":"Sat Sep 23 15:36:14 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.506,"15min":3.726},"min":{"1min":0.36399999999999999,"5min":0.30499999999999999,"15min":0.28699999999999998},"max":{"1min":0.92700000000000005,"5min":10.000999999999999,"15min":266.70299999999997},"last":0.873},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.466,"15min":3.944},"min":{"1min":0.38200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":1.028,"5min":8.5129999999999999,"15min":266.80399999999997},"last":0.88700000000000001}]},{"osd":4,"last update":"Sat Sep 23 15:36:29 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.98599999999999999,"15min":1.337},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.249},"max":{"1min":0.754,"5min":8.2560000000000002,"15min":14.068},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.60299999999999998,"5min":1.0609999999999999,"15min":1.3640000000000001},"min":{"1min":0.35899999999999999,"5min":0.311,"15min":0.23599999999999999},"max":{"1min":0.93400000000000005,"5min":8.9550000000000001,"15min":19.879999999999999},"last":0.75}]},{"osd":6,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":1.101,"15min":1.6739999999999999},"min":{"1min":0.378,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.92900000000000005,"5min":8.4139999999999997,"15min":35.667000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.55000000000000004,"5min":1.198,"15min":1.724},"min":{"1min":0.33600000000000002,"5min":0.32600000000000001,"15min":0.30499999999999999},"max":{"1min":0.84299999999999997,"5min":7.5039999999999996,"15min":36.140999999999998},"last":0.79300000000000004}]},{"osd":7,"last update":"Sat Sep 23 15:35:58 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.127,"15min":1.2989999999999999},"min":{"1min":0.42599999999999999,"5min":0.33100000000000002,"15min":0.27600000000000002},"max":{"1min":0.88200000000000001,"5min":9.7390000000000008,"15min":16.065999999999999},"last":0.77600000000000002},{"interface":"front","average":{"1min":0.64600000000000002,"5min":1.173,"15min":1.464},"min":{"1min":0.439,"5min":0.378,"15min":0.32100000000000001},"max":{"1min":0.95699999999999996,"5min":8.3320000000000007,"15min":19.908000000000001},"last":0.85699999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2899,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2093,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8620,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7093,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7287,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3927,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13981,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12149,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T15:37:02.238 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T15:37:02.238 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T15:37:02.299 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e5e7fd49546b47c9b5afb84d33b8f91c Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T15:37:02.301 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T15:37:02.311 INFO:tasks.cephadm:Teardown begin 2023-09-23T15:37:02.312 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T15:37:02.384 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T15:37:02.433 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T15:37:02.434 DEBUG:teuthology.orchestra.run.smithi017:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T15:37:02.450 DEBUG:teuthology.orchestra.run.smithi027:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T15:37:02.470 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T15:37:02.470 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T15:37:02.471 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T15:37:02.771 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:02 smithi017 ceph-mon[222878]: pgmap v1526: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:02 smithi017 ceph-mon[222878]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:02 smithi017 systemd[1]: Stopping Ceph mon.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:02 smithi017 ceph-mon[215646]: pgmap v1526: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:02 smithi017 ceph-mon[215646]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[215622]: 2023-09-23T15:37:02.760+0000 7f618f9b4700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:02.772 INFO:journalctl@ceph.mon.a.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[215622]: 2023-09-23T15:37:02.760+0000 7f618f9b4700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T15:37:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:02 smithi027 ceph-mon[195233]: pgmap v1526: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T15:37:03.099 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:02 smithi027 ceph-mon[195233]: from='client.47909 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T15:37:03.192 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE Bus STOPPING 2023-09-23T15:37:03.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T15:37:03.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE Bus STOPPED 2023-09-23T15:37:03.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE Bus STARTING 2023-09-23T15:37:03.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE Serving on http://:::9283 2023-09-23T15:37:03.193 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:02 smithi017 conmon[202315]: [23/Sep/2023:15:37:02] ENGINE Bus STARTED 2023-09-23T15:37:03.757 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T15:37:03.799 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:03.799 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T15:37:03.799 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T15:37:03.800 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T15:37:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:03 smithi017 systemd[1]: Stopping Ceph mon.c for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:04 smithi017 conmon[222854]: 2023-09-23T15:37:04.104+0000 7fd6a7cd2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:04.193 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:04 smithi017 conmon[222854]: 2023-09-23T15:37:04.104+0000 7fd6a7cd2700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T15:37:04.942 INFO:journalctl@ceph.mon.c.smithi017.stdout:Sep 23 15:37:04 smithi017 bash[705714]: 7f42350c36be2dea8073ca37fbfdf2fb1e4fa380d38de7adf8ddcfd2cfb07f9a 2023-09-23T15:37:05.090 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T15:37:05.128 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:05.129 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T15:37:05.129 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T15:37:05.129 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T15:37:05.437 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:05 smithi027 systemd[1]: Stopping Ceph mon.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:05.437 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:05 smithi027 conmon[195209]: 2023-09-23T15:37:05.434+0000 7f8b23920700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:05.437 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:05 smithi027 conmon[195209]: 2023-09-23T15:37:05.434+0000 7f8b23920700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T15:37:05.744 INFO:journalctl@ceph.mon.b.smithi027.stdout:Sep 23 15:37:05 smithi027 bash[408996]: 0f598c97295ecd84227db90daafe9d4c00bc70a6151107bdeacef5b1c35eb71b 2023-09-23T15:37:06.188 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T15:37:06.229 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:06.229 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T15:37:06.230 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T15:37:06.230 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T15:37:06.548 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:06 smithi017 systemd[1]: Stopping Ceph mgr.y for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:06.942 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:06 smithi017 conmon[202315]: 2023-09-23T15:37:06.546+0000 7fdcf412b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:06.943 INFO:journalctl@ceph.mgr.y.smithi017.stdout:Sep 23 15:37:06 smithi017 conmon[202315]: 2023-09-23T15:37:06.546+0000 7fdcf412b700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2023-09-23T15:37:07.418 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T15:37:07.456 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:07.457 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T15:37:07.457 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T15:37:07.457 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T15:37:07.850 INFO:journalctl@ceph.mgr.x.smithi027.stdout:Sep 23 15:37:07 smithi027 systemd[1]: Stopping Ceph mgr.x for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:08.422 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T15:37:08.461 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:08.461 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T15:37:08.462 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T15:37:08.462 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T15:37:08.747 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:37:08 smithi017 systemd[1]: Stopping Ceph osd.0 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:09.192 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:37:08 smithi017 conmon[239766]: 2023-09-23T15:37:08.746+0000 7f6286187700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:09.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:37:08 smithi017 conmon[239766]: 2023-09-23T15:37:08.746+0000 7f6286187700 -1 osd.0 518 *** Got signal Terminated *** 2023-09-23T15:37:09.193 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:37:08 smithi017 conmon[239766]: 2023-09-23T15:37:08.746+0000 7f6286187700 -1 osd.0 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:14.693 INFO:journalctl@ceph.osd.0.smithi017.stdout:Sep 23 15:37:14 smithi017 bash[706316]: 17a5480512e73459fc01874233b5e9c364916159607356e6c38cd77108c0bb4d 2023-09-23T15:37:16.680 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T15:37:16.724 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:16.724 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T15:37:16.724 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T15:37:16.724 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T15:37:16.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:37:16 smithi017 systemd[1]: Stopping Ceph osd.1 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:17.443 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:37:17 smithi017 conmon[360957]: 2023-09-23T15:37:17.029+0000 7f1b0ea26700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:17.443 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:37:17 smithi017 conmon[360957]: 2023-09-23T15:37:17.029+0000 7f1b0ea26700 -1 osd.1 518 *** Got signal Terminated *** 2023-09-23T15:37:17.443 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:37:17 smithi017 conmon[360957]: 2023-09-23T15:37:17.029+0000 7f1b0ea26700 -1 osd.1 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:22.943 INFO:journalctl@ceph.osd.1.smithi017.stdout:Sep 23 15:37:22 smithi017 bash[706897]: 5922ab9bf4a26db51d2e36e3fc0ff3c815bef536210e27fca85e75cce4904bf3 2023-09-23T15:37:24.662 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T15:37:24.706 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:24.706 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T15:37:24.706 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T15:37:24.707 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T15:37:24.942 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:37:24 smithi017 systemd[1]: Stopping Ceph osd.2 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:25.443 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:37:25 smithi017 conmon[255631]: 2023-09-23T15:37:24.999+0000 7f3165403700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:25.443 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:37:25 smithi017 conmon[255631]: 2023-09-23T15:37:24.999+0000 7f3165403700 -1 osd.2 518 *** Got signal Terminated *** 2023-09-23T15:37:25.443 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:37:25 smithi017 conmon[255631]: 2023-09-23T15:37:24.999+0000 7f3165403700 -1 osd.2 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:31.442 INFO:journalctl@ceph.osd.2.smithi017.stdout:Sep 23 15:37:31 smithi017 bash[707477]: 49117382ccb9167dcbc15867e49e748d7987ce5bf36b9a6c48611971edd93dab 2023-09-23T15:37:32.754 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T15:37:32.799 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:32.799 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T15:37:32.799 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T15:37:32.799 DEBUG:teuthology.orchestra.run.smithi017:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T15:37:33.127 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:32 smithi017 systemd[1]: Stopping Ceph osd.3 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:33.442 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:33 smithi017 conmon[263234]: 2023-09-23T15:37:33.126+0000 7fe1e1c7f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:33.443 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:33 smithi017 conmon[263234]: 2023-09-23T15:37:33.126+0000 7fe1e1c7f700 -1 osd.3 518 *** Got signal Terminated *** 2023-09-23T15:37:33.443 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:33 smithi017 conmon[263234]: 2023-09-23T15:37:33.126+0000 7fe1e1c7f700 -1 osd.3 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:35.639 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:35 smithi027 conmon[311800]: 2023-09-23T15:37:35.366+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:36.097 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:35 smithi027 conmon[344740]: 2023-09-23T15:37:35.637+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:36.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:36 smithi027 conmon[122090]: 2023-09-23T15:37:36.096+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:36.647 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:36 smithi027 conmon[311800]: 2023-09-23T15:37:36.361+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:37.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:36 smithi027 conmon[344740]: 2023-09-23T15:37:36.646+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:37.402 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:37 smithi027 conmon[122090]: 2023-09-23T15:37:37.101+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:37.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:37 smithi027 conmon[311800]: 2023-09-23T15:37:37.401+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:37.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:37 smithi027 conmon[344740]: 2023-09-23T15:37:37.648+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:37.850 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:37 smithi027 conmon[113211]: 2023-09-23T15:37:37.404+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:38.192 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:37 smithi017 conmon[263234]: 2023-09-23T15:37:37.935+0000 7fe1de275700 -1 osd.3 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:13.528425+0000 front 2023-09-23T15:37:13.528619+0000 (oldest deadline 2023-09-23T15:37:37.028745+0000) 2023-09-23T15:37:38.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:38 smithi027 conmon[122090]: 2023-09-23T15:37:38.053+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:38.666 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:38 smithi027 conmon[113211]: 2023-09-23T15:37:38.413+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:38.667 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:38 smithi027 conmon[311800]: 2023-09-23T15:37:38.411+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:39.036 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:38 smithi027 conmon[344740]: 2023-09-23T15:37:38.665+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:39.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:39 smithi027 conmon[122090]: 2023-09-23T15:37:39.035+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:39.443 INFO:journalctl@ceph.osd.3.smithi017.stdout:Sep 23 15:37:38 smithi017 bash[708048]: 3f1a3bdcd680e96e885d29acb5e2f243b764d37811cc873a004ae5d71046fe0e 2023-09-23T15:37:39.687 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:39 smithi027 conmon[113211]: 2023-09-23T15:37:39.439+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:39.688 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:39 smithi027 conmon[311800]: 2023-09-23T15:37:39.436+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:40.028 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:39 smithi027 conmon[344740]: 2023-09-23T15:37:39.686+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:40.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:40 smithi027 conmon[122090]: 2023-09-23T15:37:40.028+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:40.670 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:40 smithi027 conmon[113211]: 2023-09-23T15:37:40.411+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:40.671 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:40 smithi027 conmon[311800]: 2023-09-23T15:37:40.482+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:40.902 DEBUG:teuthology.orchestra.run.smithi017:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T15:37:40.945 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:40.946 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T15:37:40.946 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T15:37:40.946 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T15:37:40.992 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:40 smithi027 conmon[344740]: 2023-09-23T15:37:40.670+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:41.264 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[122090]: 2023-09-23T15:37:41.029+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:41.264 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:41 smithi027 systemd[1]: Stopping Ceph osd.4 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:41.599 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[311800]: 2023-09-23T15:37:41.442+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:41.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[113211]: 2023-09-23T15:37:41.263+0000 7fae65afa700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:41.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[113211]: 2023-09-23T15:37:41.263+0000 7fae65afa700 -1 osd.4 518 *** Got signal Terminated *** 2023-09-23T15:37:41.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[113211]: 2023-09-23T15:37:41.263+0000 7fae65afa700 -1 osd.4 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:41.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[113211]: 2023-09-23T15:37:41.406+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:42.043 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:41 smithi027 conmon[344740]: 2023-09-23T15:37:41.645+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:42.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[122090]: 2023-09-23T15:37:42.042+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:42.676 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[113211]: 2023-09-23T15:37:42.421+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:42.677 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[311800]: 2023-09-23T15:37:42.456+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:42.994 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[344740]: 2023-09-23T15:37:42.676+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:43.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[122090]: 2023-09-23T15:37:42.993+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:43.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:42 smithi027 conmon[122090]: 2023-09-23T15:37:42.993+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:43.642 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:43 smithi027 conmon[113211]: 2023-09-23T15:37:43.380+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:43.642 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:43 smithi027 conmon[311800]: 2023-09-23T15:37:43.491+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:43.994 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:43 smithi027 conmon[344740]: 2023-09-23T15:37:43.641+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:44.333 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:43 smithi027 conmon[122090]: 2023-09-23T15:37:43.993+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:44.334 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:43 smithi027 conmon[122090]: 2023-09-23T15:37:43.993+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:44.599 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:44 smithi027 conmon[113211]: 2023-09-23T15:37:44.332+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:44.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:44 smithi027 conmon[311800]: 2023-09-23T15:37:44.536+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:45.026 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:44 smithi027 conmon[344740]: 2023-09-23T15:37:44.666+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:45.346 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[122090]: 2023-09-23T15:37:45.025+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:45.347 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[122090]: 2023-09-23T15:37:45.025+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:45.599 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[113211]: 2023-09-23T15:37:45.346+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:12.277013+0000 front 2023-09-23T15:37:12.277081+0000 (oldest deadline 2023-09-23T15:37:36.976884+0000) 2023-09-23T15:37:45.600 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[113211]: 2023-09-23T15:37:45.346+0000 7fae628f9700 -1 osd.4 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:20.978642+0000 front 2023-09-23T15:37:20.978579+0000 (oldest deadline 2023-09-23T15:37:45.078386+0000) 2023-09-23T15:37:45.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[311800]: 2023-09-23T15:37:45.551+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:45.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[311800]: 2023-09-23T15:37:45.551+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:46.026 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:45 smithi027 conmon[344740]: 2023-09-23T15:37:45.652+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:46.350 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:46 smithi027 conmon[122090]: 2023-09-23T15:37:46.025+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:46.350 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:46 smithi027 conmon[122090]: 2023-09-23T15:37:46.025+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:46.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:46 smithi027 conmon[344740]: 2023-09-23T15:37:46.698+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:46.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:46 smithi027 conmon[311800]: 2023-09-23T15:37:46.545+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:46.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:46 smithi027 conmon[311800]: 2023-09-23T15:37:46.545+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:47.306 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[122090]: 2023-09-23T15:37:47.071+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:47.306 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[122090]: 2023-09-23T15:37:47.071+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:47.599 INFO:journalctl@ceph.osd.4.smithi027.stdout:Sep 23 15:37:47 smithi027 bash[409619]: f59f399d4c7db4f125d720097b086e283201893f075dceee685e364678cd3284 2023-09-23T15:37:47.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[311800]: 2023-09-23T15:37:47.501+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:47.600 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[311800]: 2023-09-23T15:37:47.502+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:48.090 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[344740]: 2023-09-23T15:37:47.735+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:48.090 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:47 smithi027 conmon[344740]: 2023-09-23T15:37:47.735+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:48.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[122090]: 2023-09-23T15:37:48.089+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:48.349 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[122090]: 2023-09-23T15:37:48.089+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:48.752 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[311800]: 2023-09-23T15:37:48.545+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:48.753 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[311800]: 2023-09-23T15:37:48.545+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:48.929 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T15:37:48.971 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:48.971 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T15:37:48.972 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T15:37:48.972 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T15:37:49.035 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[344740]: 2023-09-23T15:37:48.751+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:49.036 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:48 smithi027 conmon[344740]: 2023-09-23T15:37:48.751+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:49.288 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 systemd[1]: Stopping Ceph osd.5 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:49.288 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[122090]: 2023-09-23T15:37:49.103+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:49.288 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[122090]: 2023-09-23T15:37:49.103+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:49.288 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[122090]: 2023-09-23T15:37:49.287+0000 7ffb27ddd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:49.581 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[122090]: 2023-09-23T15:37:49.287+0000 7ffb27ddd700 -1 osd.5 518 *** Got signal Terminated *** 2023-09-23T15:37:49.581 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[122090]: 2023-09-23T15:37:49.287+0000 7ffb27ddd700 -1 osd.5 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:49.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[311800]: 2023-09-23T15:37:49.580+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:49.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[311800]: 2023-09-23T15:37:49.580+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:49.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[344740]: 2023-09-23T15:37:49.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:49.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:49 smithi027 conmon[344740]: 2023-09-23T15:37:49.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:50.569 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[122090]: 2023-09-23T15:37:50.141+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:50.570 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[122090]: 2023-09-23T15:37:50.141+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:50.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[311800]: 2023-09-23T15:37:50.568+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:50.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[311800]: 2023-09-23T15:37:50.568+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:50.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[344740]: 2023-09-23T15:37:50.717+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:50.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:50 smithi027 conmon[344740]: 2023-09-23T15:37:50.717+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:51.528 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[122090]: 2023-09-23T15:37:51.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:51.528 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[122090]: 2023-09-23T15:37:51.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:51.529 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[122090]: 2023-09-23T15:37:51.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:29.870242+0000 front 2023-09-23T15:37:29.870260+0000 (oldest deadline 2023-09-23T15:37:50.970245+0000) 2023-09-23T15:37:51.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[311800]: 2023-09-23T15:37:51.528+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:51.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[311800]: 2023-09-23T15:37:51.528+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:51.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[344740]: 2023-09-23T15:37:51.707+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:51.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:51 smithi027 conmon[344740]: 2023-09-23T15:37:51.707+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:52.485 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[122090]: 2023-09-23T15:37:52.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:52.485 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[122090]: 2023-09-23T15:37:52.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:52.485 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[122090]: 2023-09-23T15:37:52.183+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:29.870242+0000 front 2023-09-23T15:37:29.870260+0000 (oldest deadline 2023-09-23T15:37:50.970245+0000) 2023-09-23T15:37:52.752 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[311800]: 2023-09-23T15:37:52.484+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:52.752 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[311800]: 2023-09-23T15:37:52.484+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:53.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[344740]: 2023-09-23T15:37:52.750+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:53.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[344740]: 2023-09-23T15:37:52.750+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:53.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:52 smithi027 conmon[344740]: 2023-09-23T15:37:52.750+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:53.535 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[122090]: 2023-09-23T15:37:53.232+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:53.535 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[122090]: 2023-09-23T15:37:53.232+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:53.535 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[122090]: 2023-09-23T15:37:53.232+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:29.870242+0000 front 2023-09-23T15:37:29.870260+0000 (oldest deadline 2023-09-23T15:37:50.970245+0000) 2023-09-23T15:37:53.801 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[311800]: 2023-09-23T15:37:53.534+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:53.801 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[311800]: 2023-09-23T15:37:53.534+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:53.802 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[311800]: 2023-09-23T15:37:53.534+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:54.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[344740]: 2023-09-23T15:37:53.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:54.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[344740]: 2023-09-23T15:37:53.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:54.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:53 smithi027 conmon[344740]: 2023-09-23T15:37:53.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:54.501 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[122090]: 2023-09-23T15:37:54.204+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.367494+0000 front 2023-09-23T15:37:09.367470+0000 (oldest deadline 2023-09-23T15:37:35.267450+0000) 2023-09-23T15:37:54.501 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[122090]: 2023-09-23T15:37:54.204+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.168645+0000 front 2023-09-23T15:37:21.168693+0000 (oldest deadline 2023-09-23T15:37:42.868702+0000) 2023-09-23T15:37:54.501 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[122090]: 2023-09-23T15:37:54.204+0000 7ffb223d7700 -1 osd.5 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:29.870242+0000 front 2023-09-23T15:37:29.870260+0000 (oldest deadline 2023-09-23T15:37:50.970245+0000) 2023-09-23T15:37:54.753 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[311800]: 2023-09-23T15:37:54.500+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:54.754 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[311800]: 2023-09-23T15:37:54.500+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:54.754 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[311800]: 2023-09-23T15:37:54.500+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:55.029 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[344740]: 2023-09-23T15:37:54.752+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:55.029 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[344740]: 2023-09-23T15:37:54.752+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:55.029 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:54 smithi027 conmon[344740]: 2023-09-23T15:37:54.752+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:55.794 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:55 smithi027 bash[410198]: 6daba84c8455869c91f7e7f7a0711d43a020e878fbe0c19f7f6e6b17c1e61058 2023-09-23T15:37:55.795 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[311800]: 2023-09-23T15:37:55.479+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:55.795 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[311800]: 2023-09-23T15:37:55.479+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:55.795 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[311800]: 2023-09-23T15:37:55.479+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:56.061 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[344740]: 2023-09-23T15:37:55.793+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:56.062 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[344740]: 2023-09-23T15:37:55.793+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:56.062 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:55 smithi027 conmon[344740]: 2023-09-23T15:37:55.793+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:56.820 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[311800]: 2023-09-23T15:37:56.497+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:56.821 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[311800]: 2023-09-23T15:37:56.497+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:56.821 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[311800]: 2023-09-23T15:37:56.497+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:57.035 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T15:37:57.099 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:57 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.5.service: Succeeded. 2023-09-23T15:37:57.099 INFO:journalctl@ceph.osd.5.smithi027.stdout:Sep 23 15:37:57 smithi027 systemd[1]: Stopped Ceph osd.5 for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:37:57.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[344740]: 2023-09-23T15:37:56.820+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:57.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[344740]: 2023-09-23T15:37:56.820+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:57.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:56 smithi027 conmon[344740]: 2023-09-23T15:37:56.820+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:57.119 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:37:57.161 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T15:37:57.161 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T15:37:57.161 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T15:37:57.451 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 systemd[1]: Stopping Ceph osd.6 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:37:57.779 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.449+0000 7f98c6328700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:37:57.780 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.449+0000 7f98c6328700 -1 osd.6 518 *** Got signal Terminated *** 2023-09-23T15:37:57.780 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.449+0000 7f98c6328700 -1 osd.6 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:37:57.780 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.505+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:57.780 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.505+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:57.780 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[311800]: 2023-09-23T15:37:57.505+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:58.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[344740]: 2023-09-23T15:37:57.779+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:58.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[344740]: 2023-09-23T15:37:57.779+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:58.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:57 smithi027 conmon[344740]: 2023-09-23T15:37:57.779+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:58.787 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[311800]: 2023-09-23T15:37:58.524+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:58.787 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[311800]: 2023-09-23T15:37:58.524+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:58.788 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[311800]: 2023-09-23T15:37:58.524+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:59.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[344740]: 2023-09-23T15:37:58.786+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:37:59.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[344740]: 2023-09-23T15:37:58.786+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:37:59.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:58 smithi027 conmon[344740]: 2023-09-23T15:37:58.786+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:37:59.776 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[311800]: 2023-09-23T15:37:59.523+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:37:59.777 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[311800]: 2023-09-23T15:37:59.523+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:37:59.777 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[311800]: 2023-09-23T15:37:59.523+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:37:59.777 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[179123]: ts=2023-09-23T15:37:59.687Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.17:8765: connect: connection refused" 2023-09-23T15:37:59.778 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[179123]: ts=2023-09-23T15:37:59.687Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2023-09-23T15:37:59.778 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[179123]: ts=2023-09-23T15:37:59.687Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.17:8765: connect: connection refused" 2023-09-23T15:37:59.778 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[179123]: ts=2023-09-23T15:37:59.687Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.17:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.17:8765: connect: connection refused" 2023-09-23T15:38:00.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[344740]: 2023-09-23T15:37:59.775+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:00.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[344740]: 2023-09-23T15:37:59.775+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:00.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:37:59 smithi027 conmon[344740]: 2023-09-23T15:37:59.775+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:00.849 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[311800]: 2023-09-23T15:38:00.560+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:38:00.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[311800]: 2023-09-23T15:38:00.560+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:38:00.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[311800]: 2023-09-23T15:38:00.560+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:38:00.850 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[311800]: 2023-09-23T15:38:00.560+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.943521+0000 front 2023-09-23T15:37:36.943455+0000 (oldest deadline 2023-09-23T15:37:59.843277+0000) 2023-09-23T15:38:00.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[344740]: 2023-09-23T15:38:00.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:00.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[344740]: 2023-09-23T15:38:00.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:00.851 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[344740]: 2023-09-23T15:38:00.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:00.851 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:00 smithi027 conmon[344740]: 2023-09-23T15:38:00.800+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:02.099 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[311800]: 2023-09-23T15:38:01.603+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:09.340178+0000 front 2023-09-23T15:37:09.339885+0000 (oldest deadline 2023-09-23T15:37:35.239928+0000) 2023-09-23T15:38:02.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[311800]: 2023-09-23T15:38:01.603+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.141028+0000 front 2023-09-23T15:37:21.141061+0000 (oldest deadline 2023-09-23T15:37:44.641004+0000) 2023-09-23T15:38:02.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[311800]: 2023-09-23T15:38:01.603+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.142395+0000 front 2023-09-23T15:37:28.142332+0000 (oldest deadline 2023-09-23T15:37:53.442290+0000) 2023-09-23T15:38:02.100 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[311800]: 2023-09-23T15:38:01.603+0000 7f98c4129700 -1 osd.6 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.943521+0000 front 2023-09-23T15:37:36.943455+0000 (oldest deadline 2023-09-23T15:37:59.843277+0000) 2023-09-23T15:38:02.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[344740]: 2023-09-23T15:38:01.804+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:02.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[344740]: 2023-09-23T15:38:01.804+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:02.101 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[344740]: 2023-09-23T15:38:01.804+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:02.101 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:01 smithi027 conmon[344740]: 2023-09-23T15:38:01.804+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:03.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:02 smithi027 conmon[344740]: 2023-09-23T15:38:02.761+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:03.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:02 smithi027 conmon[344740]: 2023-09-23T15:38:02.761+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:03.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:02 smithi027 conmon[344740]: 2023-09-23T15:38:02.761+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:03.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:02 smithi027 conmon[344740]: 2023-09-23T15:38:02.761+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:03.599 INFO:journalctl@ceph.osd.6.smithi027.stdout:Sep 23 15:38:03 smithi027 bash[410777]: 1a318744f186dcb14734b7fc136197ab73a5c3997ba2e222a9643bb461db7a96 2023-09-23T15:38:04.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:03 smithi027 conmon[344740]: 2023-09-23T15:38:03.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:04.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:03 smithi027 conmon[344740]: 2023-09-23T15:38:03.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:04.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:03 smithi027 conmon[344740]: 2023-09-23T15:38:03.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:04.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:03 smithi027 conmon[344740]: 2023-09-23T15:38:03.728+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:04.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:04 smithi027 conmon[344740]: 2023-09-23T15:38:04.755+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:04.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:04 smithi027 conmon[344740]: 2023-09-23T15:38:04.755+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:04.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:04 smithi027 conmon[344740]: 2023-09-23T15:38:04.755+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:04.850 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:04 smithi027 conmon[344740]: 2023-09-23T15:38:04.755+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:05.093 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T15:38:05.134 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:38:05.134 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T15:38:05.135 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T15:38:05.135 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T15:38:05.349 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 systemd[1]: Stopping Ceph osd.7 for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:05.768 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.451+0000 7f34fb4ca700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T15:38:05.769 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.451+0000 7f34fb4ca700 -1 osd.7 518 *** Got signal Terminated *** 2023-09-23T15:38:05.769 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.451+0000 7f34fb4ca700 -1 osd.7 518 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T15:38:06.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.768+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:06.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.768+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:06.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.768+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:06.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:05 smithi027 conmon[344740]: 2023-09-23T15:38:05.768+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:07.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:06 smithi027 conmon[344740]: 2023-09-23T15:38:06.770+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:07.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:06 smithi027 conmon[344740]: 2023-09-23T15:38:06.770+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:07.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:06 smithi027 conmon[344740]: 2023-09-23T15:38:06.770+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:07.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:06 smithi027 conmon[344740]: 2023-09-23T15:38:06.770+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:08.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:07 smithi027 conmon[344740]: 2023-09-23T15:38:07.732+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:08.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:07 smithi027 conmon[344740]: 2023-09-23T15:38:07.732+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:08.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:07 smithi027 conmon[344740]: 2023-09-23T15:38:07.732+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:08.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:07 smithi027 conmon[344740]: 2023-09-23T15:38:07.732+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:08.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:07 smithi027 conmon[344740]: 2023-09-23T15:38:07.732+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.27:6804 osd.4 since back 2023-09-23T15:37:45.975945+0000 front 2023-09-23T15:37:45.975989+0000 (oldest deadline 2023-09-23T15:38:07.675906+0000) 2023-09-23T15:38:09.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:08 smithi027 conmon[344740]: 2023-09-23T15:38:08.769+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:09.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:08 smithi027 conmon[344740]: 2023-09-23T15:38:08.769+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:09.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:08 smithi027 conmon[344740]: 2023-09-23T15:38:08.769+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:09.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:08 smithi027 conmon[344740]: 2023-09-23T15:38:08.769+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:09.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:08 smithi027 conmon[344740]: 2023-09-23T15:38:08.769+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.27:6804 osd.4 since back 2023-09-23T15:37:45.975945+0000 front 2023-09-23T15:37:45.975989+0000 (oldest deadline 2023-09-23T15:38:07.675906+0000) 2023-09-23T15:38:10.099 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:09 smithi027 conmon[344740]: 2023-09-23T15:38:09.763+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6806 osd.0 since back 2023-09-23T15:37:11.670800+0000 front 2023-09-23T15:37:11.670784+0000 (oldest deadline 2023-09-23T15:37:35.170633+0000) 2023-09-23T15:38:10.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:09 smithi027 conmon[344740]: 2023-09-23T15:38:09.763+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6814 osd.1 since back 2023-09-23T15:37:21.572164+0000 front 2023-09-23T15:37:21.572390+0000 (oldest deadline 2023-09-23T15:37:46.872175+0000) 2023-09-23T15:38:10.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:09 smithi027 conmon[344740]: 2023-09-23T15:38:09.763+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6822 osd.2 since back 2023-09-23T15:37:28.573440+0000 front 2023-09-23T15:37:28.573351+0000 (oldest deadline 2023-09-23T15:37:52.673089+0000) 2023-09-23T15:38:10.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:09 smithi027 conmon[344740]: 2023-09-23T15:38:09.763+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.17:6830 osd.3 since back 2023-09-23T15:37:36.074665+0000 front 2023-09-23T15:37:36.074691+0000 (oldest deadline 2023-09-23T15:38:00.075082+0000) 2023-09-23T15:38:10.100 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:09 smithi027 conmon[344740]: 2023-09-23T15:38:09.763+0000 7f34f92cb700 -1 osd.7 518 heartbeat_check: no reply from 172.21.15.27:6804 osd.4 since back 2023-09-23T15:37:45.975945+0000 front 2023-09-23T15:37:45.975989+0000 (oldest deadline 2023-09-23T15:38:07.675906+0000) 2023-09-23T15:38:11.349 INFO:journalctl@ceph.osd.7.smithi027.stdout:Sep 23 15:38:11 smithi027 bash[411359]: cececd22c33f4a13a2aefed21c690cd18089ef8b0683ef6d1319a6ef1db39859 2023-09-23T15:38:12.767 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T15:38:12.807 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:38:12.807 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T15:38:12.807 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T15:38:12.808 DEBUG:teuthology.orchestra.run.smithi027:> sudo systemctl stop ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T15:38:13.090 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:12 smithi027 systemd[1]: Stopping Ceph prometheus.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:13.349 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T15:38:13.350 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T15:38:13.351 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.089Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T15:38:13.351 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.090Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T15:38:13.351 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.091Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T15:38:13.351 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.091Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T15:38:13.351 INFO:journalctl@ceph.prometheus.a.smithi027.stdout:Sep 23 15:38:13 smithi027 conmon[179123]: ts=2023-09-23T15:38:13.091Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T15:38:13.731 DEBUG:teuthology.orchestra.run.smithi027:> sudo pkill -f 'journalctl -f -n 0 -u ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T15:38:13.768 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T15:38:13.768 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T15:38:13.768 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T15:38:15.193 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 15:38:14 smithi017 systemd[1]: Stopping Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:15.692 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 15:38:15 smithi017 conmon[200517]: ts=2023-09-23T15:38:15.248Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T15:38:16.105 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 15:38:15 smithi017 bash[708937]: 6f0275940b82f12ac4dc3f60554964fb377439d5260964abb25c076d140a571f 2023-09-23T15:38:16.105 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 15:38:15 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T15:38:16.105 INFO:journalctl@ceph.alertmanager.a.smithi017.stdout:Sep 23 15:38:15 smithi017 systemd[1]: Stopped Ceph alertmanager.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:38:16.442 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 15:38:16 smithi017 systemd[1]: Stopping Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:17.192 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 15:38:16 smithi017 bash[709219]: af1ee2972cfe39dd51babebb84f873468352b112f6cc69990ad9e08970e5386a 2023-09-23T15:38:17.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 15:38:16 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T15:38:17.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 15:38:17 smithi017 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T15:38:17.193 INFO:journalctl@ceph.node-exporter.a.smithi017.stdout:Sep 23 15:38:17 smithi017 systemd[1]: Stopped Ceph node-exporter.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:38:26.371 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T15:38:27.849 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 15:38:27 smithi027 systemd[1]: Stopping Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:28.542 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 15:38:28 smithi027 bash[412545]: 71fdd501b2ad3b0b035dd3c55ef7ed42de92d538027d90aac449e2e4eaab9068 2023-09-23T15:38:28.542 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 15:38:28 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T15:38:28.542 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 15:38:28 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T15:38:28.543 INFO:journalctl@ceph.node-exporter.b.smithi027.stdout:Sep 23 15:38:28 smithi027 systemd[1]: Stopped Ceph node-exporter.b for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:38:28.849 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:38:28 smithi027 systemd[1]: Stopping Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7... 2023-09-23T15:38:29.515 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:38:29 smithi027 bash[412823]: 4653950d68ab094923207bb5fe72d48a86a334bb7b3a7915dcdbcb321e2af267 2023-09-23T15:38:29.515 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:38:29 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T15:38:29.516 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:38:29 smithi027 systemd[1]: ceph-546d6668-5a20-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T15:38:29.516 INFO:journalctl@ceph.grafana.a.smithi027.stdout:Sep 23 15:38:29 smithi027 systemd[1]: Stopped Ceph grafana.a for 546d6668-5a20-11ee-8db4-212e2dc638e7. 2023-09-23T15:38:34.425 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T15:38:34.464 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T15:38:34.494 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T15:38:34.497 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi017/crash 2023-09-23T15:38:34.497 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T15:38:34.578 INFO:teuthology.orchestra.run.smithi017.stderr:tar: /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T15:38:34.579 INFO:teuthology.orchestra.run.smithi017.stderr:tar: Error is not recoverable: exiting now 2023-09-23T15:38:34.580 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi027/crash 2023-09-23T15:38:34.581 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T15:38:34.636 INFO:teuthology.orchestra.run.smithi027.stderr:tar: /var/lib/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T15:38:34.636 INFO:teuthology.orchestra.run.smithi027.stderr:tar: Error is not recoverable: exiting now 2023-09-23T15:38:34.637 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T15:38:34.637 DEBUG:teuthology.orchestra.run.smithi017:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T15:38:34.669 INFO:teuthology.orchestra.run.smithi017.stderr:grep: /var/log/ceph/546d6668-5a20-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T15:38:34.671 INFO:tasks.cephadm:Compressing logs... 2023-09-23T15:38:34.671 DEBUG:teuthology.orchestra.run.smithi017:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T15:38:34.713 DEBUG:teuthology.orchestra.run.smithi027:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T15:38:34.741 INFO:teuthology.orchestra.run.smithi017.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T15:38:34.765 INFO:teuthology.orchestra.run.smithi027.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T15:46:05.186 INFO:tasks.cephadm:Archiving logs... 2023-09-23T15:46:05.187 DEBUG:teuthology.misc:Transferring archived files from smithi017:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi017/log 2023-09-23T15:46:05.188 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T15:46:54.667 DEBUG:teuthology.misc:Transferring archived files from smithi027:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi027/log 2023-09-23T15:46:54.688 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T15:47:29.970 INFO:tasks.cephadm:Removing cluster... 2023-09-23T15:47:29.971 DEBUG:teuthology.orchestra.run.smithi017:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 --force 2023-09-23T15:47:30.950 DEBUG:teuthology.orchestra.run.smithi027:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 546d6668-5a20-11ee-8db4-212e2dc638e7 --force 2023-09-23T15:47:31.865 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T15:47:31.865 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T15:47:31.887 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T15:47:31.916 INFO:tasks.cephadm:Teardown complete 2023-09-23T15:47:31.917 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T15:47:31.926 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T15:47:31.927 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T15:47:31.930 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T15:47:32.040 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T15:47:32.050 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T15:47:32.050 DEBUG:teuthology.orchestra.run.smithi017:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T15:47:32.053 DEBUG:teuthology.orchestra.run.smithi027:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T15:47:32.068 INFO:teuthology.orchestra.run.smithi027.stderr:bash: ntpq: command not found 2023-09-23T15:47:32.070 INFO:teuthology.orchestra.run.smithi017.stderr:bash: ntpq: command not found 2023-09-23T15:47:32.082 INFO:teuthology.orchestra.run.smithi017.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T15:47:32.082 INFO:teuthology.orchestra.run.smithi017.stdout:=============================================================================== 2023-09-23T15:47:32.082 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 14 +901us[ +901us] +/- 95ms 2023-09-23T15:47:32.082 INFO:teuthology.orchestra.run.smithi017.stdout:^* hv02.front.sepia.ceph.com 3 6 377 15 -903us[ -909us] +/- 77ms 2023-09-23T15:47:32.083 INFO:teuthology.orchestra.run.smithi017.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 16 +198us[ +192us] +/- 73ms 2023-09-23T15:47:32.083 INFO:teuthology.orchestra.run.smithi017.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T15:47:32.090 INFO:teuthology.orchestra.run.smithi027.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T15:47:32.090 INFO:teuthology.orchestra.run.smithi027.stdout:=============================================================================== 2023-09-23T15:47:32.090 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 15 +890us[ +890us] +/- 95ms 2023-09-23T15:47:32.091 INFO:teuthology.orchestra.run.smithi027.stdout:^* hv02.front.sepia.ceph.com 3 6 377 16 -923us[ -923us] +/- 77ms 2023-09-23T15:47:32.091 INFO:teuthology.orchestra.run.smithi027.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 13 +180us[ +180us] +/- 73ms 2023-09-23T15:47:32.091 INFO:teuthology.orchestra.run.smithi027.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T15:47:32.092 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T15:47:32.101 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T15:47:32.101 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T15:47:32.110 DEBUG:teuthology.orchestra.run.smithi017:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T15:47:32.251 DEBUG:teuthology.orchestra.run.smithi027:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T15:47:32.352 DEBUG:teuthology.orchestra.run.smithi017:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T15:47:32.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:47:32.386 DEBUG:teuthology.orchestra.run.smithi027:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T15:47:32.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:47:32.417 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T15:47:32.427 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T15:47:32.437 INFO:teuthology.task.internal:Duration was 4560.220421 seconds 2023-09-23T15:47:32.437 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T15:47:32.447 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T15:47:32.447 DEBUG:teuthology.orchestra.run.smithi017:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T15:47:32.449 DEBUG:teuthology.orchestra.run.smithi027:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T15:47:32.524 INFO:teuthology.orchestra.run.smithi027.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T15:47:32.533 INFO:teuthology.orchestra.run.smithi017.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T15:47:32.724 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T15:47:32.724 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi017.front.sepia.ceph.com 2023-09-23T15:47:32.724 DEBUG:teuthology.orchestra.run.smithi017:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T15:47:32.747 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi027.front.sepia.ceph.com 2023-09-23T15:47:32.747 DEBUG:teuthology.orchestra.run.smithi027:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T15:47:32.775 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T15:47:32.775 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T15:47:32.790 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T15:47:32.953 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T15:47:32.978 DEBUG:teuthology.orchestra.run.smithi017:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T15:47:32.981 DEBUG:teuthology.orchestra.run.smithi027:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T15:47:33.349 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T15:47:33.359 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T15:47:33.360 DEBUG:teuthology.orchestra.run.smithi017:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T15:47:33.395 DEBUG:teuthology.orchestra.run.smithi027:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T15:47:33.446 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T15:47:33.456 DEBUG:teuthology.orchestra.run.smithi017:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T15:47:33.458 DEBUG:teuthology.orchestra.run.smithi027:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T15:47:33.495 INFO:teuthology.orchestra.run.smithi027.stdout:kernel.core_pattern = core 2023-09-23T15:47:33.499 INFO:teuthology.orchestra.run.smithi017.stdout:kernel.core_pattern = core 2023-09-23T15:47:33.527 DEBUG:teuthology.orchestra.run.smithi017:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T15:47:33.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:47:33.570 DEBUG:teuthology.orchestra.run.smithi027:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T15:47:33.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:47:33.586 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T15:47:33.596 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T15:47:33.596 DEBUG:teuthology.misc:Transferring archived files from smithi017:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi017 2023-09-23T15:47:33.597 DEBUG:teuthology.orchestra.run.smithi017:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T15:47:33.667 DEBUG:teuthology.misc:Transferring archived files from smithi027:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691/remote/smithi027 2023-09-23T15:47:33.667 DEBUG:teuthology.orchestra.run.smithi027:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T15:47:33.711 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T15:47:33.711 DEBUG:teuthology.orchestra.run.smithi017:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T15:47:33.714 DEBUG:teuthology.orchestra.run.smithi027:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T15:47:33.759 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T15:47:33.772 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T15:47:33.772 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T15:47:33.782 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T15:47:33.782 DEBUG:teuthology.orchestra.run.smithi017:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T15:47:33.784 DEBUG:teuthology.orchestra.run.smithi027:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T15:47:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 15:47 /home/ubuntu/cephtest 2023-09-23T15:47:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: 3276801 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 15:34 /home/ubuntu/cephtest/mnt.0 2023-09-23T15:47:33.799 INFO:teuthology.orchestra.run.smithi017.stdout: 3276802 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 15:34 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T15:47:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T15:47:33.814 INFO:teuthology.orchestra.run.smithi017.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T15:47:33.815 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T15:47:33.821 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T15:47:33.832 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T15:47:33.843 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi017 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T15:47:33.843 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T15:47:33.849 INFO:teuthology.orchestra.run.smithi027.stdout: 262307 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 15:47 /home/ubuntu/cephtest 2023-09-23T15:47:33.883 DEBUG:teuthology.nuke:shortname: smithi017 2023-09-23T15:47:33.884 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T15:47:33.907 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi017.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 14:24:40.326010', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T15:47:33.910 DEBUG:teuthology.nuke:shortname: smithi027 2023-09-23T15:47:33.910 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T15:47:33.930 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi027.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401691', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 14:24:40.324539', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T15:47:33.964 INFO:teuthology.orchestra.console.smithi017:Power off 2023-09-23T15:47:33.964 DEBUG:teuthology.orchestra.console.smithi017:pexpect command: ipmitool -H smithi017.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T15:47:33.977 INFO:teuthology.orchestra.console.smithi027:Power off 2023-09-23T15:47:33.977 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T15:47:33.986 DEBUG:teuthology.orchestra.console.smithi017:power off output: Chassis Power Control: Down/Off 2023-09-23T15:47:33.986 DEBUG:teuthology.orchestra.console.smithi017:pexpect command: ipmitool -H smithi017.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:33.999 DEBUG:teuthology.orchestra.console.smithi027:power off output: Chassis Power Control: Down/Off 2023-09-23T15:47:33.999 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:34.008 DEBUG:teuthology.orchestra.console.smithi017:check power output: Chassis Power is on 2023-09-23T15:47:34.021 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is on 2023-09-23T15:47:38.009 DEBUG:teuthology.orchestra.console.smithi017:pexpect command: ipmitool -H smithi017.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:38.022 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:38.121 DEBUG:teuthology.orchestra.console.smithi017:check power output: Chassis Power is on 2023-09-23T15:47:38.133 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is on 2023-09-23T15:47:42.126 DEBUG:teuthology.orchestra.console.smithi017:pexpect command: ipmitool -H smithi017.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:42.134 DEBUG:teuthology.orchestra.console.smithi027:pexpect command: ipmitool -H smithi027.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T15:47:42.236 DEBUG:teuthology.orchestra.console.smithi017:check power output: Chassis Power is off 2023-09-23T15:47:42.243 DEBUG:teuthology.orchestra.console.smithi027:check power output: Chassis Power is off 2023-09-23T15:47:42.338 INFO:teuthology.orchestra.console.smithi017:Power off completed 2023-09-23T15:47:42.343 INFO:teuthology.orchestra.console.smithi027:Power off completed 2023-09-23T15:47:42.448 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_3.0 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 4560.220421075821 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e5e7fd49546b47c9b5afb84d33b8f91c status: fail success: false 2023-09-23T15:47:42.448 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T15:47:42.525 INFO:teuthology.run:FAIL